↰ Return to documentation for file (morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp
)
#pragma once
#include "morpheus/messages/multi.hpp"
#include "morpheus/messages/multi_inference.hpp"
#include <mrc/channel/status.hpp>// for Status
#include <mrc/node/sink_properties.hpp>// for SinkProperties<>::sink_type_t
#include <mrc/node/source_properties.hpp>// for SourceProperties<>::source_type_t
#include <mrc/segment/builder.hpp>
#include <mrc/segment/object.hpp>// for Object
#include <pymrc/node.hpp>
#include <rxcpp/rx.hpp>// for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, from
#include <cstdint>// for uint32_t
#include <memory>
#include <string>
#include <vector>
namespace morpheus {
/****** Component public implementations *******************/
/****** PreprocessNLPStage**********************************/
#pragma GCC visibility push(default)
class PreprocessNLPStage
: public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiInferenceMessage>>
{
public:
using base_t = mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiInferenceMessage>>;
using typename base_t::sink_type_t;
using typename base_t::source_type_t;
using typename base_t::subscribe_fn_t;
PreprocessNLPStage(std::string vocab_hash_file,
uint32_t sequence_length,
bool truncation,
bool do_lower_case,
bool add_special_token,
int stride = -1,
std::string column = "data");
private:
subscribe_fn_t build_operator();
std::string m_vocab_hash_file;
std::string m_column;
uint32_t m_sequence_length;
bool m_truncation;
bool m_do_lower_case;
bool m_add_special_token;
int m_stride{-1};
};
/****** PreprocessNLPStageInferenceProxy********************/
struct PreprocessNLPStageInterfaceProxy
{
static std::shared_ptr<mrc::segment::Object<PreprocessNLPStage>> init(mrc::segment::Builder& builder,
const std::string& name,
std::string vocab_hash_file,
uint32_t sequence_length,
bool truncation,
bool do_lower_case,
bool add_special_token,
int stride = -1,
std::string column = "data");
};
#pragma GCC visibility pop// end of group
} // namespace morpheus