Detailed Description

Performs target segmentation and maintains temporal model state.

Definition at line 253 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

Inheritance diagram for Segmenter:

Collaboration diagram for Segmenter:

[legend]

Public Member Functions
	Segmenter (const std::string &configFilePath)
	Construct a Segmenter instance from configuration. More...

	Segmenter (const std::string &configFilePath, const std::string &modelName)
	Construct a Segmenter instance from configuration. More...

virtual	~Segmenter ()
	Release resources owned by the Segmenter instance. More...

void	initialize (const SourceFrameInfo &frameInfo, const uint maxTargetsPerStream, const std::shared_ptr< NvTrackedObjectManager > &targetManager) override
	Initialize variables and allocate memories. More...

void	queryParams (NvMOTQuery &paramsQuery) override
	Populate the capabilities reported to the tracker. More...

void	localize (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta) override
	Localize and segment target in a new frame. More...

void	update (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta) override
	Update Segmenter after data association. More...

void	outputMaskObj (const std::shared_ptr< NvTrackedObject > target, NvMOTMask &mask) override
	Output segmentation mask of a target. More...

void	removeStream (const NvMOTStreamId videoStreamId) override
	Remove a stream's Segmenter features. More...

bool	isInitialized () override
	Report whether the module is initialized. More...

const SEGMENTER_TYPES	getSegmenterType () override
	Get the configured segmenter type. More...

	Segmenter (const std::string &configFilePath)
	Construct a Segmenter instance from configuration. More...

	Segmenter (const std::string &configFilePath, const std::string &modelName)
	Construct a Segmenter instance from configuration. More...

virtual	~Segmenter ()
	Release resources owned by the Segmenter instance. More...

void	initialize (const SourceFrameInfo &frameInfo, const uint maxTargetsPerStream, const std::shared_ptr< NvTrackedObjectManager > &targetManager) override
	Initialize variables and allocate memories. More...

void	queryParams (NvMOTQuery &paramsQuery) override
	Populate the capabilities reported to the tracker. More...

void	localize (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta) override
	Localize and segment target in a new frame. More...

void	update (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta) override
	Update Segmenter after data association. More...

void	outputMaskObj (const std::shared_ptr< NvTrackedObject > target, NvMOTMask &mask) override
	Output segmentation mask of a target. More...

void	removeStream (const NvMOTStreamId videoStreamId) override
	Remove a stream's Segmenter features. More...

bool	isInitialized () override
	Report whether the module is initialized. More...

const SEGMENTER_TYPES	getSegmenterType () override
	Get the configured segmenter type. More...

Protected Member Functions
void	doInference (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta)
	Run segmentation inference for targets in the input batch. More...

void	extractImagePatchesBatch (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta)
	Rescale low-level tracker frames into encoder input. More...

void	fillMemoryConditionedFeatures (TrackerIndex targetInd, int indInBatch)
	Construct memory attention input. More...

void	fillCudaMemoryConditionedFeatures (TrackerIndex targetInd, int indInBatch)
	Fill memory conditioned features for cuda memory bank. More...

void	initializeTrtBindings ()
	Define IO bindings for each TRT engine. More...

void	updateTrtBindingsPerStream (const uint inputRoiInd)
	Update TRT bindings per stream. More...

void	batchInfer (const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &targetList, bool usePrompt, int outputLabelIndMax, const NvMOTFrameNum frameNum, const uint inputRoiInd)
	Batch inference. More...

void	singleInfer (const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &targetList, bool usePrompt, int outputLabelIndMax, const NvMOTFrameNum frameNum, const uint inputRoiInd)
	Run segmentation for one target. More...

void	imgEncoderInfer ()
	Run the image encoder for the current input batch. More...

void	maskDecoderInfer (bool usePrompt, std::vector< std::shared_ptr< NvTrackedObject >> &targetList, int &outputLabelIndMax, const uint inputStreamInd, const NvMOTFrameNum frameNum)
	Run mask-decoder inference for the current target batch. More...

void	memAttenInfer (const std::shared_ptr< NvTrackedObject > target, const int indInBatch, const int inputStreamInd)
	Run memory attention for one tracked target. More...

void	memAttenInferBatchParser (std::vector< std::shared_ptr< NvTrackedObject >> &targetList, const int inputStreamInd)
	Assemble and run batched memory-attention input. More...

void	memEncoderInfer (const int batchSizeActual)
	Run the memory encoder for the current target batch. More...

void	setMaskDecoderPointLabels (int promptType)
	Populate mask-decoder labels for points or bounding boxes. More...

void	setMaskDecoderSinglePointLabelAsNegative (int pointNumPerTarget, int indInBatch)
	Mark a single decoder prompt point as negative. More...

void	setMaskDecoderPoints (const bool usePrompt, std::vector< std::shared_ptr< NvTrackedObject >> targetList)
	Populate mask-decoder prompt coordinates for a target batch. More...

void	postProcessing (const int indInBatch, const std::shared_ptr< NvTrackedObject > target, const NvMOTFrameNum frameNum)
	Convert one inference result into retained target state. More...

void	updateInferStatus (const int indInBatch, const std::shared_ptr< NvTrackedObject > target, const NvMOTFrameNum frameNum)
	Record a target in the pending inference batch. More...

void	removeInferStatus ()
	Remove terminated memory bank. More...

void	copyFromCuda (float *src, TensorPtr &dst, int numElements, cudaStream_t &cuStream)
	Copy a CUDA buffer into a TensorRT tensor. More...

bool	isInferStatusEmpty (TrackerIndex targetInd)
	Check whether a target has no retained inference state. More...

bool	skipInfer (std::shared_ptr< NvTrackedObject > target, const NvMOTFrameNum frameNum)
	Determine whether segmentation can be skipped for a target. More...

bool	isGoodBboxProg (const std::shared_ptr< NvTrackedObject > target, const float confidence, const NvMotDataTypes::Rect2f bboxProg, const float iouThresh=0.5)
	Check whether a propagated bounding box is reliable. More...

void	doInference (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta)
	Run segmentation inference for targets in the input batch. More...

void	extractImagePatchesBatch (const std::map< NvMOTStreamId, NvMOTFrame * > &inputMeta)
	Rescale low-level tracker frames into encoder input. More...

void	fillMemoryConditionedFeatures (TrackerIndex targetInd, int indInBatch)
	Construct memory attention input. More...

void	fillCudaMemoryConditionedFeatures (TrackerIndex targetInd, int indInBatch)
	Fill memory conditioned features for cuda memory bank. More...

void	initializeTrtBindings ()
	Define IO bindings for each TRT engine. More...

void	updateTrtBindingsPerStream (const uint inputRoiInd)
	Update TRT bindings per stream. More...

void	batchInfer (const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &targetList, bool usePrompt, int outputLabelIndMax, const NvMOTFrameNum frameNum, const uint inputRoiInd)
	Batch inference. More...

void	singleInfer (const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &targetList, bool usePrompt, int outputLabelIndMax, const NvMOTFrameNum frameNum, const uint inputRoiInd)
	Run segmentation for one target. More...

void	imgEncoderInfer ()
	Run the image encoder for the current input batch. More...

void	maskDecoderInfer (bool usePrompt, std::vector< std::shared_ptr< NvTrackedObject >> &targetList, int &outputLabelIndMax, const uint inputStreamInd, const NvMOTFrameNum frameNum)
	Run mask-decoder inference for the current target batch. More...

void	memAttenInfer (const std::shared_ptr< NvTrackedObject > target, const int indInBatch, const int inputStreamInd)
	Run memory attention for one tracked target. More...

void	memAttenInferBatchParser (std::vector< std::shared_ptr< NvTrackedObject >> &targetList, const int inputStreamInd)
	Assemble and run batched memory-attention input. More...

void	memEncoderInfer (const int batchSizeActual)
	Run the memory encoder for the current target batch. More...

void	setMaskDecoderPointLabels (int promptType)
	Populate mask-decoder labels for points or bounding boxes. More...

void	setMaskDecoderSinglePointLabelAsNegative (int pointNumPerTarget, int indInBatch)
	Mark a single decoder prompt point as negative. More...

void	setMaskDecoderPoints (const bool usePrompt, std::vector< std::shared_ptr< NvTrackedObject >> targetList)
	Populate mask-decoder prompt coordinates for a target batch. More...

void	postProcessing (const int indInBatch, const std::shared_ptr< NvTrackedObject > target, const NvMOTFrameNum frameNum)
	Convert one inference result into retained target state. More...

void	updateInferStatus (const int indInBatch, const std::shared_ptr< NvTrackedObject > target, const NvMOTFrameNum frameNum)
	Record a target in the pending inference batch. More...

void	removeInferStatus ()
	Remove terminated memory bank. More...

void	copyFromCuda (float *src, TensorPtr &dst, int numElements, cudaStream_t &cuStream)
	Copy a CUDA buffer into a TensorRT tensor. More...

bool	isInferStatusEmpty (TrackerIndex targetInd)
	Check whether a target has no retained inference state. More...

bool	skipInfer (std::shared_ptr< NvTrackedObject > target, const NvMOTFrameNum frameNum)
	Determine whether segmentation can be skipped for a target. More...

bool	isGoodBboxProg (const std::shared_ptr< NvTrackedObject > target, const float confidence, const NvMotDataTypes::Rect2f bboxProg, const float iouThresh=0.5)
	Check whether a propagated bounding box is reliable. More...

Protected Attributes
bool	m_isInit
	Segmenter status if initialized. More...

bool	m_isForLocalization
	Whether the inference is for localization. More...

SegmenterConfigParams	m_params
	Config params for Segmenter module. More...

SegmenterRunTimeParams	m_paramsRt
	Runtime parameters. More...

float *	m_directly_add_no_mem_embed_d
	Non-memory embeddings added to image-encoder output. More...

cudaStream_t	m_cuStream
	Cuda stream to run all GPU tasks. More...

cublasHandle_t	m_cublasHandle
	Cublas handle. More...

TrtModule	m_imageEncoder
	Encoder to extract features from each image. More...

TrtModule	m_maskDecoder
	Decoder taking in prompts (points, bboxes, etc.) to create segmentation mask. More...

TrtModule	m_memoryAttention
	Memory attention module. More...

TrtModule	m_memoryEncoder
	Memory encoder module. More...

float *	m_inputBuf_d

float *	m_inputBuf_h
	Input patches to encoder. More...

float *	m_featuresEncoder_d

float *	m_highResFeaturesEncoder0_d

float *	m_highResFeaturesEncoder1_d
	Image encoder output feature. More...

float *	m_visionPosEmbed_d

float *	m_visionFeats_d

float *	m_highResImgPosEmbed1_d
	Image encoder output position embedding. More...

float *	m_work_0_d

float *	m_work_1_d

float *	m_work_2_d
	Scratch buffers for memory attention. More...

float *	m_memory_obj_ptr_d

float *	m_memory_maskmem_features_d

float *	m_memory_d

float *	m_memory_pos_embed_d

float *	m_cond_frame_id_diff_d
	Memory attention inputs: object pointers, mask memory features, memory, positional embeddings, and condition-frame ID differences. More...

float *	m_pix_feat_with_mem_d
	Attention output features with memory for one target. More...

float *	m_batchPixFeatWithMem_d
	Attention output features with memory for all targets in all streams. More...

float *	m_points_d
	Input a list of points (prompt) to decoder. More...

float *	m_pointLabels_d
	Input a list of point types (single point, bounding box corners) to decoder. More...

float *	m_inputMask_d
	Input mask prompt, currently unused and set as 0. More...

float *	m_inputMaskLabel_d
	Input mask label, currently unused and set as 0. More...

float *	m_targetMaskDecSize_d

float *	m_targetMaskDecSize_h
	Output mask basic resolution. More...

float *	m_maskForMem_d
	Output mask high resolution. More...

float *	m_obj_ptr_d
	Output obj ptrs. More...

float *	m_outputIou_d
	Output IOU score. More...

float *	m_outputOccLogit_d
	Output occlusion logit; <2 means target is occluded. More...

std::vector< float >	m_outputIou_h
	Output IOU score on host. More...

std::vector< float >	m_outputOccLogit_h
	Output occlusion logit on host. More...

int *	m_outputSize_d
	Output size. More...

float *	m_maskmem_features_d

float *	m_maskmem_pos_enc_d

float *	m_temporal_code_d
	Output mask memory features and pos encoding. More...

float *	m_batchMaskDecSize_d
	Mask output for all targets in all streams. More...

float *	m_targetMaskOriginalSrcSize_d
	Mask for each target resized to original streammux size. More...

Eigen::Vector2f	m_scaleMask2Src
	Scale from mask output to the source image. More...

Eigen::Vector2f	m_scaleInput2Original
	Scale from decoder output to original frame size decided by streammux. More...

NvMotDataTypes::Size	m_srcImageSize
	Input to low level tracker. More...

NvMotDataTypes::Size	m_originalSrcImageSize
	Input to tracker plugin, usually from streammux. More...

std::shared_ptr< NvTrackedObjectManager >	m_targetManager
	Target manager providing target status to update feature gallery. More...

std::shared_ptr< ReIDFrameTransformTexture >	m_subImageCropper2
	Image cropper, may share with visual tracker in the future. More...

std::shared_ptr< ReIDCropScaler >	m_subImageCropScaler
	Image crop scaler. More...

std::map< NvMOTStreamId, std::vector< std::pair< uint, NvMotDataTypes::Rect2f > > >	m_inputRoisInBatch
	Input rois to crop. More...

std::map< TrackerIndex, std::map< NvMOTFrameNum, SegmentMemoryInfo > >	m_condFrameOutputs
	Outputs of the conditional frames. More...

std::map< TrackerIndex, std::map< NvMOTFrameNum, SegmentMemoryInfo > >	m_nonCondFrameOutputs
	Outputs of the non-conditional frames. More...

std::map< TrackerIndex, InferenceStatus >	m_inferStatus
	Inference status. More...

std::map< TrackerIndex, CudaInferenceStatus >	m_cudaInferenceStatus
	Cuda inference status. More...

std::map< TrackerIndex, std::vector< float > >	m_targetScores
	Target IOU and occlusion scores. More...

SegmenterCudaMemBank	m_cudaMemBank
	Cuda memory bank. More...

std::vector< void * >	m_bindingsImgEncoder

std::vector< void * >	m_bindingsMaskDecoder

std::vector< void * >	m_bindingsMemAtten

std::vector< void * >	m_bindingsMemEncoder
	TensorRT bindings. More...

Constructor & Destructor Documentation

◆ Segmenter() [1/4]

Segmenter::Segmenter ( const std::string & configFilePath )

Construct a Segmenter instance from configuration.

Parameters

[in] configFilePath Path to the tracker configuration file.

◆ Segmenter() [2/4]

Segmenter::Segmenter	(	const std::string &	configFilePath,
		const std::string &	modelName
	)

Construct a Segmenter instance from configuration.

Parameters

[in]	configFilePath	Path to the tracker configuration file.
[in]	modelName	Configured model name.

◆ ~Segmenter() [1/2]

virtual Segmenter::~Segmenter ( )

virtual

Release resources owned by the Segmenter instance.

◆ Segmenter() [3/4]

Segmenter::Segmenter ( const std::string & configFilePath )

Construct a Segmenter instance from configuration.

Parameters

[in] configFilePath Path to the tracker configuration file.

◆ Segmenter() [4/4]

Segmenter::Segmenter	(	const std::string &	configFilePath,
		const std::string &	modelName
	)

Construct a Segmenter instance from configuration.

Parameters

[in]	configFilePath	Path to the tracker configuration file.
[in]	modelName	Configured model name.

◆ ~Segmenter() [2/2]

virtual Segmenter::~Segmenter ( )

virtual

Release resources owned by the Segmenter instance.

Member Function Documentation

◆ batchInfer() [1/2]

void Segmenter::batchInfer	(	const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &	targetList,
		bool	usePrompt,
		int	outputLabelIndMax,
		const NvMOTFrameNum	frameNum,
		const uint	inputRoiInd
	)

protected

Batch inference.

Parameters

[in]	targetList	Targets included in the inference operation.
[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	outputLabelIndMax	Highest decoder output-label index to process.
[in]	frameNum	Frame number associated with the inference result.
[in]	inputRoiInd	Region index within the input batch.

◆ batchInfer() [2/2]

void Segmenter::batchInfer	(	const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &	targetList,
		bool	usePrompt,
		int	outputLabelIndMax,
		const NvMOTFrameNum	frameNum,
		const uint	inputRoiInd
	)

protected

Batch inference.

Parameters

[in]	targetList	Targets included in the inference operation.
[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	outputLabelIndMax	Highest decoder output-label index to process.
[in]	frameNum	Frame number associated with the inference result.
[in]	inputRoiInd	Region index within the input batch.

◆ copyFromCuda() [1/2]

void Segmenter::copyFromCuda	(	float *	src,
		TensorPtr &	dst,
		int	numElements,
		cudaStream_t &	cuStream
	)

protected

Copy a CUDA buffer into a TensorRT tensor.

Parameters

[in]	src	The source tensor.
[out]	dst	Destination tensor to populate.
[in]	numElements	Number of tensor elements to copy.
[in]	cuStream	CUDA stream that runs the GPU work.

◆ copyFromCuda() [2/2]

void Segmenter::copyFromCuda	(	float *	src,
		TensorPtr &	dst,
		int	numElements,
		cudaStream_t &	cuStream
	)

protected

Copy a CUDA buffer into a TensorRT tensor.

Parameters

[in]	src	The source tensor.
[out]	dst	Destination tensor to populate.
[in]	numElements	Number of tensor elements to copy.
[in]	cuStream	CUDA stream that runs the GPU work.

◆ doInference() [1/2]

void Segmenter::doInference ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

protected

Run segmentation inference for targets in the input batch.

Parameters

[in] inputMeta A batch of input frames.

◆ doInference() [2/2]

void Segmenter::doInference ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

protected

Run segmentation inference for targets in the input batch.

Parameters

[in] inputMeta A batch of input frames.

◆ extractImagePatchesBatch() [1/2]

void Segmenter::extractImagePatchesBatch ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

protected

Rescale low-level tracker frames into encoder input.

Parameters

[in] inputMeta Input frames keyed by stream ID.

◆ extractImagePatchesBatch() [2/2]

void Segmenter::extractImagePatchesBatch ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

protected

Rescale low-level tracker frames into encoder input.

Parameters

[in] inputMeta Input frames keyed by stream ID.

◆ fillCudaMemoryConditionedFeatures() [1/2]

void Segmenter::fillCudaMemoryConditionedFeatures	(	TrackerIndex	targetInd,
		int	indInBatch
	)

protected

Fill memory conditioned features for cuda memory bank.

Parameters

[in]	targetInd	Index of the target.
[in]	indInBatch	Index of this target in the inference batch.

◆ fillCudaMemoryConditionedFeatures() [2/2]

void Segmenter::fillCudaMemoryConditionedFeatures	(	TrackerIndex	targetInd,
		int	indInBatch
	)

protected

Fill memory conditioned features for cuda memory bank.

Parameters

[in]	targetInd	Index of the target.
[in]	indInBatch	Index of this target in the inference batch.

◆ fillMemoryConditionedFeatures() [1/2]

void Segmenter::fillMemoryConditionedFeatures	(	TrackerIndex	targetInd,
		int	indInBatch
	)

protected

Construct memory attention input.

Parameters

[in]	targetInd	Index of the target.
[in]	indInBatch	Index of this target in the inference batch.

◆ fillMemoryConditionedFeatures() [2/2]

void Segmenter::fillMemoryConditionedFeatures	(	TrackerIndex	targetInd,
		int	indInBatch
	)

protected

Construct memory attention input.

Parameters

[in]	targetInd	Index of the target.
[in]	indInBatch	Index of this target in the inference batch.

◆ getSegmenterType() [1/2]

const SEGMENTER_TYPES Segmenter::getSegmenterType ( )

inlineoverridevirtual

Get the configured segmenter type.

Implements ISegmenter.

Definition at line 318 of file 9.2/sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

References m_params, and SegmenterConfigParams::m_segmenterType.

◆ getSegmenterType() [2/2]

const SEGMENTER_TYPES Segmenter::getSegmenterType ( )

inlineoverridevirtual

Get the configured segmenter type.

Implements ISegmenter.

Definition at line 318 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

References m_params, and SegmenterConfigParams::m_segmenterType.

◆ imgEncoderInfer() [1/2]

void Segmenter::imgEncoderInfer ( )

protected

Run the image encoder for the current input batch.

◆ imgEncoderInfer() [2/2]

void Segmenter::imgEncoderInfer ( )

protected

Run the image encoder for the current input batch.

◆ initialize() [1/2]

void Segmenter::initialize	(	const SourceFrameInfo &	frameInfo,
		const uint	maxTargetsPerStream,
		const std::shared_ptr< NvTrackedObjectManager > &	targetManager
	)

overridevirtual

Initialize variables and allocate memories.

Parameters

[in]	frameInfo	Metadata about the source image, such as size and color.
[in]	maxTargetsPerStream	Maximum number of targets to track in each stream.
[in]	targetManager	Target manager of the tracked objects.

Implements ISegmenter.

◆ initialize() [2/2]

void Segmenter::initialize	(	const SourceFrameInfo &	frameInfo,
		const uint	maxTargetsPerStream,
		const std::shared_ptr< NvTrackedObjectManager > &	targetManager
	)

overridevirtual

Initialize variables and allocate memories.

Parameters

[in]	frameInfo	Metadata about the source image, such as size and color.
[in]	maxTargetsPerStream	Maximum number of targets to track in each stream.
[in]	targetManager	Target manager of the tracked objects.

Implements ISegmenter.

◆ initializeTrtBindings() [1/2]

void Segmenter::initializeTrtBindings ( )

protected

Define IO bindings for each TRT engine.

◆ initializeTrtBindings() [2/2]

void Segmenter::initializeTrtBindings ( )

protected

Define IO bindings for each TRT engine.

◆ isGoodBboxProg() [1/2]

bool Segmenter::isGoodBboxProg	(	const std::shared_ptr< NvTrackedObject >	target,
		const float	confidence,
		const NvMotDataTypes::Rect2f	bboxProg,
		const float	iouThresh = `0.5`
	)

inlineprotected

Check whether a propagated bounding box is reliable.

Parameters

[in]	target	Tracked target to process.
[in]	confidence	Target confidence used to validate propagation.
[in]	bboxProg	Propagated target bounding box.
[in]	iouThresh	Minimum intersection-over-union threshold.

◆ isGoodBboxProg() [2/2]

bool Segmenter::isGoodBboxProg	(	const std::shared_ptr< NvTrackedObject >	target,
		const float	confidence,
		const NvMotDataTypes::Rect2f	bboxProg,
		const float	iouThresh = `0.5`
	)

inlineprotected

Check whether a propagated bounding box is reliable.

Parameters

[in]	target	Tracked target to process.
[in]	confidence	Target confidence used to validate propagation.
[in]	bboxProg	Propagated target bounding box.
[in]	iouThresh	Minimum intersection-over-union threshold.

◆ isInferStatusEmpty() [1/2]

bool Segmenter::isInferStatusEmpty ( TrackerIndex targetInd )

inlineprotected

Check whether a target has no retained inference state.

Parameters

[in] targetInd Index of the target.

Returns: True if the infer status is empty, false otherwise.

Definition at line 545 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

References m_cudaInferenceStatus, m_inferStatus, m_params, SegmenterConfigParams::m_useCudaMemBank, CudaInferenceStatus::numStatusUncond, InferenceStatus::statusCond, and InferenceStatus::statusUncond.

◆ isInferStatusEmpty() [2/2]

bool Segmenter::isInferStatusEmpty ( TrackerIndex targetInd )

inlineprotected

Check whether a target has no retained inference state.

Parameters

[in] targetInd Index of the target.

Returns: True if the infer status is empty, false otherwise.

Definition at line 545 of file 9.2/sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

References m_cudaInferenceStatus, m_inferStatus, m_params, SegmenterConfigParams::m_useCudaMemBank, CudaInferenceStatus::numStatusUncond, InferenceStatus::statusCond, and InferenceStatus::statusUncond.

◆ isInitialized() [1/2]

bool Segmenter::isInitialized ( )

inlineoverridevirtual

Report whether the module is initialized.

Implements ISegmenter.

Definition at line 315 of file 9.2/sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

References m_isInit.

◆ isInitialized() [2/2]

bool Segmenter::isInitialized ( )

inlineoverridevirtual

Report whether the module is initialized.

Implements ISegmenter.

Definition at line 315 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

References m_isInit.

◆ localize() [1/2]

void Segmenter::localize ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

overridevirtual

Localize and segment target in a new frame.

Parameters

[in] inputMeta A batch of input frames.

Implements ISegmenter.

◆ localize() [2/2]

void Segmenter::localize ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

overridevirtual

Localize and segment target in a new frame.

Parameters

[in] inputMeta A batch of input frames.

Implements ISegmenter.

◆ maskDecoderInfer() [1/2]

void Segmenter::maskDecoderInfer	(	bool	usePrompt,
		std::vector< std::shared_ptr< NvTrackedObject >> &	targetList,
		int &	outputLabelIndMax,
		const uint	inputStreamInd,
		const NvMOTFrameNum	frameNum
	)

protected

Run mask-decoder inference for the current target batch.

Parameters

[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	targetList	Targets included in the inference operation.
[out]	outputLabelIndMax	Index of the output label with the highest IoU.
[in]	inputStreamInd	Index of the input stream being processed.
[in]	frameNum	Frame number associated with the inference result.

◆ maskDecoderInfer() [2/2]

void Segmenter::maskDecoderInfer	(	bool	usePrompt,
		std::vector< std::shared_ptr< NvTrackedObject >> &	targetList,
		int &	outputLabelIndMax,
		const uint	inputStreamInd,
		const NvMOTFrameNum	frameNum
	)

protected

Run mask-decoder inference for the current target batch.

Parameters

[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	targetList	Targets included in the inference operation.
[out]	outputLabelIndMax	Index of the output label with the highest IoU.
[in]	inputStreamInd	Index of the input stream being processed.
[in]	frameNum	Frame number associated with the inference result.

◆ memAttenInfer() [1/2]

void Segmenter::memAttenInfer	(	const std::shared_ptr< NvTrackedObject >	target,
		const int	indInBatch,
		const int	inputStreamInd
	)

protected

Run memory attention for one tracked target.

Parameters

[in]	target	Tracked target to process.
[in]	indInBatch	Target index within the inference batch.
[in]	inputStreamInd	Index of the input stream being processed.

◆ memAttenInfer() [2/2]

void Segmenter::memAttenInfer	(	const std::shared_ptr< NvTrackedObject >	target,
		const int	indInBatch,
		const int	inputStreamInd
	)

protected

Run memory attention for one tracked target.

Parameters

[in]	target	Tracked target to process.
[in]	indInBatch	Target index within the inference batch.
[in]	inputStreamInd	Index of the input stream being processed.

◆ memAttenInferBatchParser() [1/2]

void Segmenter::memAttenInferBatchParser	(	std::vector< std::shared_ptr< NvTrackedObject >> &	targetList,
		const int	inputStreamInd
	)

protected

Assemble and run batched memory-attention input.

Parameters

[in]	targetList	Targets included in the inference operation.
[in]	inputStreamInd	Index of the input stream being processed.

◆ memAttenInferBatchParser() [2/2]

void Segmenter::memAttenInferBatchParser	(	std::vector< std::shared_ptr< NvTrackedObject >> &	targetList,
		const int	inputStreamInd
	)

protected

Assemble and run batched memory-attention input.

Parameters

[in]	targetList	Targets included in the inference operation.
[in]	inputStreamInd	Index of the input stream being processed.

◆ memEncoderInfer() [1/2]

void Segmenter::memEncoderInfer ( const int batchSizeActual )

protected

Run the memory encoder for the current target batch.

Parameters

[in] batchSizeActual Number of targets in the current batch.

◆ memEncoderInfer() [2/2]

void Segmenter::memEncoderInfer ( const int batchSizeActual )

protected

Run the memory encoder for the current target batch.

Parameters

[in] batchSizeActual Number of targets in the current batch.

◆ outputMaskObj() [1/2]

void Segmenter::outputMaskObj	(	const std::shared_ptr< NvTrackedObject >	target,
		NvMOTMask &	mask
	)

overridevirtual

Output segmentation mask of a target.

Parameters

[in]	target	The target.
[out]	mask	The output mask.

Implements ISegmenter.

◆ outputMaskObj() [2/2]

void Segmenter::outputMaskObj	(	const std::shared_ptr< NvTrackedObject >	target,
		NvMOTMask &	mask
	)

overridevirtual

Output segmentation mask of a target.

Parameters

[in]	target	The target.
[out]	mask	The output mask.

Implements ISegmenter.

◆ postProcessing() [1/2]

void Segmenter::postProcessing	(	const int	indInBatch,
		const std::shared_ptr< NvTrackedObject >	target,
		const NvMOTFrameNum	frameNum
	)

protected

Convert one inference result into retained target state.

Parameters

[in]	indInBatch	Index of this target in the inference batch.
[in]	target	Tracked target to process.
[in]	frameNum	Frame number associated with the inference result.

◆ postProcessing() [2/2]

void Segmenter::postProcessing	(	const int	indInBatch,
		const std::shared_ptr< NvTrackedObject >	target,
		const NvMOTFrameNum	frameNum
	)

protected

Convert one inference result into retained target state.

Parameters

[in]	indInBatch	Index of this target in the inference batch.
[in]	target	Tracked target to process.
[in]	frameNum	Frame number associated with the inference result.

◆ queryParams() [1/2]

void Segmenter::queryParams ( NvMOTQuery & paramsQuery )

overridevirtual

Populate the capabilities reported to the tracker.

Parameters

[out] paramsQuery The output query parameters.

Implements ISegmenter.

◆ queryParams() [2/2]

void Segmenter::queryParams ( NvMOTQuery & paramsQuery )

overridevirtual

Populate the capabilities reported to the tracker.

Parameters

[out] paramsQuery The output query parameters.

Implements ISegmenter.

◆ removeInferStatus() [1/2]

void Segmenter::removeInferStatus ( )

protected

Remove terminated memory bank.

◆ removeInferStatus() [2/2]

void Segmenter::removeInferStatus ( )

protected

Remove terminated memory bank.

◆ removeStream() [1/2]

void Segmenter::removeStream ( const NvMOTStreamId videoStreamId )

overridevirtual

Remove a stream's Segmenter features.

Parameters

[in] videoStreamId Video stream id.

Implements ISegmenter.

◆ removeStream() [2/2]

void Segmenter::removeStream ( const NvMOTStreamId videoStreamId )

overridevirtual

Remove a stream's Segmenter features.

Parameters

[in] videoStreamId Video stream id.

Implements ISegmenter.

◆ setMaskDecoderPointLabels() [1/2]

void Segmenter::setMaskDecoderPointLabels ( int promptType )

protected

Populate mask-decoder labels for points or bounding boxes.

Parameters

[in] promptType Prompt representation selected for the decoder.

◆ setMaskDecoderPointLabels() [2/2]

void Segmenter::setMaskDecoderPointLabels ( int promptType )

protected

Populate mask-decoder labels for points or bounding boxes.

Parameters

[in] promptType Prompt representation selected for the decoder.

◆ setMaskDecoderPoints() [1/2]

void Segmenter::setMaskDecoderPoints	(	const bool	usePrompt,
		std::vector< std::shared_ptr< NvTrackedObject >>	targetList
	)

protected

Populate mask-decoder prompt coordinates for a target batch.

Parameters

[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	targetList	Targets included in the inference operation.

◆ setMaskDecoderPoints() [2/2]

void Segmenter::setMaskDecoderPoints	(	const bool	usePrompt,
		std::vector< std::shared_ptr< NvTrackedObject >>	targetList
	)

protected

Populate mask-decoder prompt coordinates for a target batch.

Parameters

[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	targetList	Targets included in the inference operation.

◆ setMaskDecoderSinglePointLabelAsNegative() [1/2]

void Segmenter::setMaskDecoderSinglePointLabelAsNegative	(	int	pointNumPerTarget,
		int	indInBatch
	)

protected

Mark a single decoder prompt point as negative.

Parameters

[in]	pointNumPerTarget	Number of prompt points allocated to each target.
[in]	indInBatch	Target index within the inference batch.

◆ setMaskDecoderSinglePointLabelAsNegative() [2/2]

void Segmenter::setMaskDecoderSinglePointLabelAsNegative	(	int	pointNumPerTarget,
		int	indInBatch
	)

protected

Mark a single decoder prompt point as negative.

Parameters

[in]	pointNumPerTarget	Number of prompt points allocated to each target.
[in]	indInBatch	Target index within the inference batch.

◆ singleInfer() [1/2]

void Segmenter::singleInfer	(	const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &	targetList,
		bool	usePrompt,
		int	outputLabelIndMax,
		const NvMOTFrameNum	frameNum,
		const uint	inputRoiInd
	)

protected

Run segmentation for one target.

Parameters

[in]	targetList	Targets included in the inference operation.
[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	outputLabelIndMax	Highest decoder output-label index to process.
[in]	frameNum	Frame number associated with the inference result.
[in]	inputRoiInd	Region index within the input batch.

◆ singleInfer() [2/2]

void Segmenter::singleInfer	(	const std::map< uint32_t, std::shared_ptr< NvTrackedObject >> &	targetList,
		bool	usePrompt,
		int	outputLabelIndMax,
		const NvMOTFrameNum	frameNum,
		const uint	inputRoiInd
	)

protected

Run segmentation for one target.

Parameters

[in]	targetList	Targets included in the inference operation.
[in]	usePrompt	Whether decoder prompt data is enabled.
[in]	outputLabelIndMax	Highest decoder output-label index to process.
[in]	frameNum	Frame number associated with the inference result.
[in]	inputRoiInd	Region index within the input batch.

◆ skipInfer() [1/2]

bool Segmenter::skipInfer	(	std::shared_ptr< NvTrackedObject >	target,
		const NvMOTFrameNum	frameNum
	)

protected

Determine whether segmentation can be skipped for a target.

Parameters

[in]	target	Tracked target to process.
[in]	frameNum	Frame number associated with the inference result.

◆ skipInfer() [2/2]

bool Segmenter::skipInfer	(	std::shared_ptr< NvTrackedObject >	target,
		const NvMOTFrameNum	frameNum
	)

protected

Determine whether segmentation can be skipped for a target.

Parameters

[in]	target	Tracked target to process.
[in]	frameNum	Frame number associated with the inference result.

◆ update() [1/2]

void Segmenter::update ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

overridevirtual

Update Segmenter after data association.

Parameters

[in] inputMeta A batch of input frames.

Implements ISegmenter.

◆ update() [2/2]

void Segmenter::update ( const std::map< NvMOTStreamId, NvMOTFrame * > & inputMeta )

overridevirtual

Update Segmenter after data association.

Parameters

[in] inputMeta A batch of input frames.

Implements ISegmenter.

◆ updateInferStatus() [1/2]

void Segmenter::updateInferStatus	(	const int	indInBatch,
		const std::shared_ptr< NvTrackedObject >	target,
		const NvMOTFrameNum	frameNum
	)

protected

Record a target in the pending inference batch.

Parameters

[in]	indInBatch	Index of this target in the inference batch.
[in]	target	Tracked target to process.
[in]	frameNum	Frame number associated with the inference result.

◆ updateInferStatus() [2/2]

void Segmenter::updateInferStatus	(	const int	indInBatch,
		const std::shared_ptr< NvTrackedObject >	target,
		const NvMOTFrameNum	frameNum
	)

protected

Record a target in the pending inference batch.

Parameters

[in]	indInBatch	Index of this target in the inference batch.
[in]	target	Tracked target to process.
[in]	frameNum	Frame number associated with the inference result.

◆ updateTrtBindingsPerStream() [1/2]

void Segmenter::updateTrtBindingsPerStream ( const uint inputRoiInd )

protected

Update TRT bindings per stream.

Parameters

[in] inputRoiInd Region index within the input batch.

◆ updateTrtBindingsPerStream() [2/2]

void Segmenter::updateTrtBindingsPerStream ( const uint inputRoiInd )

protected

Update TRT bindings per stream.

Parameters

[in] inputRoiInd Region index within the input batch.

Field Documentation

◆ m_batchMaskDecSize_d

float * Segmenter::m_batchMaskDecSize_d

protected

Mask output for all targets in all streams.

Definition at line 366 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_batchPixFeatWithMem_d

float * Segmenter::m_batchPixFeatWithMem_d

protected

Attention output features with memory for all targets in all streams.

Definition at line 345 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_bindingsImgEncoder

std::vector< void * > Segmenter::m_bindingsImgEncoder

protected

Definition at line 387 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_bindingsMaskDecoder

std::vector< void * > Segmenter::m_bindingsMaskDecoder

protected

Definition at line 387 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_bindingsMemAtten

std::vector< void * > Segmenter::m_bindingsMemAtten

protected

Definition at line 387 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_bindingsMemEncoder

std::vector< void * > Segmenter::m_bindingsMemEncoder

protected

TensorRT bindings.

Definition at line 387 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_cond_frame_id_diff_d

float * Segmenter::m_cond_frame_id_diff_d

protected

Memory attention inputs: object pointers, mask memory features, memory, positional embeddings, and condition-frame ID differences.

Definition at line 343 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_condFrameOutputs

std::map< TrackerIndex, std::map< NvMOTFrameNum, SegmentMemoryInfo > > Segmenter::m_condFrameOutputs

protected

Outputs of the conditional frames.

Definition at line 381 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_cublasHandle

cublasHandle_t Segmenter::m_cublasHandle

protected

Cublas handle.

Definition at line 328 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_cudaInferenceStatus

std::map< TrackerIndex, CudaInferenceStatus > Segmenter::m_cudaInferenceStatus

protected

Cuda inference status.

Definition at line 384 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

Referenced by isInferStatusEmpty().

◆ m_cudaMemBank

SegmenterCudaMemBank Segmenter::m_cudaMemBank

protected

Cuda memory bank.

Definition at line 386 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_cuStream

cudaStream_t Segmenter::m_cuStream

protected

Cuda stream to run all GPU tasks.

Definition at line 327 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_directly_add_no_mem_embed_d

float * Segmenter::m_directly_add_no_mem_embed_d

protected

Non-memory embeddings added to image-encoder output.

These embeddings are used when memory attention does not process the first or only frame.

Definition at line 325 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_featuresEncoder_d

float * Segmenter::m_featuresEncoder_d

protected

Definition at line 338 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_highResFeaturesEncoder0_d

float * Segmenter::m_highResFeaturesEncoder0_d

protected

Definition at line 338 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_highResFeaturesEncoder1_d

float * Segmenter::m_highResFeaturesEncoder1_d

protected

Image encoder output feature.

Definition at line 338 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_highResImgPosEmbed1_d

float * Segmenter::m_highResImgPosEmbed1_d

protected

Image encoder output position embedding.

Definition at line 339 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_imageEncoder

TrtModule Segmenter::m_imageEncoder

protected

Encoder to extract features from each image.

Definition at line 331 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_inferStatus

std::map< TrackerIndex, InferenceStatus > Segmenter::m_inferStatus

protected

Inference status.

Definition at line 383 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

Referenced by isInferStatusEmpty().

◆ m_inputBuf_d

float * Segmenter::m_inputBuf_d

protected

Definition at line 337 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_inputBuf_h

float * Segmenter::m_inputBuf_h

protected

Input patches to encoder.

Definition at line 337 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_inputMask_d

float * Segmenter::m_inputMask_d

protected

Input mask prompt, currently unused and set as 0.

Definition at line 350 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_inputMaskLabel_d

float * Segmenter::m_inputMaskLabel_d

protected

Input mask label, currently unused and set as 0.

Definition at line 351 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_inputRoisInBatch

std::map< NvMOTStreamId, std::vector< std::pair< uint, NvMotDataTypes::Rect2f > > > Segmenter::m_inputRoisInBatch

protected

Input rois to crop.

Definition at line 380 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_isForLocalization

bool Segmenter::m_isForLocalization

protected

Whether the inference is for localization.

Definition at line 322 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_isInit

bool Segmenter::m_isInit

protected

Segmenter status if initialized.

Definition at line 321 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

Referenced by isInitialized().

◆ m_maskDecoder

TrtModule Segmenter::m_maskDecoder

protected

Decoder taking in prompts (points, bboxes, etc.) to create segmentation mask.

Definition at line 332 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_maskForMem_d

float * Segmenter::m_maskForMem_d

protected

Output mask high resolution.

Definition at line 353 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_maskmem_features_d

float * Segmenter::m_maskmem_features_d

protected

Definition at line 363 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_maskmem_pos_enc_d

float * Segmenter::m_maskmem_pos_enc_d

protected

Definition at line 363 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_memory_d

float * Segmenter::m_memory_d

protected

Definition at line 343 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_memory_maskmem_features_d

float * Segmenter::m_memory_maskmem_features_d

protected

Definition at line 343 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_memory_obj_ptr_d

float * Segmenter::m_memory_obj_ptr_d

protected

Definition at line 343 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_memory_pos_embed_d

float * Segmenter::m_memory_pos_embed_d

protected

Definition at line 343 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_memoryAttention

TrtModule Segmenter::m_memoryAttention

protected

Memory attention module.

Definition at line 333 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_memoryEncoder

TrtModule Segmenter::m_memoryEncoder

protected

Memory encoder module.

Definition at line 334 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_nonCondFrameOutputs

std::map< TrackerIndex, std::map< NvMOTFrameNum, SegmentMemoryInfo > > Segmenter::m_nonCondFrameOutputs

protected

Outputs of the non-conditional frames.

Definition at line 382 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_obj_ptr_d

float * Segmenter::m_obj_ptr_d

protected

Output obj ptrs.

Definition at line 354 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_originalSrcImageSize

NvMotDataTypes::Size Segmenter::m_originalSrcImageSize

protected

Input to tracker plugin, usually from streammux.

Definition at line 373 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_outputIou_d

float * Segmenter::m_outputIou_d

protected

Output IOU score.

Definition at line 355 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_outputIou_h

std::vector< float > Segmenter::m_outputIou_h

protected

Output IOU score on host.

Definition at line 358 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_outputOccLogit_d

float * Segmenter::m_outputOccLogit_d

protected

Output occlusion logit; <2 means target is occluded.

A value greater than 6 indicates high confidence that the object is present.

Definition at line 356 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_outputOccLogit_h

std::vector< float > Segmenter::m_outputOccLogit_h

protected

Output occlusion logit on host.

Definition at line 359 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_outputSize_d

int * Segmenter::m_outputSize_d

protected

Output size.

Definition at line 360 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_params

SegmenterConfigParams Segmenter::m_params

protected

Config params for Segmenter module.

Definition at line 323 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

Referenced by getSegmenterType(), and isInferStatusEmpty().

◆ m_paramsRt

SegmenterRunTimeParams Segmenter::m_paramsRt

protected

Runtime parameters.

Definition at line 324 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_pix_feat_with_mem_d

float * Segmenter::m_pix_feat_with_mem_d

protected

Attention output features with memory for one target.

Definition at line 344 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_pointLabels_d

float * Segmenter::m_pointLabels_d

protected

Input a list of point types (single point, bounding box corners) to decoder.

Definition at line 349 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_points_d

float * Segmenter::m_points_d

protected

Input a list of points (prompt) to decoder.

Definition at line 348 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_scaleInput2Original

Eigen::Vector2f Segmenter::m_scaleInput2Original

protected

Scale from decoder output to original frame size decided by streammux.

Definition at line 371 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_scaleMask2Src

Eigen::Vector2f Segmenter::m_scaleMask2Src

protected

Scale from mask output to the source image.

Definition at line 370 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_srcImageSize

NvMotDataTypes::Size Segmenter::m_srcImageSize

protected

Input to low level tracker.

Definition at line 372 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_subImageCropper2

std::shared_ptr< ReIDFrameTransformTexture > Segmenter::m_subImageCropper2

protected

Image cropper, may share with visual tracker in the future.

Definition at line 377 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_subImageCropScaler

std::shared_ptr< ReIDCropScaler > Segmenter::m_subImageCropScaler

protected

Image crop scaler.

Uses VPI crop scaler.

Definition at line 378 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_targetManager

std::shared_ptr< NvTrackedObjectManager > Segmenter::m_targetManager

protected

Target manager providing target status to update feature gallery.

Definition at line 376 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_targetMaskDecSize_d

float * Segmenter::m_targetMaskDecSize_d

protected

Definition at line 352 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_targetMaskDecSize_h

float * Segmenter::m_targetMaskDecSize_h

protected

Output mask basic resolution.

Definition at line 352 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_targetMaskOriginalSrcSize_d

float * Segmenter::m_targetMaskOriginalSrcSize_d

protected

Mask for each target resized to original streammux size.

Definition at line 367 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_targetScores

std::map< TrackerIndex, std::vector< float > > Segmenter::m_targetScores

protected

Target IOU and occlusion scores.

Definition at line 385 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_temporal_code_d

float * Segmenter::m_temporal_code_d

protected

Output mask memory features and pos encoding.

Definition at line 363 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_visionFeats_d

float * Segmenter::m_visionFeats_d

protected

Definition at line 339 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_visionPosEmbed_d

float * Segmenter::m_visionPosEmbed_d

protected

Definition at line 339 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_work_0_d

float * Segmenter::m_work_0_d

protected

Definition at line 342 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_work_1_d

float * Segmenter::m_work_1_d

protected

Definition at line 342 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

◆ m_work_2_d

float * Segmenter::m_work_2_d

protected

Scratch buffers for memory attention.

Definition at line 342 of file sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp.

The documentation for this class was generated from the following file:

sources/libs/nvmultiobjecttracker/include/modules/Segmenter/Segmenter.hpp


NVIDIA DeepStream SDK API Reference	9.2 Release

NVIDIA DeepStream SDK API Reference

9.2 Release

Detailed Description

Public Member Functions

Protected Member Functions

Protected Attributes

Constructor & Destructor Documentation

◆ Segmenter() [1/4]

◆ Segmenter() [2/4]

◆ ~Segmenter() [1/2]

◆ Segmenter() [3/4]

◆ Segmenter() [4/4]

◆ ~Segmenter() [2/2]

Member Function Documentation

◆ batchInfer() [1/2]

◆ batchInfer() [2/2]

◆ copyFromCuda() [1/2]

◆ copyFromCuda() [2/2]

◆ doInference() [1/2]

◆ doInference() [2/2]

◆ extractImagePatchesBatch() [1/2]

◆ extractImagePatchesBatch() [2/2]

◆ fillCudaMemoryConditionedFeatures() [1/2]

◆ fillCudaMemoryConditionedFeatures() [2/2]

◆ fillMemoryConditionedFeatures() [1/2]

◆ fillMemoryConditionedFeatures() [2/2]

◆ getSegmenterType() [1/2]

◆ getSegmenterType() [2/2]

◆ imgEncoderInfer() [1/2]

◆ imgEncoderInfer() [2/2]

◆ initialize() [1/2]

◆ initialize() [2/2]

◆ initializeTrtBindings() [1/2]

◆ initializeTrtBindings() [2/2]

◆ isGoodBboxProg() [1/2]

◆ isGoodBboxProg() [2/2]

◆ isInferStatusEmpty() [1/2]

◆ isInferStatusEmpty() [2/2]

◆ isInitialized() [1/2]

◆ isInitialized() [2/2]

◆ localize() [1/2]

◆ localize() [2/2]

◆ maskDecoderInfer() [1/2]

◆ maskDecoderInfer() [2/2]

◆ memAttenInfer() [1/2]

◆ memAttenInfer() [2/2]

◆ memAttenInferBatchParser() [1/2]

◆ memAttenInferBatchParser() [2/2]

◆ memEncoderInfer() [1/2]

◆ memEncoderInfer() [2/2]

◆ outputMaskObj() [1/2]

◆ outputMaskObj() [2/2]

◆ postProcessing() [1/2]

◆ postProcessing() [2/2]

◆ queryParams() [1/2]

◆ queryParams() [2/2]

◆ removeInferStatus() [1/2]

◆ removeInferStatus() [2/2]

◆ removeStream() [1/2]

◆ removeStream() [2/2]

◆ setMaskDecoderPointLabels() [1/2]

◆ setMaskDecoderPointLabels() [2/2]

◆ setMaskDecoderPoints() [1/2]

◆ setMaskDecoderPoints() [2/2]

◆ setMaskDecoderSinglePointLabelAsNegative() [1/2]

◆ setMaskDecoderSinglePointLabelAsNegative() [2/2]

◆ singleInfer() [1/2]

◆ singleInfer() [2/2]

◆ skipInfer() [1/2]

◆ skipInfer() [2/2]

◆ update() [1/2]

◆ update() [2/2]

◆ updateInferStatus() [1/2]

◆ updateInferStatus() [2/2]

◆ updateTrtBindingsPerStream() [1/2]

◆ updateTrtBindingsPerStream() [2/2]

Field Documentation

◆ m_batchMaskDecSize_d

◆ m_batchPixFeatWithMem_d

◆ m_bindingsImgEncoder