sources/libs/nvmultiobjecttracker/include/modules/VisualTracker/VisualTracker.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef VISUALTRACKERLIB_VISUALTRACKER_HPP_
#define VISUALTRACKERLIB_VISUALTRACKER_HPP_
 
#include <map>
#include <unordered_map>
#include "modules/cuDCFv2/cuDCF.h"
#include "modules/cuDCFv2/featureExtractor.hpp"
#include "modules/VisualTracker/IVisualTracker.hpp"
#include "modules/NvMultiObjectTracker/NvTrackerParams.hpp"
#include "modules/NvMultiObjectTracker/NvTrackedObject.hpp"
#include "modules/utility/dataTypes.hpp"
 
#ifdef VIZ_DEBUG
 
class TrackerVizOutput;
#endif
 
class VisualTrackerConfigParams : public NvTrackerParams
{
public:
    VisualTrackerConfigParams();
 
    VisualTrackerConfigParams(const std::string &configFilePath, const std::string moduleName);
 
    VisualTrackerConfigParams(const YAML::Node &fn);
 
    void read(const YAML::Node &fn) override;
 
    int m_showGpuTiming; 
    int m_debugVerboseLevel; 
    int m_debugVerboseLevel_gpumemory; 
    int m_debugVerboseLevel_cufft; 
 
    // [Visual feature types]
    bool m_useColorNames; 
    bool m_useHog; 
    bool m_useGray; 
    bool m_useRgb; 
 
    // [VPI]
    bool m_useHighPrecisionFeature; 
    bool m_useCpuBackends4VPI; 
 
    // [Feature extraction]
    float m_featureImgSize; 
    float m_featureFocusOffsetFactor_y; 
    float m_modelMaskingWindowFactor; 
    int m_hogCellSize; 
 
    // [DCF][Core]
    float m_filterLr; 
    float m_filterChannelWeightsLr; 
    float m_gaussianSigma; 
    float m_lambda; 
    float m_peakCorrLr; 
    float m_minConfidence4SegmentationFilter; 
 
    // [FFT]
    int m_cufftHandleQuantFactor; 
 
    // [Viz Debug]
    bool m_generateTrackerVisualization; 
    bool m_vizWriteSrcImgPatchesAndCorrResp; 
    bool m_vizWriteBirdsEyeView; 
    bool m_vizWriteAllTrajectoriesOnEveryFrame; 
    bool m_vizWriteEachCompletedTrajectory; 
    bool m_vizWriteSegMask; 
    int m_vizJpegImageQuality; 
    std::string m_vizOutDirectory; 
    int m_vizDebugStreamId; 
 
    // [Custom parameters for debugging]
    int m_debugParamInt0; 
    int m_debugParamInt1; 
    int m_debugParamInt2; 
    float m_debugParamFloat0; 
    float m_debugParamFloat1; 
    float m_debugParamFloat2; 
 
    // [Configuration parameters]
    VISUAL_TRACKER_TYPES m_visualTrackerType; 
    bool m_isInit = false; 
};
 
struct VisualTrackerRunTimeParams
{
    uint maxStreams; 
    uint maxTargetsToTrack; 
    uint maxDetectorBboxesPerFrame; 
    NvMotDataTypes::Size originalSrcImageSize; 
    NvMotDataTypes::Size srcImageSize; 
    size_t srcImagePitch; 
    NvBufSurfaceColorFormat colorFormat; 
    NvMotDataTypes::Size templateSize; 
    NvMotDataTypes::Size singleFeatureMatSize; 
    int detectorInputInterval; 
    std::map<NvMOTStreamId, uint32_t> isDetectionDonePerStream; 
    int singleFeatureChannelElementSize; 
    int singleFeatureChannelElementSize_F; 
    int featureVecBatchElementSize; 
    int featureVecMaxBatchElementSize; 
    int featureVecMaxBatchElementSize_F; 
 
    // [CUDA configuration]
    int featureChannelSize; 
    int cudaKernelBatchSize; 
    int cudaKernelMaxBatchSize; 
    int detCorrRespSearchWindowSize; 
};
 
class VisualTracker : public IVisualTracker
{
public:
 
    VisualTracker(const std::string &configFilePath);
 
    virtual ~VisualTracker();
 
    void initialize(
        const SourceFrameInfo &frameInfo, const uint maxTargetsToTrack,
        const uint maxDetectorBboxesPerFrame, const std::shared_ptr<NvTrackedObjectManager> &targetManager
    ) override;
 
    void queryParams(NvMOTQuery &paramsQuery);
 
    void localize(
        const std::map<NvMOTStreamId, NvMOTFrame *> inputMeta,
        const std::map<NvMOTStreamId, std::deque<DetectedObjInfo>> &detectedObjects
    ) override;
 
    void update(const std::map<NvMOTStreamId, NvMOTFrame *> inputMeta) override;
 
    bool isInitialized() override { return m_isInit; }
 
    const VISUAL_TRACKER_TYPES getVisualTrackerType() override { return m_params.m_visualTrackerType; }
 
    void updateParams(const YAML::Node &fn) { m_params.read(fn); }
 
protected:
 
    bool m_isInit; 
    VisualTrackerConfigParams m_params; 
    VisualTrackerRunTimeParams m_paramsRt; 
    std::map<NvMOTStreamId, NvMOTFrame *> m_inputMeta; 
    std::shared_ptr<NvTrackedObjectManager> m_targetManager; 
 
    // [Feature extraction]
    FeatureExtractor m_featureExtractor; 
    std::map<NvMOTStreamId, std::vector<std::pair<uint, NvMotDataTypes::Rect2f>>> m_featureRoisInBatch; 
 
    // [cuFFT operations]
    uint m_cufftPlanSize; 
    cudaStream_t m_cudaStream; 
    cufftHandle *m_cufftHandle_C2R_perCh; 
    cufftHandle *m_cufftHandle_C2R_perObj; 
    cufftHandle *m_cufftHandle_R2C_perCh; 
    void *m_cufftWorkspce; 
 
    // [Fourier-space CUDA buffers: each target uses half the image size because of conjugate symmetry]
    cufftComplex *m_ftrs_F; 
    cufftComplex *m_desiredResp_F; 
    cufftComplex *m_dcfFilter_F; 
    cufftComplex *m_work_F_1; 
    cufftComplex *m_work_F_2; 
 
    // [CUDA memory buffers for pixel processing]
    float *m_maskingWindow4Src; 
    float *m_maskingWindow4Model; 
    float *m_maskingSegWindow4Model; 
    float *m_corrResponses_d; 
    float *m_channelWeights; 
    float *m_channelWeightSum; 
    float *m_work_I; 
 
    // [CUDA and host memory buffers for miscellaneous data]
    bool *m_trackerDoProcessing_h, *m_trackerDoProcessing_d; 
    uint *m_detectorBboxes_size_h, *m_detectorBboxes_size_d; 
    uint32_t *m_trackerAge_h, *m_trackerAge_d; 
    float2 *m_targetDisplacement2D_h, *m_targetDisplacement2D_d; 
    detectorBbox_t *m_detectorBboxes_h, *m_detectorBboxes_d; 
    float *m_confidenceAtDetectedPt_h, *m_confidenceAtDetectedPt_d; 
    float *m_corrResponseAtDetectedPt_h, *m_corrResponseAtDetectedPt_d; 
    float *m_maxCorrResponse_h, *m_maxCorrResponse_d; 
    float *m_maxCorrResponse_ma_d, *m_maxCorrResponse_ma_h; 
    float *m_trackingConfidence_d, *m_trackingConfidence_h; 
    float4 *m_maskSegRois_h, *m_maskSegRois_d; 
    bool *m_trackerUseMaskSeg_h, *m_trackerUseMaskSeg_d; 
    cudaEvent_t m_eventStart; 
    cudaEvent_t m_eventStop; 
 
    void checkInputOutput(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta);
 
    void updateData4Processing(const int lastTrackerInd);
 
    void allocateCudaMemories();
 
    void makeBatchAndSendFloatMatToDevice(
        const NvMotDataTypes::RowMajorMatrixXf &srcMat, void *devMem, uint batchSize, float *workarea_host
    );
 
    void updateLocalizationResults(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta);
 
    Eigen::Vector2f convertSrcImgCoordToCorrRespCoord(
        const NvMotDataTypes::Rect2f &roi, const Eigen::Vector2f pt_img
    ) const;
 
    void cudaCalculateResponse(float *ftrs);
 
    void getDetectorBboxCorrelationResponses(
        const std::map<NvMOTStreamId, std::deque<DetectedObjInfo>> &detectedObjects
    );
 
    void cudaFilterUpdate(const float *ftrs);
 
    inline int getCufftPlanId(int num_target);
 
    float getTemplateRescaleRatio(const NvMotDataTypes::Rect2f &roi) const;
 
    float *extractVisualFeatures(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta, const bool isForLocalization);
 
    #ifdef VIZ_DEBUG
        std::shared_ptr<TrackerVizOutput> m_vizOutput; 
        float *m_batchCudaArray_host; 
 
        std::vector<cv::Mat> fetchBatchDataToHost_perObj(const float *deviceData, const bool multiChannelInput=false);
 
        std::map<NvMOTStreamId, cv::Mat> getSrcImagesAtHost();
 
        friend class TrackerVizOutput;
    #endif
};
 
#endif // VISUALTRACKERLIB_VISUALTRACKER_HPP_