sources/libs/nvmultiobjecttracker/include/modules/VisualTracker/DCFTracker.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef VISUALTRACKERLIB_DCFTRACKER_HPP_
#define VISUALTRACKERLIB_DCFTRACKER_HPP_
 
#include <map>
#include <unordered_map>
#include "modules/cuDCFv2/cuDCF.h"
#include "modules/VisualTracker/IVisualTracker.hpp"
#include "modules/NvMultiObjectTracker/NvTrackerParams.hpp"
#include "modules/NvMultiObjectTracker/NvTrackedObject.hpp"
#include "modules/utility/dataTypes.hpp"
 
// VPI.
#include "vpi/Stream.h"
#include "vpi/Image.h"
#include "vpi/Array.h"
#include "vpi/algo/CropScaler.h"
#include "vpi/algo/DCFTracker.h"
 
#if defined(__aarch64__)
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include "cudaEGL.h"
#endif
 
#ifdef VIZ_DEBUG
 
class TrackerVizOutput;
#endif
 
class DCFTrackerConfigParams : public NvTrackerParams
{
public:
    DCFTrackerConfigParams();
 
    DCFTrackerConfigParams(const std::string &configFilePath, const std::string moduleName);
 
    DCFTrackerConfigParams(const YAML::Node &fn);
 
    void read(const YAML::Node &fn) override;
 
    int m_showGpuTiming; 
    int m_debugVerboseLevel; 
    int m_debugVerboseLevel_gpumemory; 
    int m_debugVerboseLevel_cufft; 
 
    // [Visual feature types]
    bool m_useColorNames; 
    bool m_useHog; 
    bool m_useGray; 
    bool m_useRgb; 
 
    // [VPI]
    bool m_useHighPrecisionFeature; 
    VPIBackend m_vpiBackend4CropnScale; 
    VPIBackend m_vpiBackend4DcfTracker; 
 
    // [Feature extraction]
    float m_featureImgSize; 
    float m_featureFocusOffsetFactor_y; 
    float m_modelMaskingWindowFactor; 
    int m_hogCellSize; 
 
    // [DCF][Core]
    float m_filterLr; 
    float m_filterChannelWeightsLr; 
    float m_gaussianSigma; 
    float m_lambda; 
    float m_peakCorrLr; 
 
    // [FFT]
    int m_cufftHandleQuantFactor; 
 
    // [Viz Debug]
    bool m_generateTrackerVisualization; 
    bool m_vizWriteSrcImgPatchesAndCorrResp; 
    bool m_vizWriteBirdsEyeView; 
    bool m_vizWriteAllTrajectoriesOnEveryFrame; 
    bool m_vizWriteEachCompletedTrajectory; 
    int m_vizJpegImageQuality; 
    std::string m_vizOutDirectory; 
    int m_vizDebugStreamId; 
 
    // [Custom parameters for debugging]
    int m_debugParamInt0; 
    int m_debugParamInt1; 
    int m_debugParamInt2; 
    float m_debugParamFloat0; 
    float m_debugParamFloat1; 
    float m_debugParamFloat2; 
 
    // [Configuration parameters]
    VISUAL_TRACKER_TYPES m_visualTrackerType; 
    bool m_isInit = false; 
};
 
struct DCFTrackerRunTimeParams
{
    uint maxStreams; 
    uint maxTargetsToTrack; 
    uint maxDetectorBboxesPerFrame; 
    uint m_maxBufSurfAddrSize; 
    NvMotDataTypes::Size originalSrcImageSize; 
    NvMotDataTypes::Size srcImageSize; 
    size_t srcImagePitch; 
    NvBufSurfaceColorFormat colorFormat; 
    NvMotDataTypes::Size templateSize; 
    NvMotDataTypes::Size singleFeatureMatSize; 
    int detectorInputInterval; 
    std::map<NvMOTStreamId, uint32_t> isDetectionDonePerStream; 
    bool skipBatchUpdate; 
    int singleFeatureChannelElementSize; 
    int detCorrRespSearchWindowSize; 
};
 
class DCFTracker : public IVisualTracker
{
public:
 
    DCFTracker(const std::string &configFilePath);
 
    virtual ~DCFTracker();
 
    void initialize(
        const SourceFrameInfo &frameInfo, const uint maxTargetsToTrack,
        const uint maxDetectorBboxesPerFrame, const std::shared_ptr<NvTrackedObjectManager> &targetManager
    ) override;
 
    void queryParams(NvMOTQuery &paramsQuery);
 
    void localize(
        const std::map<NvMOTStreamId, NvMOTFrame *> inputMeta,
        const std::map<NvMOTStreamId, std::deque<DetectedObjInfo>> &detectedObjects
    ) override;
 
    void update(const std::map<NvMOTStreamId, NvMOTFrame *> inputMeta) override;
 
    bool isInitialized() override { return m_isInit; }
 
    const VISUAL_TRACKER_TYPES getVisualTrackerType() override { return m_params.m_visualTrackerType; }
 
    void updateParams(const YAML::Node &fn) { m_params.read(fn); }
 
protected:
 
    bool m_isInit; 
    DCFTrackerConfigParams m_params; 
    DCFTrackerRunTimeParams m_paramsRt; 
    std::shared_ptr<NvTrackedObjectManager> m_targetManager; 
    std::map<NvMOTStreamId, std::vector<std::pair<uint, NvMotDataTypes::Rect2f>>> m_featureRoisInBatch; 
 
    // [cuFFT operations]
    cudaStream_t m_cudaStream; 
 
    // [CUDA memory buffers for pixel processing]
    float *m_corrResponses_d; 
    float *m_channelWeights; 
 
    // [CUDA and host memory buffers for miscellaneous data]
    bool *m_trackerDoProcessing_h, *m_trackerDoProcessing_d; 
    uint *m_detectorBboxes_size_h, *m_detectorBboxes_size_d; 
    uint32_t *m_trackerAge_h, *m_trackerAge_d; 
    detectorBbox_t *m_detectorBboxes_h, *m_detectorBboxes_d; 
    float *m_confidenceAtDetectedPt_h, *m_confidenceAtDetectedPt_d; 
    float *m_corrResponseAtDetectedPt_h, *m_corrResponseAtDetectedPt_d; 
    float *m_maxCorrResponse_h, *m_maxCorrResponse_d; 
    float *m_maxCorrResponse_ma_d, *m_maxCorrResponse_ma_h; 
    float *m_trackingConfidence_d, *m_trackingConfidence_h; 
    float *m_maskingWindow4Src; 
    float *m_maskingWindow4Model; 
    int m_featureChannelSize; 
    int m_cudaKernelMaxBatchSize; 
 
    // [VPI library interactions]
    VPIDCFTrackerCreationParams m_vpiCreateParams; 
    VPIDCFTrackerParams m_vpiRuntimeParams; 
    VPIPayload m_vpiDcfPayload; 
    VPIPayload m_vpiCropPayload; 
    VPIStream m_vpiStream; 
    int m_vpiMemFlags = VPI_BACKEND_CUDA | VPI_RESTRICT_MEM_USAGE; 
 
    // [Input frame sequences]
    VPIImage *m_vpiInputFrameSequence; 
    VPIImageData *m_vpiInputFrameDataList; 
    std::vector<bool> m_isFrameCreated; 
    int32_t *m_enabledSequences; 
 
    // [Tracked objects]
    std::vector<VPIDCFTrackedBoundingBox> m_vpiDCFBboxes; 
    VPIArrayData m_vpiDCFBboxesData; 
    VPIArray m_vpiDCFBboxesArray; 
    int32_t m_vpiBboxesArraySize; 
    std::vector<VPIDCFTrackedBoundingBox> m_vpiDCFBboxesLocalized; 
    VPIArrayData m_vpiDCFBboxesLocalizedData; 
    VPIArray m_vpiDCFBboxesLocalizedArray; 
 
    // [Cropped and scaled object patches]
    uint8_t *m_vpiObjPatches_d; 
    #if defined (VIZ_DEBUG)
        uint8_t *m_vpiObjPatches_h; 
    #endif
    int m_vpiObjPatchesNumChannels; 
    VPIImageData m_vpiObjPatchesData; 
    VPIImage m_vpiObjPatches; 
 
    // [VPI images and arrays wrapping CUDA memory]
    VPIImageData m_vpiMaskingWindow4SrcData; 
    VPIImage m_vpiMaskingWindow4SrcImage; 
    VPIImageData m_vpiMaskingWindow4ModelData; 
    VPIImage m_vpiMaskingWindow4ModelImage; 
    VPIImageData m_vpiCorrResponsesData; 
    VPIImage m_vpiCorrResponsesImage; 
    VPIArrayData m_vpiMaxCorrResponseArrayData; 
    VPIArray m_vpiMaxCorrResponseArray; 
    VPIArrayData m_vpiChannelWeightsArrayData; 
    VPIArray m_vpiChannelWeightsArray; 
 
    // [CUDA pointers from NvBufSurface objects]
    int m_integrated_gpu; 
    int m_deviceId; 
    uint8_t *m_surfCudaMemPtr[NVBUF_MAX_PLANES]; 
    #if defined(__aarch64__)
 
        // [Cached EGL mappings]
        std::unordered_map<const NvBufSurfaceParams *, CUgraphicsResource> m_graphicsResourceAddrMap; 
        std::unordered_map<const NvBufSurfaceParams *, CUeglFrame> m_eglFrameAddrMap; 
    #endif
 
    cudaEvent_t m_eventStart; 
    cudaEvent_t m_eventStop; 
 
    void checkInputOutput(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta);
 
    void setVpiInputFrameSequence(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta);
 
    void allocateCudaMemories();
 
    void makeBatchAndSendFloatMatToDevice(
        const NvMotDataTypes::RowMajorMatrixXf &srcMat, void *devMem, uint batchSize, float *workarea_host
    );
 
    void updateLocalizationResults(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta);
 
    Eigen::Vector2f convertSrcImgCoordToCorrRespCoord(
        const NvMotDataTypes::Rect2f &roi, const Eigen::Vector2f pt_img
    ) const;
 
    void cudaCalculateResponse(void);
 
    void getDetectorBboxCorrelationResponses(
        const std::map<NvMOTStreamId, std::deque<DetectedObjInfo>> &detectedObjects
    );
 
    void cudaFilterUpdate();
 
    inline int getCufftPlanId(int num_target);
 
    float getTemplateRescaleRatio(const NvMotDataTypes::Rect2f &roi) const;
 
    void performCropAndScale(const std::map<NvMOTStreamId, NvMOTFrame *> &inputMeta, const bool isForLocalization);
 
    inline void getCudaMemPointers(NvBufSurfaceParams *baseSurface);
 
    #ifdef VIZ_DEBUG
        // [Source images in host memory]
        std::map<NvMOTStreamId, cv::Mat> m_srcFramesAtHost; 
        uint8_t *m_idata_host; 
 
        std::map<NvMOTStreamId, std::map<uint, cv::Mat > > getImagePatches2Host_OCV_batch();
 
        std::shared_ptr<TrackerVizOutput> m_vizOutput; 
        float *m_batchCudaArray_host; 
        std::map<uint, std::pair<NvMOTStreamId, uint> > m_targetNumInd2RoisInBatchMapper; 
        float2 *m_targetDisplacement2D_h; 
 
        std::vector<cv::Mat> fetchBatchDataToHost_perObj(const float *deviceData);
 
        std::map<NvMOTStreamId, cv::Mat> getSrcImagesAtHost();
 
        void uploadDeviceData2HostMemories(const std::map<NvMOTStreamId, NvMOTFrame*> &inputMeta);
 
        const std::map<NvMOTStreamId, cv::Mat>& getSrcFramesAtHost() const { return m_srcFramesAtHost; }
 
        friend class TrackerVizOutput;
    #endif
};
 
#endif // VISUALTRACKERLIB_DCFTRACKER_HPP_