sources/libs/nvmultiobjecttracker/include/modules/cuDCFv2/featureExtractor.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef CUDCFLIB_FEATUREEXTRACTOR_HPP_
#define CUDCFLIB_FEATUREEXTRACTOR_HPP_
 
#include <cuda_runtime.h>
#include <cuda_fp16.h>
#include <cuComplex.h>
 
#if defined(VIZ_DEBUG)
#include <opencv2/core.hpp>
#endif
 
// VPI.
#include "vpi/Stream.h"
#include "vpi/Image.h"
#include "vpi/experimental/ColorNames.h"
#include "vpi/experimental/HOG.h"
#include "vpi/CUDAInterop.h"
#include "modules/utility/cuda_common.hpp"
#include "modules/cuDCFv2/cuDCFFrameTransformTexture.hpp"
 
class FeatureExtractor
{
public:
    FeatureExtractor();
 
    virtual ~FeatureExtractor();
 
    enum FEATURE_TYPES : uint
    {
        COLOR_NAMES = 0x1, 
        HOG = 0x10 
    };
    static const uint NUM_CHANNELS_COLORNAMES = 10; 
    static const uint NUM_CHANNELS_HOG_SENSITIVE = 18; 
    static const uint NUM_CHANNELS_HOG_INSENSITIVE = NUM_CHANNELS_HOG_SENSITIVE/2; 
 
    bool isInitialized() { return m_initGpu; }
 
    uint getNumChannels() { return m_numFeatureChannelsOut;}
 
    void initialize_ColorNames();
 
    bool extractFeatures_VPI(
        const std::map<NvMOTStreamId, NvMOTFrame*> &nvFramesInBatch,
        const std::map<NvMOTStreamId, std::vector<std::pair<uint, NvMotDataTypes::Rect2f> > > &roisInBatch,
        const uint lastBboxInd, const bool doUpdateImages
    );
 
    bool initialize(
        const int debugVerboseLevel, const int maxStreamNum, const int maxBufSurfAddrSize, const int maxfeatureImgNum,
        const NvMotDataTypes::Size srcImgSize, const uint srcImgPitch, const NvBufSurfaceColorFormat colorFormatType,
        const NvMotDataTypes::Size templateImgSize, const int hogCellSize, const uint featureTypes,
        const VPIBackend vpiDevType = VPI_BACKEND_CUDA, const VPIImageFormat vpiImgOutputType = VPI_IMAGE_FORMAT_U16
    );
 
    uchar *getOutputFeaturePtrs_VPI(
        uint &width, uint &height, uint2 &outBatchingGrid, uint &channels, uint &pitch, uint &imgNum
    );
 
    #if defined(VIZ_DEBUG)
 
        std::vector<std::vector<cv::Mat> > getExtractedFeaturesAtHost_VPI() const;
 
        std::map<NvMOTStreamId, std::map<uint, cv::Mat > > getSrcImgPatches()
        {
            assert(m_subImageCropper2 != nullptr && m_subImageCropper2->isInitialized());
 
            if(m_colorFormatType == NvBufSurfaceColorFormat::NVBUF_COLOR_FORMAT_NV12)
            {
                return m_subImageCropper2->getImagePatches2Host_OCV_mosaic();
            }
            else
            {
                return m_subImageCropper2->getImagePatches2Host_OCV_batch();
            }
        }
    #endif
 
    uint getSrcImgNum() { return m_srcImgPatchNum; }
 
    uint2 getOutBatchingGrid() const { return m_outBatchingGrid; }
 
    uint getOutputFeatureChannelSize() const { return m_numFeatureChannelsOut; }
 
    bool isVpiOutputInHighPrecision() const { return m_vpiImgOutputType == VPI_IMAGE_FORMAT_U16; }
 
    #ifdef VIZ_DEBUG
 
        const std::map<NvMOTStreamId, cv::Mat> &getSourceImagesAtHost() const
        {
            return m_subImageCropper2->getSourceImagesAtHost();
        }
    #endif
 
protected:
 
    void initialize_VPI();
 
    const uint m_featureTypeHog = VPI_HOG_CONTRAST_SENSITIVE; 
    const uint m_numOfChannelsByHog = FeatureExtractor::NUM_CHANNELS_HOG_SENSITIVE; 
    const uint m_numOfChannelsByColorNames = FeatureExtractor::NUM_CHANNELS_COLORNAMES; 
    uint8_t *m_srcImgPatchPtr_d[2]; 
    uint m_srcSubImgWidth[2]; 
    uint m_srcSubImgHeight[2]; 
    uint m_dstSubImgHeight[2]; 
    uint2 m_outBatchingGrid; 
    uint m_srcImgColorChannels; 
    NvBufSurfaceColorFormat m_colorFormatType; 
    uint m_srcImgPatchPitch[2]; 
    uint m_srcImgPatchNum; 
    uint m_maxfeatureImgNum; 
    uint m_maxNumFramesInBatch; 
    uint m_numFeatureChannelsOut; 
    uint m_featureTypes; 
    cuComplex *m_batch_ftrs_device; 
    size_t m_batch_ftrs_device_pitch; 
    void *m_batch_array_host_real; 
    std::shared_ptr<cuDCFFrameTransformTexture> m_subImageCropper2; 
    bool m_initGpu; 
 
    // [Configuration parameters]
    int m_debugVerboseLevel; 
    const int m_cudaBlockSize = 32; 
    const bool m_showGpuTiming = false; 
    int m_hogCellSize; 
 
    // [CUDA timer]
    cudaEvent_t m_eventStart; 
    cudaEvent_t m_eventStop; 
 
    // [VPI]
    bool m_isVpiInitialized; 
    VPIImage m_vpiInputImage[2]; 
    VPIImageData m_vpiInputImageData[2]; 
    std::vector<VPIImage> m_vpiOutFeatures; 
    std::vector<VPIImageData> m_vpiOutFeaturesData; 
    VPIStream m_vpiStream; 
    VPIPayload m_vpiColorNames; 
    VPIPayload m_vpiHOG; 
    void *m_batch_vpi_ftrs; 
    size_t m_batch_vpi_ftrs_pitch; 
    VPIBackend m_vpiDevType; 
    const int m_vpiImgFlags = VPI_BACKEND_CUDA | VPI_RESTRICT_MEM_USAGE; 
    VPIImageFormat m_vpiImgOutputType; 
};
 
#endif // CUDCFLIB_FEATUREEXTRACTOR_HPP_