9.0/sources/gst-plugins/gst-nvinferserver/gstnvinferserver_impl.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
 *
 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
 * property and proprietary rights in and to this material, related
 * documentation and any modifications thereto. Any use, reproduction,
 * disclosure or distribution of this material and related documentation
 * without an express license agreement from NVIDIA CORPORATION or
 * its affiliates is strictly prohibited.
 */
 
#ifndef __GSTNVINFERSERVER_IMPL_H__
#define __GSTNVINFERSERVER_IMPL_H__
 
#include <glib.h>
#include <gst/gst.h>
#include <string.h>
#include <sys/time.h>
 
#include <condition_variable>
#include <list>
#include <memory>
#include <mutex>
#include <thread>
#include <vector>
 
#include <nvbufsurface.h>
#include <nvbufsurftransform.h>
 
#include "infer_icontext.h"
#include "infer_utils.h"
 
#include "nvdsmeta.h"
#include "nvtx3/nvToolsExt.h"
 
#include "gstnvinferserver_meta_utils.h"
#include "infer_datatypes.h"
#include "infer_post_datatypes.h"
#include "nvdsinferserver_config.pb.h"
#include "nvdsinferserver_plugin.pb.h"
 
void gst_nvinfer_server_logger(uint32_t unique_id, NvDsInferLogLevel log_level,
    const char* log_message, void* user_ctx);
 
namespace dsis = nvdsinferserver;
namespace ic = nvdsinferserver::config;
 
using SharedInferContext = std::shared_ptr<dsis::IInferContext>;
 
namespace nvdsinferserver {
class BatchSurfaceBuffer;
}
 
namespace gstnvinferserver {
 
struct GstNvInferServerObjectHistory {
    bool under_inference = FALSE;
    NvOSD_RectParams last_inferred_coords = {0, 0, 0, 0};
    uint64_t last_inferred_frame_num = 0;
    uint64_t last_accessed_frame_num = 0;
    InferClassificationOutput cached_info;
};
 
using SharedObjHistory = std::shared_ptr<GstNvInferServerObjectHistory>;
using WeakObjHistory = std::weak_ptr<GstNvInferServerObjectHistory>;
 
typedef std::unordered_map<uint64_t, SharedObjHistory>
    GstNvInferServerObjectHistoryMap;
 
typedef struct {
    GstNvInferServerObjectHistoryMap object_history_map;
    gulong last_cleanup_frame_num = 0;
    gulong last_seen_frame_num = 0;
} GstNvInferServerSourceInfo;
 
struct GstNvInferServerProperties {
    std::string configPath;
    uint32_t maxBatchSize = 0;
    int32_t interval = -1;
    uint32_t uniqueId = 0;
    uint32_t processMode = 0;
    int32_t inferOnGieId = -1;
    bool inputTensorFromMeta = false;
    std::vector<int32_t> operateOnClassIds;
    std::string classifierType;
};
 
using RawOutputCallback = std::function<void(void *, NvDsInferNetworkInfo&,
    NvDsInferLayerInfo*, uint32_t, uint32_t)>;
 
struct RequestBuffer;
 
class GstNvInferServerImpl {
private:
    class ObjTrackingData {
    public:
        ObjTrackingData() {}
 
        bool initSource(uint32_t id) {
            m_SourceInfo.emplace(id, GstNvInferServerSourceInfo{});
            return true;
        }
 
        void removeSource(uint32_t id) { m_SourceInfo.erase(id); }
 
        GstNvInferServerSourceInfo* findSource(int id);
 
        bool hasSource(uint32_t id) const {
            return m_SourceInfo.find(id) != m_SourceInfo.end();
        }
 
        SharedObjHistory findObjHistory(
            uint32_t sourceId, uint64_t objId);
 
        SharedObjHistory newHistory(
            uint32_t sourceId, uint64_t objId);
 
        void clear() { m_SourceInfo.clear(); }
 
        void clearUpHistory(uint64_t seqId);
 
    private:
        std::unordered_map<uint32_t, GstNvInferServerSourceInfo> m_SourceInfo;
 
        gulong m_LastMapCleanupSeqId = 0;
    };
 
    using FuncItem = std::function<void()>;
 
public:
    GstNvInferServerImpl(GstNvInferServer* infer);
 
    ~GstNvInferServerImpl();
 
    void setRawoutputCb(RawOutputCallback cb) {
        m_RawoutputCb = cb;
    }
 
    NvDsInferStatus start();
 
    NvDsInferStatus stop();
 
    NvDsInferStatus processBatchMeta(NvDsBatchMeta* batchMeta,
        NvBufSurface* inSurf, uint64_t seqId, void *gstBuf);
 
    NvDsInferStatus sync();
 
    NvDsInferStatus queueOperation(FuncItem func);
 
 
    bool addTrackingSource(uint32_t sourceId);
 
    void eraseTrackingSource(uint32_t sourceId);
 
    void resetIntervalCounter();
 
    // helper functions
    uint32_t uniqueId() const { return m_GstProperties.uniqueId; }
    const std::string& classifierType() const { return m_GstProperties.classifierType; }
    uint32_t maxBatchSize() const;
    nvtxDomainHandle_t nvtxDomain() {
        return m_NvtxDomain.get();
    }
    bool isAsyncMode() const;
    bool canSupportGpu(int gpuId) const;
    NvDsInferStatus lastError() const;
    const ic::PluginControl& config() const { return m_PluginConfig; }
    void updateInterval(guint interval)  {m_PluginConfig.mutable_input_control()->set_interval(interval);}
private:
    NvDsInferStatus processFullFrame(NvDsBatchMeta* batchMeta,
        NvBufSurface* inSurf, uint64_t seqId, void *gstBuf);
 
    NvDsInferStatus processObjects(NvDsBatchMeta* batchMeta,
        NvBufSurface* inSurf, uint64_t seqId, void *gstBuf);
 
    NvDsInferStatus processInputTensor(
        NvDsBatchMeta* batchMeta, NvBufSurface* inSurf, uint64_t seqId, void* gstBuf);
 
    NvDsInferStatus batchInference(
        std::shared_ptr<RequestBuffer> reqBuf, std::vector<dsis::SharedBatchBuf> batchBuf);
 
    void InferenceDone(std::shared_ptr<RequestBuffer> req);
 
    NvDsInferStatus attachBatchDetection(
        RequestBuffer* req, dsis::SharedIBatchBuffer output);
 
    NvDsInferStatus attachBatchClassification(
        RequestBuffer* req, dsis::SharedIBatchBuffer output);
 
    NvDsInferStatus attachBatchSegmentation(
        RequestBuffer* req, dsis::SharedIBatchBuffer output);
 
    NvDsInferStatus handleOutputTensors(RequestBuffer* req);
 
    void updateLastError(NvDsInferStatus s);
 
    ic::PluginControl& mutableConfig() { return m_PluginConfig; }
    bool isClassify() const;
    bool isDetection() const;
    bool isSegmentation() const;
    bool isOtherNetowrk() const;
    bool isFullFrame() const;
    bool maintainAspectRatio() const;
    bool symmetricPadding() const;
    bool needOutputTensorMeta() const;
    bool hasCustomProcess() const;
    uint32_t inferInterval() const { return m_PluginConfig.input_control().interval(); }
    bool inputTensorFromMeta() const
    {
        return m_PluginConfig.infer_config().has_input_tensor_from_meta();
    }
    bool validatePluginConfig(
        ic::PluginControl& config, const std::string& path,
        const GstNvInferServerProperties& update);
 
    bool shouldInferObject(NvDsObjectMeta* obj_meta, uint32_t frameNum,
        GstNvInferServerObjectHistory* history);
 
    bool outputLoop(FuncItem func);
 
public:
    GstNvInferServerProperties m_GstProperties;
 
private:
    using OutputThread = dsis::QueueThread<std::list<FuncItem>>;
 
    SharedInferContext m_InferCtx;
    GstNvInferServer* m_GstPlugin = nullptr;
    mutable std::mutex m_ProcessMutex;
    ic::PluginControl m_PluginConfig;
    ObjTrackingData m_ObjTrackingData;
 
    std::unique_ptr<OutputThread> m_OutputThread;
 
    std::unique_ptr<nvtxDomainRegistration, std::function<void(nvtxDomainRegistration*)>>
        m_NvtxDomain;
    bool m_Stopped = false;
    NvDsInferStatus m_LastInferError = NVDSINFER_SUCCESS;
 
    RawOutputCallback m_RawoutputCb = nullptr;
 
    uint32_t m_IntervalCounter = 0;
    uint64_t m_UntrackedObjectWarnPts = UINT64_C(-1);
    bool m_1stInferDone = false;
};
 
} // namespace gstnvinferserver
 
#endif