infer_cuda_context.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2018-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
 *
 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
 * property and proprietary rights in and to this material, related
 * documentation and any modifications thereto. Any use, reproduction,
 * disclosure or distribution of this material and related documentation
 * without an express license agreement from NVIDIA CORPORATION or
 * its affiliates is strictly prohibited.
 */
 
#ifndef __INFER_CUDA_CONTEXT_H__
#define __INFER_CUDA_CONTEXT_H__
 
#include <shared_mutex>
 
#include "infer_base_context.h"
#include "infer_common.h"
#include "infer_datatypes.h"
#include "infer_utils.h"
 
namespace nvdsinferserver {
 
class CropSurfaceConverter;
class NetworkPreprocessor;
class Postprocessor;
class CudaEventInPool;
 
class InferCudaContext : public InferBaseContext {
public:
    InferCudaContext();
    ~InferCudaContext() override;
 
    SharedSysMem acquireTensorHostBuf(const std::string& name, size_t bytes);
 
    SharedCuEvent acquireTensorHostEvent();
 
protected:
    NvDsInferStatus fixateInferenceInfo(
        const ic::InferenceConfig& config, BaseBackend& backend) override;
    NvDsInferStatus createPreprocessor(const ic::PreProcessParams& params,
        std::vector<UniqPreprocessor>& processors) override;
    NvDsInferStatus createPostprocessor(const ic::PostProcessParams& params,
        UniqPostprocessor& processor) override;
    NvDsInferStatus allocateResource(
        const ic::InferenceConfig& config) override;
    NvDsInferStatus preInference(
        SharedBatchArray& inputs, const ic::InferenceConfig& config) override;
    NvDsInferStatus extraOutputTensorCheck(
        SharedBatchArray& outputs, SharedOptions inOptions) override;
    void notifyError(NvDsInferStatus status) override;
    NvDsInferStatus deinit() override;
    void getNetworkInputInfo(NvDsInferNetworkInfo &networkInfo) override {
        networkInfo = m_NetworkImageInfo;
    }
    int tensorPoolSize() const;
 
private:
    NvDsInferStatus addHostTensorPool(const LayerDescription& layer,
        int poolSize, int gpuId);
 
    NvDsInferStatus ensureLstmInputReady(
        SharedBatchArray& inputs, const ic::InferenceConfig& config);
 
    NvDsInferStatus initFixedExtraInputLayers(
        SharedBatchArray& inputs, const ic::InferenceConfig& config);
 
    bool hasExtraProcess() const { return m_ExtraProcessor.get(); }
 
    NvDsInferStatus loadExtraProcessor(
        const ic::InferenceConfig& config, BaseBackend& backend, const std::string& primaryTensor);
 
protected:
 
    NvDsInferNetworkInfo m_NetworkImageInfo{0, 0, 0};
    InferMediaFormat m_NetworkImageFormat = InferMediaFormat::kRGB;
    std::string m_NetworkImageName;
    InferTensorOrder m_InputTensorOrder = InferTensorOrder::kNone;
    InferDataType m_InputDataType = InferDataType::kFp32;
 
    std::vector<SharedCudaTensorBuf> m_ExtraInputs;
    MapBufferPool<std::string, UniqSysMem> m_HostTensorPool;
    SharedBufPool<std::unique_ptr<CudaEventInPool>> m_HostTensorEvents;
 
    UniqLstmController m_LstmController;
    UniqStreamManager m_MultiStreamManager;
    UniqInferExtraProcessor m_ExtraProcessor;
 
    CropSurfaceConverter* m_SurfaceConverter = nullptr;
    NetworkPreprocessor* m_NetworkPreprocessor = nullptr;
    Postprocessor* m_FinalProcessor = nullptr;
};
 
} // namespace nvdsinferserver
 
#endif