infer_trtis_context.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
 *
 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
 * property and proprietary rights in and to this material, related
 * documentation and any modifications thereto. Any use, reproduction,
 * disclosure or distribution of this material and related documentation
 * without an express license agreement from NVIDIA CORPORATION or
 * its affiliates is strictly prohibited.
 */
 
#ifndef __INFER_TRTIS_CONTEXT_H__
#define __INFER_TRTIS_CONTEXT_H__
 
#include "infer_common.h"
#include "infer_cuda_context.h"
 
namespace nvdsinferserver {
 
class InferTrtISContext : public InferCudaContext {
public:
    InferTrtISContext();
 
    ~InferTrtISContext() override;
 
    NvDsInferStatus deinit() override;
 
    SharedCuStream& mainStream() override
    {
        assert(m_MainStream);
        return m_MainStream;
    }
 
private:
    NvDsInferStatus createNNBackend(const ic::BackendParams& params,
        int maxBatchSize, UniqBackend& backend);
 
    NvDsInferStatus allocateResource(
        const ic::InferenceConfig& config) override;
 
    NvDsInferStatus specifyBackendDims(
        BaseBackend* be, const std::string& model, const ic::BackendParams& params);
 
    NvDsInferStatus getConfigInOutMap(
        const ic::BackendParams& params,
        std::unordered_map<std::string, InferDims>& inputs,
        std::set<std::string>& outputs);
 
private:
    SharedCuStream m_MainStream;
    TrtISBackend* m_Backend{nullptr};
};
 
} // namespace nvdsinferserver
 
#endif