Is this page helpful?

Using the Native Runtime API#

This tutorial demonstrates how to build a TensorRT-RTX engine and run inference programmatically using the C++ or Python API. The example uses a simple convolutional neural network to illustrate the core workflow.

Note

The native API is intended for advanced use cases where you need fine-grained control over network construction. Most users should use the ONNX-based workflow instead.

The workflow has two phases:

  1. Build: Define a network, configure the builder, and serialize the engine to a file

  2. Infer: Deserialize the engine, allocate GPU memory, and execute inference

Build the Engine#

Define a simple convolution network and serialize it to an engine file.

Listing 4 Build the engine (C++)#
 1#include "NvInfer.h"
 2#include <cuda_runtime.h>
 3#include <fstream>
 4#include <iostream>
 5#include <memory>
 6#include <vector>
 7
 8using namespace nvinfer1;
 9
10// Minimal logger implementation
11class Logger : public ILogger {
12    void log(Severity severity, const char* msg) noexcept override {
13        if (severity <= Severity::kWARNING) std::cout << msg << std::endl;
14    }
15} gLogger;
16
17void buildEngine(const char* fileName) {
18    auto builder = createInferBuilder(gLogger);
19    auto network = builder->createNetworkV2(0U);
20
21    // Input tensor of shape {1, 1, 3, 3}
22    auto input = network->addInput("input", DataType::kFLOAT, Dims4{1, 1, 3, 3});
23
24    // Convolution weights: 3x3 kernel, 1 input channel, 1 output channel
25    std::vector<float> weightValues(9, 1.f);
26    std::vector<float> biasValues(1, 0.f);
27    Weights W{DataType::kFLOAT, weightValues.data(), 9};
28    Weights B{DataType::kFLOAT, biasValues.data(), 1};
29
30    auto conv = network->addConvolutionNd(*input, 1, DimsHW{3, 3}, W, B);
31    conv->setStrideNd(Dims2{1, 1});
32
33    // Mark network output
34    auto output = conv->getOutput(0);
35    output->setName("output");
36    network->markOutput(*output);
37
38    // Build and serialize
39    auto config = builder->createBuilderConfig();
40    config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, 1ULL << 30);
41    auto engine = builder->buildSerializedNetwork(*network, *config);
42
43    // Save to file
44    std::ofstream out(fileName, std::ios::binary);
45    out.write(static_cast<const char*>(engine->data()), engine->size());
46    out.close();
47
48    // Clean up
49    // Note: While this example uses manual cleanup for clarity,
50    // production code should use smart pointers, such as std::unique_ptr
51    // to ensure proper resource management.
52    delete engine;
53    delete network;
54    delete config;
55    delete builder;
56}
Listing 5 Build the engine (Python)#
 1import tensorrt_rtx as trt_rtx
 2import numpy as np
 3
 4GLOBAL_LOGGER = trt_rtx.Logger(trt_rtx.Logger.WARNING)
 5
 6def build_engine(file_name):
 7    builder = trt_rtx.Builder(GLOBAL_LOGGER)
 8    network = builder.create_network(0)
 9
10    # Input tensor of shape (1, 1, 3, 3)
11    input_t = network.add_input(
12        name="input", dtype=trt_rtx.float32, shape=(1, 1, 3, 3))
13
14    # Convolution: 3x3 kernel, 1 output channel
15    w = np.full((1, 1, 3, 3), 1.0, dtype=np.float32)
16    b = np.zeros(1, dtype=np.float32)
17    conv = network.add_convolution_nd(
18        input=input_t, num_output_maps=1,
19        kernel_shape=(3, 3), kernel=w, bias=b)
20    conv.stride_nd = (1, 1)
21
22    # Mark output
23    conv.get_output(0).name = "output"
24    network.mark_output(conv.get_output(0))
25
26    # Build and serialize
27    config = builder.create_builder_config()
28    config.set_memory_pool_limit(trt_rtx.MemoryPoolType.WORKSPACE, 1 << 30)
29    host_mem = builder.build_serialized_network(network, config)
30
31    with open(file_name, "wb") as f:
32        f.write(host_mem)

Run Inference#

Load the serialized engine, allocate device memory, run inference, and read back the result.

 1void performInference(const char* fileName, const std::vector<float>& inputData) {
 2    // Note: Error handling omitted for brevity. Production code should
 3    // check return values and handle CUDA errors appropriately.
 4    auto runtime = createInferRuntime(gLogger);
 5
 6    // Deserialize engine from file
 7    std::ifstream is(fileName, std::ios::binary);
 8    is.seekg(0, is.end);
 9    size_t nbBytes = is.tellg();
10    is.seekg(0, is.beg);
11    std::vector<uint8_t> data(nbBytes);
12    is.read(reinterpret_cast<char*>(data.data()), nbBytes);
13    is.close();
14
15    auto engine = runtime->deserializeCudaEngine(data.data(), nbBytes);
16    auto execContext = engine->createExecutionContext();
17
18    // Allocate device memory
19    void* bindings[2];
20    size_t inputNbBytes = inputData.size() * sizeof(float);
21    size_t outputNbBytes = 1 * sizeof(float);
22    cudaMalloc(&bindings[0], inputNbBytes);
23    cudaMalloc(&bindings[1], outputNbBytes);
24
25    // Copy input to device
26    cudaMemcpy(bindings[0], inputData.data(), inputNbBytes, cudaMemcpyHostToDevice);
27
28    // Bind input/output addresses by tensor name (replaces deprecated executeV2 binding-array pattern)
29    const char* inputName = engine->getIOTensorName(0);
30    const char* outputName = engine->getIOTensorName(1);
31    execContext->setTensorAddress(inputName, bindings[0]);
32    execContext->setTensorAddress(outputName, bindings[1]);
33
34    // Run inference asynchronously on a CUDA stream
35    cudaStream_t stream;
36    cudaStreamCreate(&stream);
37    execContext->enqueueV3(stream);
38    cudaStreamSynchronize(stream);
39    cudaStreamDestroy(stream);
40
41    // Copy result back to host
42    float result;
43    cudaMemcpy(&result, bindings[1], outputNbBytes, cudaMemcpyDeviceToHost);
44    std::cout << "Result = " << result << std::endl;
45
46    // Clean up
47    // Note: Production code should use smart pointers and RAII patterns
48    // for automatic resource management. Error handling for CUDA calls
49    // should also be implemented.
50    cudaFree(bindings[0]);
51    cudaFree(bindings[1]);
52    delete execContext;
53    delete engine;
54    delete runtime;
55}

 1import numpy as np
 2import pycuda.driver as cuda
 3import pycuda.autoprimaryctx  # Use autoprimaryctx, not autoinit
 4
 5def perform_inference(file_name, input_data):
 6    runtime = trt_rtx.Runtime(GLOBAL_LOGGER)
 7
 8    with open(file_name, "rb") as f:
 9        engine = runtime.deserialize_cuda_engine(f.read())
10
11    exec_context = engine.create_execution_context()
12
13    # Allocate device memory
14    d_input = cuda.mem_alloc(input_data.nbytes)
15    output_data = np.empty((1,), dtype=np.float32)
16    d_output = cuda.mem_alloc(output_data.nbytes)
17
18    # Bind input/output addresses by tensor name (replaces deprecated execute_v2 binding-list pattern)
19    input_name = engine.get_tensor_name(0)
20    output_name = engine.get_tensor_name(1)
21    exec_context.set_tensor_address(input_name, int(d_input))
22    exec_context.set_tensor_address(output_name, int(d_output))
23
24    # Copy input to device and run inference asynchronously on a CUDA stream
25    cuda.memcpy_htod(d_input, input_data)
26    stream = cuda.Stream()
27    exec_context.execute_async_v3(stream.handle)
28    stream.synchronize()
29
30    # Copy result back to host
31    cuda.memcpy_dtoh(output_data, d_output)
32
33    # Clean up
34    d_input.free()
35    d_output.free()
36
37    return output_data

For thread-safety rules, buffer lifetime contracts, object lifetimes, and other runtime concepts, refer to How TensorRT-RTX Works.

Next Steps#