TensorRT 8.5.3
NvInfer.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
4 *
5 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
6 * property and proprietary rights in and to this material, related
7 * documentation and any modifications thereto. Any use, reproduction,
8 * disclosure or distribution of this material and related documentation
9 * without an express license agreement from NVIDIA CORPORATION or
10 * its affiliates is strictly prohibited.
11 */
12
13#ifndef NV_INFER_H
14#define NV_INFER_H
15
16#include "NvInferLegacyDims.h"
17#include "NvInferRuntime.h"
18
27//
30
36
42namespace nvinfer1
43{
44
52enum class LayerType : int32_t
53{
54 kCONVOLUTION = 0,
56 kACTIVATION = 2,
57 kPOOLING = 3,
58 kLRN = 4,
59 kSCALE = 5,
60 kSOFTMAX = 6,
61 kDECONVOLUTION = 7,
62 kCONCATENATION = 8,
63 kELEMENTWISE = 9,
64 kPLUGIN = 10,
65 kUNARY = 11,
66 kPADDING = 12,
67 kSHUFFLE = 13,
68 kREDUCE = 14,
69 kTOPK = 15,
70 kGATHER = 16,
71 kMATRIX_MULTIPLY = 17,
72 kRAGGED_SOFTMAX = 18,
73 kCONSTANT = 19,
74 kRNN_V2 = 20,
75 kIDENTITY = 21,
76 kPLUGIN_V2 = 22,
77 kSLICE = 23,
78 kSHAPE = 24,
79 kPARAMETRIC_RELU = 25,
80 kRESIZE = 26,
81 kTRIP_LIMIT = 27,
82 kRECURRENCE = 28,
83 kITERATOR = 29,
84 kLOOP_OUTPUT = 30,
85 kSELECT = 31,
86 kFILL = 32,
87 kQUANTIZE = 33,
88 kDEQUANTIZE = 34,
89 kCONDITION = 35,
92 kSCATTER = 38,
93 kEINSUM = 39,
94 kASSERTION = 40,
95 kONE_HOT = 41,
96 kNON_ZERO = 42,
97 kGRID_SAMPLE = 43,
98 kNMS = 44,
99};
100
106template <>
107constexpr inline int32_t EnumMax<LayerType>() noexcept
108{
109 return 45;
110}
111
118using TensorFormats = uint32_t;
119
125enum class ActivationType : int32_t
126{
127 kRELU = 0,
128 kSIGMOID = 1,
129 kTANH = 2,
130 kLEAKY_RELU = 3,
131 kELU = 4,
132 kSELU = 5,
133 kSOFTSIGN = 6,
134 kSOFTPLUS = 7,
135 kCLIP = 8,
136 kHARD_SIGMOID = 9,
137 kSCALED_TANH = 10,
139};
140
141namespace impl
142{
148template <>
150{
151 static constexpr int32_t kVALUE = 12;
152};
153} // namespace impl
154
171class ITensor : public INoCopy
172{
173public:
188 void setName(char const* name) noexcept
189 {
190 mImpl->setName(name);
191 }
192
200 char const* getName() const noexcept
201 {
202 return mImpl->getName();
203 }
204
219 void setDimensions(Dims dimensions) noexcept
220 {
221 mImpl->setDimensions(dimensions);
222 }
223
232 Dims getDimensions() const noexcept
233 {
234 return mImpl->getDimensions();
235 }
236
247 void setType(DataType type) noexcept
248 {
249 mImpl->setType(type);
250 }
251
259 DataType getType() const noexcept
260 {
261 return mImpl->getType();
262 }
263
274 bool setDynamicRange(float min, float max) noexcept
275 {
276 return mImpl->setDynamicRange(min, max);
277 }
278
282 bool isNetworkInput() const noexcept
283 {
284 return mImpl->isNetworkInput();
285 }
286
290 bool isNetworkOutput() const noexcept
291 {
292 return mImpl->isNetworkOutput();
293 }
294
312 void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
313 {
314 mImpl->setBroadcastAcrossBatch(broadcastAcrossBatch);
315 }
316
328 bool getBroadcastAcrossBatch() const noexcept
329 {
330 return mImpl->getBroadcastAcrossBatch();
331 }
332
339 {
340 return mImpl->getLocation();
341 }
342
353 void setLocation(TensorLocation location) noexcept
354 {
355 mImpl->setLocation(location);
356 }
357
363 bool dynamicRangeIsSet() const noexcept
364 {
365 return mImpl->dynamicRangeIsSet();
366 }
367
371 void resetDynamicRange() noexcept
372 {
373 mImpl->resetDynamicRange();
374 }
375
381 float getDynamicRangeMin() const noexcept
382 {
383 return mImpl->getDynamicRangeMin();
384 }
385
391 float getDynamicRangeMax() const noexcept
392 {
393 return mImpl->getDynamicRangeMax();
394 }
395
410 void setAllowedFormats(TensorFormats formats) noexcept
411 {
412 mImpl->setAllowedFormats(formats);
413 }
414
424 {
425 return mImpl->getAllowedFormats();
426 }
427
458 bool isShapeTensor() const noexcept
459 {
460 return mImpl->isShapeTensor();
461 }
462
481 bool isExecutionTensor() const noexcept
482 {
483 return mImpl->isExecutionTensor();
484 }
485
507 void setDimensionName(int32_t index, char const* name) noexcept
508 {
509 mImpl->setDimensionName(index, name);
510 }
511
522 char const* getDimensionName(int32_t index) const noexcept
523 {
524 return mImpl->getDimensionName(index);
525 }
526
527protected:
528 apiv::VTensor* mImpl;
529 virtual ~ITensor() noexcept = default;
530};
531
539class ILayer : public INoCopy
540{
541public:
547 LayerType getType() const noexcept
548 {
549 return mLayer->getType();
550 }
551
561 void setName(char const* name) noexcept
562 {
563 mLayer->setName(name);
564 }
565
571 char const* getName() const noexcept
572 {
573 return mLayer->getName();
574 }
575
579 int32_t getNbInputs() const noexcept
580 {
581 return mLayer->getNbInputs();
582 }
583
592 ITensor* getInput(int32_t index) const noexcept
593 {
594 return mLayer->getInput(index);
595 }
596
600 int32_t getNbOutputs() const noexcept
601 {
602 return mLayer->getNbOutputs();
603 }
604
611 ITensor* getOutput(int32_t index) const noexcept
612 {
613 return mLayer->getOutput(index);
614 }
615
628 void setInput(int32_t index, ITensor& tensor) noexcept
629 {
630 return mLayer->setInput(index, tensor);
631 }
632
656 void setPrecision(DataType dataType) noexcept
657 {
658 mLayer->setPrecision(dataType);
659 }
660
668 DataType getPrecision() const noexcept
669 {
670 return mLayer->getPrecision();
671 }
672
680 bool precisionIsSet() const noexcept
681 {
682 return mLayer->precisionIsSet();
683 }
684
690 void resetPrecision() noexcept
691 {
692 mLayer->resetPrecision();
693 }
694
728 void setOutputType(int32_t index, DataType dataType) noexcept
729 {
730 mLayer->setOutputType(index, dataType);
731 }
732
742 DataType getOutputType(int32_t index) const noexcept
743 {
744 return mLayer->getOutputType(index);
745 }
746
755 bool outputTypeIsSet(int32_t index) const noexcept
756 {
757 return mLayer->outputTypeIsSet(index);
758 }
759
767 void resetOutputType(int32_t index) noexcept
768 {
769 return mLayer->resetOutputType(index);
770 }
771
772protected:
773 virtual ~ILayer() noexcept = default;
774 apiv::VLayer* mLayer;
775};
776
999enum class PaddingMode : int32_t
1000{
1002 kEXPLICIT_ROUND_UP = 1,
1003 kSAME_UPPER = 2,
1004 kSAME_LOWER = 3,
1005 kCAFFE_ROUND_DOWN = 4,
1006 kCAFFE_ROUND_UP = 5
1007};
1008
1009namespace impl
1010{
1016template <>
1018{
1019 static constexpr int32_t kVALUE = 6;
1020};
1021} // namespace impl
1022
1036{
1037public:
1047 TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
1048 {
1049 mImpl->setKernelSize(kernelSize);
1050 }
1051
1060 {
1061 return mImpl->getKernelSize();
1062 }
1063
1071 void setNbOutputMaps(int32_t nbOutputMaps) noexcept
1072 {
1073 mImpl->setNbOutputMaps(nbOutputMaps);
1074 }
1075
1081 int32_t getNbOutputMaps() const noexcept
1082 {
1083 return mImpl->getNbOutputMaps();
1084 }
1085
1097 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
1098 {
1099 mImpl->setStride(stride);
1100 }
1101
1108 {
1109 return mImpl->getStride();
1110 }
1111
1127 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
1128 {
1129 return mImpl->setPadding(padding);
1130 }
1131
1140 {
1141 return mImpl->getPadding();
1142 }
1143
1159 void setNbGroups(int32_t nbGroups) noexcept
1160 {
1161 mImpl->setNbGroups(nbGroups);
1162 }
1163
1169 int32_t getNbGroups() const noexcept
1170 {
1171 return mImpl->getNbGroups();
1172 }
1173
1183 void setKernelWeights(Weights weights) noexcept
1184 {
1185 mImpl->setKernelWeights(weights);
1186 }
1187
1193 Weights getKernelWeights() const noexcept
1194 {
1195 return mImpl->getKernelWeights();
1196 }
1197
1208 void setBiasWeights(Weights weights) noexcept
1209 {
1210 mImpl->setBiasWeights(weights);
1211 }
1212
1218 Weights getBiasWeights() const noexcept
1219 {
1220 return mImpl->getBiasWeights();
1221 }
1222
1234 TRT_DEPRECATED void setDilation(DimsHW dilation) noexcept
1235 {
1236 return mImpl->setDilation(dilation);
1237 }
1238
1247 {
1248 return mImpl->getDilation();
1249 }
1250
1263 void setPrePadding(Dims padding) noexcept
1264 {
1265 mImpl->setPrePadding(padding);
1266 }
1267
1273 Dims getPrePadding() const noexcept
1274 {
1275 return mImpl->getPrePadding();
1276 }
1277
1290 void setPostPadding(Dims padding) noexcept
1291 {
1292 mImpl->setPostPadding(padding);
1293 }
1294
1300 Dims getPostPadding() const noexcept
1301 {
1302 return mImpl->getPostPadding();
1303 }
1304
1314 void setPaddingMode(PaddingMode paddingMode) noexcept
1315 {
1316 mImpl->setPaddingMode(paddingMode);
1317 }
1318
1327 {
1328 return mImpl->getPaddingMode();
1329 }
1330
1339 void setKernelSizeNd(Dims kernelSize) noexcept
1340 {
1341 mImpl->setKernelSizeNd(kernelSize);
1342 }
1343
1349 Dims getKernelSizeNd() const noexcept
1350 {
1351 return mImpl->getKernelSizeNd();
1352 }
1353
1364 void setStrideNd(Dims stride) noexcept
1365 {
1366 mImpl->setStrideNd(stride);
1367 }
1368
1374 Dims getStrideNd() const noexcept
1375 {
1376 return mImpl->getStrideNd();
1377 }
1378
1392 void setPaddingNd(Dims padding) noexcept
1393 {
1394 mImpl->setPaddingNd(padding);
1395 }
1396
1404 Dims getPaddingNd() const noexcept
1405 {
1406 return mImpl->getPaddingNd();
1407 }
1408
1418 void setDilationNd(Dims dilation) noexcept
1419 {
1420 mImpl->setDilationNd(dilation);
1421 }
1422
1428 Dims getDilationNd() const noexcept
1429 {
1430 return mImpl->getDilationNd();
1431 }
1432
1446 using ILayer::setInput;
1447
1448protected:
1449 virtual ~IConvolutionLayer() noexcept = default;
1450 apiv::VConvolutionLayer* mImpl;
1451};
1452
1485{
1486public:
1494 void setNbOutputChannels(int32_t nbOutputs) noexcept
1495 {
1496 mImpl->setNbOutputChannels(nbOutputs);
1497 }
1498
1504 int32_t getNbOutputChannels() const noexcept
1505 {
1506 return mImpl->getNbOutputChannels();
1507 }
1508
1514 void setKernelWeights(Weights weights) noexcept
1515 {
1516 mImpl->setKernelWeights(weights);
1517 }
1518
1524 Weights getKernelWeights() const noexcept
1525 {
1526 return mImpl->getKernelWeights();
1527 }
1528
1536 void setBiasWeights(Weights weights) noexcept
1537 {
1538 mImpl->setBiasWeights(weights);
1539 }
1540
1546 Weights getBiasWeights() const noexcept
1547 {
1548 return mImpl->getBiasWeights();
1549 }
1550
1572 using ILayer::setInput;
1573
1574protected:
1575 virtual ~IFullyConnectedLayer() noexcept = default;
1576 apiv::VFullyConnectedLayer* mImpl;
1577};
1578
1593{
1594public:
1603 {
1604 mImpl->setActivationType(type);
1605 }
1606
1613 {
1614 return mImpl->getActivationType();
1615 }
1616
1627 void setAlpha(float alpha) noexcept
1628 {
1629 mImpl->setAlpha(alpha);
1630 }
1631
1641 void setBeta(float beta) noexcept
1642 {
1643 mImpl->setBeta(beta);
1644 }
1645
1650 float getAlpha() const noexcept
1651 {
1652 return mImpl->getAlpha();
1653 }
1654
1659 float getBeta() const noexcept
1660 {
1661 return mImpl->getBeta();
1662 }
1663
1664protected:
1665 virtual ~IActivationLayer() noexcept = default;
1666 apiv::VActivationLayer* mImpl;
1667};
1668
1674enum class PoolingType : int32_t
1675{
1676 kMAX = 0, // Maximum over elements
1677 kAVERAGE = 1, // Average over elements. If the tensor is padded, the count includes the padding
1678 kMAX_AVERAGE_BLEND = 2 // Blending between max and average pooling: (1-blendFactor)*maxPool + blendFactor*avgPool
1679};
1680
1681namespace impl
1682{
1688template <>
1690{
1691 static constexpr int32_t kVALUE = 3;
1692};
1693} // namespace impl
1694
1706class IPoolingLayer : public ILayer
1707{
1708public:
1716 void setPoolingType(PoolingType type) noexcept
1717 {
1718 mImpl->setPoolingType(type);
1719 }
1720
1727 {
1728 return mImpl->getPoolingType();
1729 }
1730
1740 TRT_DEPRECATED void setWindowSize(DimsHW windowSize) noexcept
1741 {
1742 mImpl->setWindowSize(windowSize);
1743 }
1744
1753 {
1754 return mImpl->getWindowSize();
1755 }
1756
1768 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
1769 {
1770 mImpl->setStride(stride);
1771 }
1772
1781 {
1782 return mImpl->getStride();
1783 }
1784
1796 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
1797 {
1798 mImpl->setPadding(padding);
1799 }
1800
1811 {
1812 return mImpl->getPadding();
1813 }
1814
1825 void setBlendFactor(float blendFactor) noexcept
1826 {
1827 mImpl->setBlendFactor(blendFactor);
1828 }
1829
1838 float getBlendFactor() const noexcept
1839 {
1840 return mImpl->getBlendFactor();
1841 }
1842
1855 void setAverageCountExcludesPadding(bool exclusive) noexcept
1856 {
1857 mImpl->setAverageCountExcludesPadding(exclusive);
1858 }
1859
1867 {
1868 return mImpl->getAverageCountExcludesPadding();
1869 }
1870
1884 void setPrePadding(Dims padding) noexcept
1885 {
1886 mImpl->setPrePadding(padding);
1887 }
1888
1894 Dims getPrePadding() const noexcept
1895 {
1896 return mImpl->getPrePadding();
1897 }
1898
1912 void setPostPadding(Dims padding) noexcept
1913 {
1914 mImpl->setPostPadding(padding);
1915 }
1916
1922 Dims getPostPadding() const noexcept
1923 {
1924 return mImpl->getPostPadding();
1925 }
1926
1935 void setPaddingMode(PaddingMode paddingMode) noexcept
1936 {
1937 mImpl->setPaddingMode(paddingMode);
1938 }
1939
1947 {
1948 return mImpl->getPaddingMode();
1949 }
1950
1959 void setWindowSizeNd(Dims windowSize) noexcept
1960 {
1961 mImpl->setWindowSizeNd(windowSize);
1962 }
1963
1969 Dims getWindowSizeNd() const noexcept
1970 {
1971 return mImpl->getWindowSizeNd();
1972 }
1973
1984 void setStrideNd(Dims stride) noexcept
1985 {
1986 mImpl->setStrideNd(stride);
1987 }
1988
1994 Dims getStrideNd() const noexcept
1995 {
1996 return mImpl->getStrideNd();
1997 }
1998
2013 void setPaddingNd(Dims padding) noexcept
2014 {
2015 mImpl->setPaddingNd(padding);
2016 }
2017
2025 Dims getPaddingNd() const noexcept
2026 {
2027 return mImpl->getPaddingNd();
2028 }
2029
2030protected:
2031 virtual ~IPoolingLayer() noexcept = default;
2032 apiv::VPoolingLayer* mImpl;
2033};
2034
2044class ILRNLayer : public ILayer
2045{
2046public:
2056 void setWindowSize(int32_t windowSize) noexcept
2057 {
2058 mImpl->setWindowSize(windowSize);
2059 }
2060
2066 int32_t getWindowSize() const noexcept
2067 {
2068 return mImpl->getWindowSize();
2069 }
2070
2077 void setAlpha(float alpha) noexcept
2078 {
2079 mImpl->setAlpha(alpha);
2080 }
2081
2087 float getAlpha() const noexcept
2088 {
2089 return mImpl->getAlpha();
2090 }
2091
2098 void setBeta(float beta) noexcept
2099 {
2100 mImpl->setBeta(beta);
2101 }
2102
2108 float getBeta() const noexcept
2109 {
2110 return mImpl->getBeta();
2111 }
2112
2119 void setK(float k) noexcept
2120 {
2121 mImpl->setK(k);
2122 }
2123
2129 float getK() const noexcept
2130 {
2131 return mImpl->getK();
2132 }
2133
2134protected:
2135 virtual ~ILRNLayer() noexcept = default;
2136 apiv::VLRNLayer* mImpl;
2137};
2138
2144enum class ScaleMode : int32_t
2145{
2146 kUNIFORM = 0,
2147 kCHANNEL = 1,
2148 kELEMENTWISE = 2
2149};
2150
2156template <>
2157constexpr inline int32_t EnumMax<ScaleMode>() noexcept
2158{
2159 return 3;
2160}
2161
2188class IScaleLayer : public ILayer
2189{
2190public:
2196 void setMode(ScaleMode mode) noexcept
2197 {
2198 mImpl->setMode(mode);
2199 }
2200
2206 ScaleMode getMode() const noexcept
2207 {
2208 return mImpl->getMode();
2209 }
2210
2216 void setShift(Weights shift) noexcept
2217 {
2218 mImpl->setShift(shift);
2219 }
2220
2226 Weights getShift() const noexcept
2227 {
2228 return mImpl->getShift();
2229 }
2230
2236 void setScale(Weights scale) noexcept
2237 {
2238 mImpl->setScale(scale);
2239 }
2240
2246 Weights getScale() const noexcept
2247 {
2248 return mImpl->getScale();
2249 }
2250
2256 void setPower(Weights power) noexcept
2257 {
2258 mImpl->setPower(power);
2259 }
2260
2266 Weights getPower() const noexcept
2267 {
2268 return mImpl->getPower();
2269 }
2270
2281 int32_t getChannelAxis() const noexcept
2282 {
2283 return mImpl->getChannelAxis();
2284 }
2285
2302 void setChannelAxis(int32_t channelAxis) noexcept
2303 {
2304 mImpl->setChannelAxis(channelAxis);
2305 }
2306
2307protected:
2308 virtual ~IScaleLayer() noexcept = default;
2309 apiv::VScaleLayer* mImpl;
2310};
2311
2333class ISoftMaxLayer : public ILayer
2334{
2335public:
2366 void setAxes(uint32_t axes) noexcept
2367 {
2368 mImpl->setAxes(axes);
2369 }
2370
2376 uint32_t getAxes() const noexcept
2377 {
2378 return mImpl->getAxes();
2379 }
2380
2381protected:
2382 virtual ~ISoftMaxLayer() noexcept = default;
2383 apiv::VSoftMaxLayer* mImpl;
2384};
2385
2399{
2400public:
2413 void setAxis(int32_t axis) noexcept
2414 {
2415 mImpl->setAxis(axis);
2416 }
2417
2423 int32_t getAxis() const noexcept
2424 {
2425 return mImpl->getAxis();
2426 }
2427
2428protected:
2429 virtual ~IConcatenationLayer() noexcept = default;
2430 apiv::VConcatenationLayer* mImpl;
2431};
2432
2441{
2442public:
2454 TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
2455 {
2456 mImpl->setKernelSize(kernelSize);
2457 }
2458
2467 {
2468 return mImpl->getKernelSize();
2469 }
2470
2478 void setNbOutputMaps(int32_t nbOutputMaps) noexcept
2479 {
2480 mImpl->setNbOutputMaps(nbOutputMaps);
2481 }
2482
2488 int32_t getNbOutputMaps() const noexcept
2489 {
2490 return mImpl->getNbOutputMaps();
2491 }
2492
2504 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
2505 {
2506 mImpl->setStride(stride);
2507 }
2508
2517 {
2518 return mImpl->getStride();
2519 }
2520
2536 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
2537 {
2538 mImpl->setPadding(padding);
2539 }
2540
2551 {
2552 return mImpl->getPadding();
2553 }
2554
2570 void setNbGroups(int32_t nbGroups) noexcept
2571 {
2572 mImpl->setNbGroups(nbGroups);
2573 }
2574
2580 int32_t getNbGroups() const noexcept
2581 {
2582 return mImpl->getNbGroups();
2583 }
2584
2594 void setKernelWeights(Weights weights) noexcept
2595 {
2596 mImpl->setKernelWeights(weights);
2597 }
2598
2604 Weights getKernelWeights() const noexcept
2605 {
2606 return mImpl->getKernelWeights();
2607 }
2608
2619 void setBiasWeights(Weights weights) noexcept
2620 {
2621 mImpl->setBiasWeights(weights);
2622 }
2623
2629 Weights getBiasWeights() const noexcept
2630 {
2631 return mImpl->getBiasWeights();
2632 }
2633
2647 void setPrePadding(Dims padding) noexcept
2648 {
2649 mImpl->setPrePadding(padding);
2650 }
2651
2657 Dims getPrePadding() const noexcept
2658 {
2659 return mImpl->getPrePadding();
2660 }
2661
2675 void setPostPadding(Dims padding) noexcept
2676 {
2677 mImpl->setPostPadding(padding);
2678 }
2679
2685 Dims getPostPadding() const noexcept
2686 {
2687 return mImpl->getPostPadding();
2688 }
2689
2699 void setPaddingMode(PaddingMode paddingMode) noexcept
2700 {
2701 mImpl->setPaddingMode(paddingMode);
2702 }
2703
2712 {
2713 return mImpl->getPaddingMode();
2714 }
2715
2726 void setKernelSizeNd(Dims kernelSize) noexcept
2727 {
2728 mImpl->setKernelSizeNd(kernelSize);
2729 }
2730
2736 Dims getKernelSizeNd() const noexcept
2737 {
2738 return mImpl->getKernelSizeNd();
2739 }
2740
2753 void setStrideNd(Dims stride) noexcept
2754 {
2755 mImpl->setStrideNd(stride);
2756 }
2757
2763 Dims getStrideNd() const noexcept
2764 {
2765 return mImpl->getStrideNd();
2766 }
2767
2781 void setPaddingNd(Dims padding) noexcept
2782 {
2783 mImpl->setPaddingNd(padding);
2784 }
2785
2793 Dims getPaddingNd() const noexcept
2794 {
2795 return mImpl->getPaddingNd();
2796 }
2797
2809 using ILayer::setInput;
2810
2817 void setDilationNd(Dims dilation) noexcept
2818 {
2819 mImpl->setDilationNd(dilation);
2820 }
2821
2827 Dims getDilationNd() const noexcept
2828 {
2829 return mImpl->getDilationNd();
2830 }
2831
2832protected:
2833 virtual ~IDeconvolutionLayer() noexcept = default;
2834 apiv::VDeconvolutionLayer* mImpl;
2835};
2836
2850enum class ElementWiseOperation : int32_t
2851{
2852 kSUM = 0,
2853 kPROD = 1,
2854 kMAX = 2,
2855 kMIN = 3,
2856 kSUB = 4,
2857 kDIV = 5,
2858 kPOW = 6,
2859 kFLOOR_DIV = 7,
2860 kAND = 8,
2861 kOR = 9,
2862 kXOR = 10,
2863 kEQUAL = 11,
2864 kGREATER = 12,
2865 kLESS = 13
2866};
2867
2868namespace impl
2869{
2875template <>
2877{
2878 static constexpr int32_t kVALUE = 14;
2879};
2880} // namespace impl
2881
2902{
2903public:
2914 {
2915 return mImpl->setOperation(op);
2916 }
2917
2926 {
2927 return mImpl->getOperation();
2928 }
2929
2930protected:
2931 apiv::VElementWiseLayer* mImpl;
2932 virtual ~IElementWiseLayer() noexcept = default;
2933};
2934
2940enum class GatherMode : int32_t
2941{
2942 kDEFAULT = 0,
2943 kELEMENT = 1,
2944 kND = 2
2945};
2946
2952template <>
2953constexpr inline int32_t EnumMax<GatherMode>() noexcept
2954{
2955 return 3;
2956}
2957
3036class IGatherLayer : public ILayer
3037{
3038public:
3048 void setGatherAxis(int32_t axis) noexcept
3049 {
3050 mImpl->setGatherAxis(axis);
3051 }
3052
3059 int32_t getGatherAxis() const noexcept
3060 {
3061 return mImpl->getGatherAxis();
3062 }
3063
3080 void setNbElementWiseDims(int32_t elementWiseDims) noexcept
3081 {
3082 mImpl->setNbElementWiseDims(elementWiseDims);
3083 }
3084
3090 int32_t getNbElementWiseDims() const noexcept
3091 {
3092 return mImpl->getNbElementWiseDims();
3093 }
3094
3100 void setMode(GatherMode mode) noexcept
3101 {
3102 mImpl->setMode(mode);
3103 }
3104
3110 GatherMode getMode() const noexcept
3111 {
3112 return mImpl->getMode();
3113 }
3114
3115protected:
3116 apiv::VGatherLayer* mImpl;
3117 virtual ~IGatherLayer() noexcept = default;
3118};
3119
3199enum class RNNOperation : int32_t
3200{
3201 kRELU = 0,
3202 kTANH = 1,
3203 kLSTM = 2,
3204 kGRU = 3
3205};
3206
3212template <>
3213constexpr inline int32_t EnumMax<RNNOperation>() noexcept
3214{
3215 return 4;
3216}
3217
3225enum class RNNDirection : int32_t
3226{
3227 kUNIDIRECTION = 0,
3228 kBIDIRECTION = 1
3229};
3230
3236template <>
3237constexpr inline int32_t EnumMax<RNNDirection>() noexcept
3238{
3239 return 2;
3240}
3241
3257enum class RNNInputMode : int32_t
3258{
3259 kLINEAR = 0,
3260 kSKIP = 1
3261};
3262
3268template <>
3269constexpr inline int32_t EnumMax<RNNInputMode>() noexcept
3270{
3271 return 2;
3272}
3273
3281enum class RNNGateType : int32_t
3282{
3283 kINPUT = 0,
3284 kOUTPUT = 1,
3285 kFORGET = 2,
3286 kUPDATE = 3,
3287 kRESET = 4,
3288 kCELL = 5,
3289 kHIDDEN = 6
3290};
3291
3297template <>
3298constexpr inline int32_t EnumMax<RNNGateType>() noexcept
3299{
3300 return 7;
3301}
3302
3316{
3317public:
3318 int32_t getLayerCount() const noexcept
3319 {
3320 return mImpl->getLayerCount();
3321 }
3322 int32_t getHiddenSize() const noexcept
3323 {
3324 return mImpl->getHiddenSize();
3325 }
3326 int32_t getMaxSeqLength() const noexcept
3327 {
3328 return mImpl->getMaxSeqLength();
3329 }
3330 int32_t getDataLength() const noexcept
3331 {
3332 return mImpl->getDataLength();
3333 }
3334
3349 void setSequenceLengths(ITensor& seqLengths) noexcept
3350 {
3351 return mImpl->setSequenceLengths(seqLengths);
3352 }
3353
3361 ITensor* getSequenceLengths() const noexcept
3362 {
3363 return mImpl->getSequenceLengths();
3364 }
3365
3371 void setOperation(RNNOperation op) noexcept
3372 {
3373 mImpl->setOperation(op);
3374 }
3375
3382 {
3383 return mImpl->getOperation();
3384 }
3385
3391 void setInputMode(RNNInputMode op) noexcept
3392 {
3393 mImpl->setInputMode(op);
3394 }
3395
3402 {
3403 return mImpl->getInputMode();
3404 }
3405
3416 void setDirection(RNNDirection op) noexcept
3417 {
3418 mImpl->setDirection(op);
3419 }
3420
3427 {
3428 return mImpl->getDirection();
3429 }
3430
3485 void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept
3486 {
3487 mImpl->setWeightsForGate(layerIndex, gate, isW, weights);
3488 }
3489
3495 Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
3496 {
3497 return mImpl->getWeightsForGate(layerIndex, gate, isW);
3498 }
3499
3520 void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept
3521 {
3522 mImpl->setBiasForGate(layerIndex, gate, isW, bias);
3523 }
3524
3530 Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
3531 {
3532 return mImpl->getBiasForGate(layerIndex, gate, isW);
3533 }
3534
3547 void setHiddenState(ITensor& hidden) noexcept
3548 {
3549 mImpl->setHiddenState(hidden);
3550 }
3551
3557 ITensor* getHiddenState() const noexcept
3558 {
3559 return mImpl->getHiddenState();
3560 }
3561
3576 void setCellState(ITensor& cell) noexcept
3577 {
3578 mImpl->setCellState(cell);
3579 }
3580
3586 ITensor* getCellState() const noexcept
3587 {
3588 return mImpl->getCellState();
3589 }
3590
3591protected:
3592 apiv::VRNNv2Layer* mImpl;
3593 virtual ~IRNNv2Layer() noexcept = default;
3594};
3595
3606{
3607public:
3614 {
3615 return mImpl->getPlugin();
3616 }
3617
3618protected:
3619 apiv::VPluginV2Layer* mImpl;
3620 virtual ~IPluginV2Layer() noexcept = default;
3621};
3622
3636enum class UnaryOperation : int32_t
3637{
3638 kEXP = 0,
3639 kLOG = 1,
3640 kSQRT = 2,
3641 kRECIP = 3,
3642 kABS = 4,
3643 kNEG = 5,
3644 kSIN = 6,
3645 kCOS = 7,
3646 kTAN = 8,
3647 kSINH = 9,
3648 kCOSH = 10,
3649 kASIN = 11,
3650 kACOS = 12,
3651 kATAN = 13,
3652 kASINH = 14,
3653 kACOSH = 15,
3654 kATANH = 16,
3655 kCEIL = 17,
3656 kFLOOR = 18,
3657 kERF = 19,
3658 kNOT = 20,
3659 kSIGN = 21,
3660 kROUND = 22
3661};
3662
3668template <>
3669constexpr inline int32_t EnumMax<UnaryOperation>() noexcept
3670{
3671 return 23;
3672}
3673
3681class IUnaryLayer : public ILayer
3682{
3683public:
3692 {
3693 mImpl->setOperation(op);
3694 }
3695
3702 {
3703 return mImpl->getOperation();
3704 }
3705
3706protected:
3707 apiv::VUnaryLayer* mImpl;
3708 virtual ~IUnaryLayer() noexcept = default;
3709};
3710
3729enum class ReduceOperation : int32_t
3730{
3731 kSUM = 0,
3732 kPROD = 1,
3733 kMAX = 2,
3734 kMIN = 3,
3735 kAVG = 4
3736};
3737
3743template <>
3744constexpr inline int32_t EnumMax<ReduceOperation>() noexcept
3745{
3746 return 5;
3747}
3748
3756class IReduceLayer : public ILayer
3757{
3758public:
3765 {
3766 mImpl->setOperation(op);
3767 }
3768
3775 {
3776 return mImpl->getOperation();
3777 }
3778
3784 void setReduceAxes(uint32_t reduceAxes) noexcept
3785 {
3786 mImpl->setReduceAxes(reduceAxes);
3787 }
3788
3794 uint32_t getReduceAxes() const noexcept
3795 {
3796 return mImpl->getReduceAxes();
3797 }
3798
3804 void setKeepDimensions(bool keepDimensions) noexcept
3805 {
3806 mImpl->setKeepDimensions(keepDimensions);
3807 }
3808
3814 bool getKeepDimensions() const noexcept
3815 {
3816 return mImpl->getKeepDimensions();
3817 }
3818
3819protected:
3820 apiv::VReduceLayer* mImpl;
3821 virtual ~IReduceLayer() noexcept = default;
3822};
3823
3834class IPaddingLayer : public ILayer
3835{
3836public:
3846 TRT_DEPRECATED void setPrePadding(DimsHW padding) noexcept
3847 {
3848 mImpl->setPrePadding(padding);
3849 }
3850
3859 {
3860 return mImpl->getPrePadding();
3861 }
3862
3872 TRT_DEPRECATED void setPostPadding(DimsHW padding) noexcept
3873 {
3874 mImpl->setPostPadding(padding);
3875 }
3876
3885 {
3886 return mImpl->getPostPadding();
3887 }
3888
3898 void setPrePaddingNd(Dims padding) noexcept
3899 {
3900 mImpl->setPrePaddingNd(padding);
3901 }
3902
3910 Dims getPrePaddingNd() const noexcept
3911 {
3912 return mImpl->getPrePaddingNd();
3913 }
3914
3924 void setPostPaddingNd(Dims padding) noexcept
3925 {
3926 mImpl->setPostPaddingNd(padding);
3927 }
3928
3936 Dims getPostPaddingNd() const noexcept
3937 {
3938 return mImpl->getPostPaddingNd();
3939 }
3940
3941protected:
3942 apiv::VPaddingLayer* mImpl;
3943 virtual ~IPaddingLayer() noexcept = default;
3944};
3945
3947{
3954 int32_t order[Dims::MAX_DIMS];
3955};
3956
3969class IShuffleLayer : public ILayer
3970{
3971public:
3981 void setFirstTranspose(Permutation permutation) noexcept
3982 {
3983 mImpl->setFirstTranspose(permutation);
3984 }
3985
3994 {
3995 return mImpl->getFirstTranspose();
3996 }
3997
4018 void setReshapeDimensions(Dims dimensions) noexcept
4019 {
4020 mImpl->setReshapeDimensions(dimensions);
4021 }
4022
4032 {
4033 return mImpl->getReshapeDimensions();
4034 }
4035
4041 //
4064 using ILayer::setInput;
4065
4078 void setSecondTranspose(Permutation permutation) noexcept
4079 {
4080 mImpl->setSecondTranspose(permutation);
4081 }
4082
4091 {
4092 return mImpl->getSecondTranspose();
4093 }
4094
4106 void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
4107 {
4108 return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);
4109 }
4110
4119 bool getZeroIsPlaceholder() const noexcept
4120 {
4121 return mImpl->getZeroIsPlaceholder();
4122 }
4123
4124protected:
4125 apiv::VShuffleLayer* mImpl;
4126 virtual ~IShuffleLayer() noexcept = default;
4127};
4128
4134enum class SampleMode : int32_t
4135{
4136 kSTRICT_BOUNDS = 0,
4138 kWRAP = 1,
4139 kCLAMP = 2,
4140 kFILL = 3,
4141 kREFLECT = 4,
4144};
4145
4148
4154template <>
4155constexpr inline int32_t EnumMax<SampleMode>() noexcept
4156{
4157 return 5;
4158}
4159
4202class ISliceLayer : public ILayer
4203{
4204public:
4214 void setStart(Dims start) noexcept
4215 {
4216 mImpl->setStart(start);
4217 }
4218
4229 Dims getStart() const noexcept
4230 {
4231 return mImpl->getStart();
4232 }
4233
4243 void setSize(Dims size) noexcept
4244 {
4245 return mImpl->setSize(size);
4246 }
4247
4258 Dims getSize() const noexcept
4259 {
4260 return mImpl->getSize();
4261 }
4262
4272 void setStride(Dims stride) noexcept
4273 {
4274 mImpl->setStride(stride);
4275 }
4276
4287 Dims getStride() const noexcept
4288 {
4289 return mImpl->getStride();
4290 }
4291
4297 void setMode(SliceMode mode) noexcept
4298 {
4299 mImpl->setMode(mode);
4300 }
4301
4307 SliceMode getMode() const noexcept
4308 {
4309 return mImpl->getMode();
4310 }
4311
4333 using ILayer::setInput;
4334
4335protected:
4336 apiv::VSliceLayer* mImpl;
4337 virtual ~ISliceLayer() noexcept = default;
4338};
4339
4352class IShapeLayer : public ILayer
4353{
4354protected:
4355 apiv::VShapeLayer* mImpl;
4356 virtual ~IShapeLayer() noexcept = default;
4357};
4358
4364enum class TopKOperation : int32_t
4365{
4366 kMAX = 0,
4367 kMIN = 1,
4368};
4369
4375template <>
4376constexpr inline int32_t EnumMax<TopKOperation>() noexcept
4377{
4378 return 2;
4379}
4380
4388class ITopKLayer : public ILayer
4389{
4390public:
4396 void setOperation(TopKOperation op) noexcept
4397 {
4398 mImpl->setOperation(op);
4399 }
4400
4407 {
4408 return mImpl->getOperation();
4409 }
4410
4418 void setK(int32_t k) noexcept
4419 {
4420 mImpl->setK(k);
4421 }
4422
4428 int32_t getK() const noexcept
4429 {
4430 return mImpl->getK();
4431 }
4432
4438 void setReduceAxes(uint32_t reduceAxes) noexcept
4439 {
4440 mImpl->setReduceAxes(reduceAxes);
4441 }
4442
4448 uint32_t getReduceAxes() const noexcept
4449 {
4450 return mImpl->getReduceAxes();
4451 }
4452
4453protected:
4454 apiv::VTopKLayer* mImpl;
4455 virtual ~ITopKLayer() noexcept = default;
4456};
4457
4464enum class MatrixOperation : int32_t
4465{
4469 kNONE,
4470
4472 kTRANSPOSE,
4473
4484 kVECTOR
4485};
4486
4492template <>
4493constexpr inline int32_t EnumMax<MatrixOperation>() noexcept
4494{
4495 return 3;
4496}
4497
4524{
4525public:
4532 void setOperation(int32_t index, MatrixOperation op) noexcept
4533 {
4534 mImpl->setOperation(index, op);
4535 }
4536
4544 MatrixOperation getOperation(int32_t index) const noexcept
4545 {
4546 return mImpl->getOperation(index);
4547 }
4548
4549protected:
4550 apiv::VMatrixMultiplyLayer* mImpl;
4551 virtual ~IMatrixMultiplyLayer() noexcept = default;
4552};
4553
4575class INonZeroLayer : public ILayer
4576{
4577protected:
4578 virtual ~INonZeroLayer() noexcept = default;
4579 apiv::VNonZeroLayer* mImpl;
4580};
4581
4597{
4598protected:
4599 apiv::VRaggedSoftMaxLayer* mImpl;
4600 virtual ~IRaggedSoftMaxLayer() noexcept = default;
4601};
4602
4631{
4632protected:
4633 apiv::VIdentityLayer* mImpl;
4634 virtual ~IIdentityLayer() noexcept = default;
4635};
4636
4646{
4647public:
4657 void setWeights(Weights weights) noexcept
4658 {
4659 mImpl->setWeights(weights);
4660 }
4661
4667 Weights getWeights() const noexcept
4668 {
4669 return mImpl->getWeights();
4670 }
4671
4679 void setDimensions(Dims dimensions) noexcept
4680 {
4681 mImpl->setDimensions(dimensions);
4682 }
4683
4691 Dims getDimensions() const noexcept
4692 {
4693 return mImpl->getDimensions();
4694 }
4695
4696protected:
4697 apiv::VConstantLayer* mImpl;
4698 virtual ~IConstantLayer() noexcept = default;
4699};
4700
4711{
4712protected:
4713 apiv::VParametricReLULayer* mImpl;
4714 virtual ~IParametricReLULayer() noexcept = default;
4715};
4716
4722enum class InterpolationMode : int32_t
4723{
4724 kNEAREST = 0,
4725 kLINEAR = 1,
4726 kCUBIC = 2
4727};
4728
4731
4732namespace impl
4733{
4739template <>
4741{
4742 static constexpr int32_t kVALUE = 3;
4743};
4744} // namespace impl
4745
4754{
4767 kALIGN_CORNERS = 0,
4768
4775 kASYMMETRIC = 1,
4776
4783 kHALF_PIXEL = 2,
4784};
4785
4786namespace impl
4787{
4793template <>
4795{
4796 static constexpr int32_t kVALUE = 3;
4797};
4798} // namespace impl
4799
4807enum class ResizeSelector : int32_t
4808{
4810 kFORMULA = 0,
4811
4813 kUPPER = 1,
4814};
4815
4816namespace impl
4817{
4823template <>
4825{
4826 static constexpr int32_t kVALUE = 2;
4827};
4828} // namespace impl
4829
4837enum class ResizeRoundMode : int32_t
4838{
4840 kHALF_UP = 0,
4841
4843 kHALF_DOWN = 1,
4844
4846 kFLOOR = 2,
4847
4849 kCEIL = 3,
4850};
4851
4852namespace impl
4853{
4859template <>
4861{
4862 static constexpr int32_t kVALUE = 4;
4863};
4864} // namespace impl
4865
4902class IResizeLayer : public ILayer
4903{
4904public:
4923 void setOutputDimensions(Dims dimensions) noexcept
4924 {
4925 return mImpl->setOutputDimensions(dimensions);
4926 }
4927
4933 Dims getOutputDimensions() const noexcept
4934 {
4935 return mImpl->getOutputDimensions();
4936 }
4937
4963 void setScales(float const* scales, int32_t nbScales) noexcept
4964 {
4965 mImpl->setScales(scales, nbScales);
4966 }
4967
4982 int32_t getScales(int32_t size, float* scales) const noexcept
4983 {
4984 return mImpl->getScales(size, scales);
4985 }
4986
4994 void setResizeMode(ResizeMode resizeMode) noexcept
4995 {
4996 mImpl->setResizeMode(resizeMode);
4997 }
4998
5004 ResizeMode getResizeMode() const noexcept
5005 {
5006 return mImpl->getResizeMode();
5007 }
5008
5020 TRT_DEPRECATED void setAlignCorners(bool alignCorners) noexcept
5021 {
5022 mImpl->setAlignCorners(alignCorners);
5023 }
5024
5032 TRT_DEPRECATED bool getAlignCorners() const noexcept
5033 {
5034 return mImpl->getAlignCorners();
5035 }
5036
5056 using ILayer::setInput;
5057
5068 {
5069 mImpl->setCoordinateTransformation(coordTransform);
5070 }
5071
5078 {
5079 return mImpl->getCoordinateTransformation();
5080 }
5081
5093 {
5094 mImpl->setSelectorForSinglePixel(selector);
5095 }
5096
5103 {
5104 return mImpl->getSelectorForSinglePixel();
5105 }
5106
5117 {
5118 mImpl->setNearestRounding(value);
5119 }
5120
5127 {
5128 return mImpl->getNearestRounding();
5129 }
5130
5148 void setCubicCoeff(float A) noexcept
5149 {
5150 mImpl->setCubicCoeff(A);
5151 }
5152
5158 float getCubicCoeff() const noexcept
5159 {
5160 return mImpl->getCubicCoeff();
5161 }
5162
5171 void setExcludeOutside(bool excludeFlag) noexcept
5172 {
5173 mImpl->setExcludeOutside(excludeFlag);
5174 }
5175
5181 bool getExcludeOutside() const noexcept
5182 {
5183 return mImpl->getExcludeOutside();
5184 }
5185
5186protected:
5187 virtual ~IResizeLayer() noexcept = default;
5188 apiv::VResizeLayer* mImpl;
5189};
5190
5192enum class LoopOutput : int32_t
5193{
5195 kLAST_VALUE = 0,
5196
5198 kCONCATENATE = 1,
5199
5201 kREVERSE = 2
5202};
5203
5209template <>
5210constexpr inline int32_t EnumMax<LoopOutput>() noexcept
5211{
5212 return 3;
5213}
5214
5216enum class TripLimit : int32_t
5217{
5218
5219 kCOUNT = 0,
5220 kWHILE = 1
5221};
5222
5228template <>
5229constexpr inline int32_t EnumMax<TripLimit>() noexcept
5230{
5231 return 2;
5232}
5233
5234class ILoop;
5235
5237{
5238public:
5240 ILoop* getLoop() const noexcept
5241 {
5242 return mBoundary->getLoop();
5243 }
5244
5245protected:
5246 virtual ~ILoopBoundaryLayer() noexcept = default;
5247 apiv::VLoopBoundaryLayer* mBoundary;
5248};
5249
5256{
5257public:
5260 {
5261 return mBoundary->getConditional();
5262 }
5263
5264protected:
5265 virtual ~IIfConditionalBoundaryLayer() noexcept = default;
5266 apiv::VConditionalBoundaryLayer* mBoundary;
5267};
5268
5273{
5274public:
5275protected:
5276 virtual ~IConditionLayer() noexcept = default;
5277 apiv::VConditionLayer* mImpl;
5278};
5279
5286{
5287public:
5288protected:
5289 virtual ~IIfConditionalOutputLayer() noexcept = default;
5290 apiv::VConditionalOutputLayer* mImpl;
5291};
5292
5297{
5298public:
5299protected:
5300 virtual ~IIfConditionalInputLayer() noexcept = default;
5301 apiv::VConditionalInputLayer* mImpl;
5302};
5303
5326{
5327public:
5338 {
5339 return mImpl->setCondition(condition);
5340 }
5341
5353 IIfConditionalOutputLayer* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput) noexcept
5354 {
5355 return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);
5356 }
5357
5366 {
5367 return mImpl->addInput(input);
5368 }
5369
5380 void setName(char const* name) noexcept
5381 {
5382 mImpl->setName(name);
5383 }
5384
5390 char const* getName() const noexcept
5391 {
5392 return mImpl->getName();
5393 }
5394
5395protected:
5396 virtual ~IIfConditional() noexcept = default;
5397 apiv::VIfConditional* mImpl;
5398};
5399
5400
5402{
5403public:
5409 //
5422 using ILayer::setInput;
5423
5424protected:
5425 virtual ~IRecurrenceLayer() noexcept = default;
5426 apiv::VRecurrenceLayer* mImpl;
5427};
5428
5447{
5448public:
5449 LoopOutput getLoopOutput() const noexcept
5450 {
5451 return mImpl->getLoopOutput();
5452 }
5453
5466 void setAxis(int32_t axis) noexcept
5467 {
5468 mImpl->setAxis(axis);
5469 }
5470
5472 int32_t getAxis() const noexcept
5473 {
5474 return mImpl->getAxis();
5475 }
5476
5482 //
5497 using ILayer::setInput;
5498
5499protected:
5500 virtual ~ILoopOutputLayer() noexcept = default;
5501 apiv::VLoopOutputLayer* mImpl;
5502};
5503
5505{
5506public:
5507 TripLimit getTripLimit() const noexcept
5508 {
5509 return mImpl->getTripLimit();
5510 }
5511
5512protected:
5513 virtual ~ITripLimitLayer() noexcept = default;
5514 apiv::VTripLimitLayer* mImpl;
5515};
5516
5518{
5519public:
5521 void setAxis(int32_t axis) noexcept
5522 {
5523 mImpl->setAxis(axis);
5524 }
5525
5527 int32_t getAxis() const noexcept
5528 {
5529 return mImpl->getAxis();
5530 }
5531
5537 void setReverse(bool reverse) noexcept
5538 {
5539 mImpl->setReverse(reverse);
5540 }
5541
5543 bool getReverse() const noexcept
5544 {
5545 return mImpl->getReverse();
5546 }
5547
5548protected:
5549 virtual ~IIteratorLayer() noexcept = default;
5550 apiv::VIteratorLayer* mImpl;
5551};
5552
5558class ILoop : public INoCopy
5559{
5560public:
5567 IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept
5568 {
5569 return mImpl->addRecurrence(initialValue);
5570 }
5571
5589 {
5590 return mImpl->addTripLimit(tensor, limit);
5591 }
5592
5601 IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept
5602 {
5603 return mImpl->addIterator(tensor, axis, reverse);
5604 }
5605
5613 ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept
5614 {
5615 return mImpl->addLoopOutput(tensor, outputKind, axis);
5616 }
5617
5628 void setName(char const* name) noexcept
5629 {
5630 mImpl->setName(name);
5631 }
5632
5638 char const* getName() const noexcept
5639 {
5640 return mImpl->getName();
5641 }
5642
5643protected:
5644 virtual ~ILoop() noexcept = default;
5645 apiv::VLoop* mImpl;
5646};
5647
5651class ISelectLayer : public ILayer
5652{
5653protected:
5654 virtual ~ISelectLayer() noexcept = default;
5655 apiv::VSelectLayer* mImpl;
5656};
5657
5673{
5674public:
5683 void setMessage(char const* message) noexcept
5684 {
5685 mImpl->setMessage(message);
5686 }
5687
5693 char const* getMessage() const noexcept
5694 {
5695 return mImpl->getMessage();
5696 }
5697
5698protected:
5699 virtual ~IAssertionLayer() noexcept = default;
5700
5701 apiv::VAssertionLayer* mImpl;
5702};
5703
5711enum class FillOperation : int32_t
5712{
5713 kLINSPACE = 0,
5714 kRANDOM_UNIFORM = 1,
5715 kRANDOM_NORMAL = 2
5716};
5717
5723template <>
5724constexpr inline int32_t EnumMax<FillOperation>() noexcept
5725{
5726 return 3;
5727}
5728
5754class IFillLayer : public ILayer
5755{
5756public:
5765 //
5766 void setDimensions(Dims dimensions) noexcept
5767 {
5768 mImpl->setDimensions(dimensions);
5769 }
5770
5781 Dims getDimensions() const noexcept
5782 {
5783 return mImpl->getDimensions();
5784 }
5785
5791 void setOperation(FillOperation op) noexcept
5792 {
5793 mImpl->setOperation(op);
5794 }
5795
5802 {
5803 return mImpl->getOperation();
5804 }
5805
5819 //
5820 void setAlpha(double alpha) noexcept
5821 {
5822 mImpl->setAlpha(alpha);
5823 }
5824
5835 double getAlpha() const noexcept
5836 {
5837 return mImpl->getAlpha();
5838 }
5839
5854 void setBeta(double beta) noexcept
5855 {
5856 mImpl->setBeta(beta);
5857 }
5858
5869 double getBeta() const noexcept
5870 {
5871 return mImpl->getBeta();
5872 }
5873
5906 using ILayer::setInput;
5907
5908protected:
5909 virtual ~IFillLayer() noexcept = default;
5910 apiv::VFillLayer* mImpl;
5911};
5912
5971{
5972public:
5981 int32_t getAxis() const noexcept
5982 {
5983 return mImpl->getAxis();
5984 }
5992 void setAxis(int32_t axis) noexcept
5993 {
5994 mImpl->setAxis(axis);
5995 }
5996
5997protected:
5998 virtual ~IQuantizeLayer() noexcept = default;
5999 apiv::VQuantizeLayer* mImpl;
6000};
6001
6057{
6058public:
6067 int32_t getAxis() const noexcept
6068 {
6069 return mImpl->getAxis();
6070 }
6078 void setAxis(int32_t axis) noexcept
6079 {
6080 mImpl->setAxis(axis);
6081 }
6082
6083protected:
6084 virtual ~IDequantizeLayer() noexcept = default;
6085 apiv::VDequantizeLayer* mImpl;
6086};
6087
6123class IEinsumLayer : public ILayer
6124{
6125public:
6135 bool setEquation(char const* equation) noexcept
6136 {
6137 return mImpl->setEquation(equation);
6138 }
6139
6145 char const* getEquation() const noexcept
6146 {
6147 return mImpl->getEquation();
6148 }
6149
6150protected:
6151 virtual ~IEinsumLayer() noexcept = default;
6152 apiv::VEinsumLayer* mImpl;
6153};
6154
6160enum class ScatterMode : int32_t
6161{
6162 kELEMENT = 0,
6163 kND = 1,
6164};
6165
6171template <>
6172constexpr inline int32_t EnumMax<ScatterMode>() noexcept
6173{
6174 return 2;
6175}
6176
6233class IScatterLayer : public ILayer
6234{
6235public:
6241 void setMode(ScatterMode mode) noexcept
6242 {
6243 mImpl->setMode(mode);
6244 }
6245
6251 ScatterMode getMode() const noexcept
6252 {
6253 return mImpl->getMode();
6254 }
6255
6261 void setAxis(int32_t axis) noexcept
6262 {
6263 mImpl->setAxis(axis);
6264 }
6265
6269 int32_t getAxis() const noexcept
6270 {
6271 return mImpl->getAxis();
6272 }
6273
6274protected:
6275 apiv::VScatterLayer* mImpl;
6276 virtual ~IScatterLayer() noexcept = default;
6277}; // class IScatterLayer
6278
6306class IOneHotLayer : public ILayer
6307{
6308public:
6314 void setAxis(int32_t axis) noexcept
6315 {
6316 mImpl->setAxis(axis);
6317 }
6318
6322 int32_t getAxis() const noexcept
6323 {
6324 return mImpl->getAxis();
6325 }
6326
6327protected:
6328 apiv::VOneHotLayer* mImpl;
6329};
6330
6342{
6343public:
6350 {
6351 mImpl->setInterpolationMode(mode);
6352 }
6353
6362 {
6363 return mImpl->getInterpolationMode();
6364 }
6365
6371 void setAlignCorners(bool alignCorners) noexcept
6372 {
6373 mImpl->setAlignCorners(alignCorners);
6374 }
6375
6383 bool getAlignCorners() const noexcept
6384 {
6385 return mImpl->getAlignCorners();
6386 }
6387
6395 bool setSampleMode(SampleMode mode) noexcept
6396 {
6397 return mImpl->setSampleMode(mode);
6398 }
6399
6407 SampleMode getSampleMode() const noexcept
6408 {
6409 return mImpl->getSampleMode();
6410 }
6411
6412protected:
6413 apiv::VGridSampleLayer* mImpl;
6414 virtual ~IGridSampleLayer() noexcept = default;
6415}; // class IGridSampleLayer
6416
6422enum class BoundingBoxFormat : int32_t
6423{
6425 kCORNER_PAIRS = 0,
6427 kCENTER_SIZES = 1
6428};
6429
6435template <>
6436constexpr inline int32_t EnumMax<BoundingBoxFormat>() noexcept
6437{
6438 return 2;
6439}
6440
6484class INMSLayer : public ILayer
6485{
6486public:
6497 {
6498 mImpl->setBoundingBoxFormat(fmt);
6499 }
6500
6509 {
6510 return mImpl->getBoundingBoxFormat();
6511 }
6512
6522 void setTopKBoxLimit(int32_t limit) noexcept
6523 {
6524 mImpl->setTopKBoxLimit(limit);
6525 }
6526
6532 int32_t getTopKBoxLimit() const noexcept
6533 {
6534 return mImpl->getTopKBoxLimit();
6535 }
6536
6555 using ILayer::setInput;
6556
6557protected:
6558 apiv::VNMSLayer* mImpl;
6559 virtual ~INMSLayer() noexcept = default;
6560}; // class INMSLayer
6561
6582{
6583public:
6584 virtual ~INetworkDefinition() noexcept = default;
6585
6623 ITensor* addInput(char const* name, DataType type, Dims dimensions) noexcept
6624 {
6625 return mImpl->addInput(name, type, dimensions);
6626 }
6627
6637 void markOutput(ITensor& tensor) noexcept
6638 {
6639 mImpl->markOutput(tensor);
6640 }
6641
6661 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6662 {
6663 return mImpl->addConvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6664 }
6665
6684 ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
6685 {
6686 return mImpl->addFullyConnected(input, nbOutputs, kernelWeights, biasWeights);
6687 }
6688
6704 {
6705 return mImpl->addActivation(input, type);
6706 }
6707
6723 {
6724 return mImpl->addPooling(input, type, windowSize);
6725 }
6726
6741 ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept
6742 {
6743 return mImpl->addLRN(input, window, alpha, beta, k);
6744 }
6745
6768 IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
6769 {
6770 return mImpl->addScale(input, mode, shift, scale, power);
6771 }
6772
6782 {
6783 return mImpl->addSoftMax(input);
6784 }
6785
6798 IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept
6799 {
6800 return mImpl->addConcatenation(inputs, nbInputs);
6801 }
6802
6822 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6823 {
6824 return mImpl->addDeconvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6825 }
6826
6850 {
6851 return mImpl->addElementWise(input1, input2, op);
6852 }
6853
6871 IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept
6872 {
6873 return mImpl->addUnary(input, operation);
6874 }
6875
6888 TRT_DEPRECATED IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept
6889 {
6890 return mImpl->addPadding(input, prePadding, postPadding);
6891 }
6892
6903 {
6904 return mImpl->addShuffle(input);
6905 }
6906
6919 IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept
6920 {
6921 return mImpl->addOneHot(indices, values, depth, axis);
6922 }
6923
6931 int32_t getNbLayers() const noexcept
6932 {
6933 return mImpl->getNbLayers();
6934 }
6935
6945 ILayer* getLayer(int32_t index) const noexcept
6946 {
6947 return mImpl->getLayer(index);
6948 }
6949
6957 int32_t getNbInputs() const noexcept
6958 {
6959 return mImpl->getNbInputs();
6960 }
6961
6973 ITensor* getInput(int32_t index) const noexcept
6974 {
6975 return mImpl->getInput(index);
6976 }
6977
6987 int32_t getNbOutputs() const noexcept
6988 {
6989 return mImpl->getNbOutputs();
6990 }
6991
7003 ITensor* getOutput(int32_t index) const noexcept
7004 {
7005 return mImpl->getOutput(index);
7006 }
7007
7015 TRT_DEPRECATED void destroy() noexcept
7016 {
7017 delete this;
7018 }
7019
7043 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
7044 {
7045 return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);
7046 }
7047
7076 ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
7077 {
7078 return mImpl->addTopK(input, op, k, reduceAxes);
7079 }
7080
7092 IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept
7093 {
7094 return mImpl->addGather(data, indices, axis);
7095 }
7096
7108 IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept
7109 {
7110 return mImpl->addGatherV2(data, indices, mode);
7111 }
7112
7127 {
7128 return mImpl->addRaggedSoftMax(input, bounds);
7129 }
7130
7148 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
7149 {
7150 return mImpl->addMatrixMultiply(input0, op0, input1, op1);
7151 }
7152
7163 {
7164 return mImpl->addNonZero(input);
7165 }
7166
7189 IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept
7190 {
7191 return mImpl->addConstant(dimensions, weights);
7192 }
7193
7259 ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
7260 {
7261 return mImpl->addRNNv2(input, layerCount, hiddenSize, maxSeqLen, op);
7262 }
7263
7274 {
7275 return mImpl->addIdentity(input);
7276 }
7277
7288 void removeTensor(ITensor& tensor) noexcept
7289 {
7290 mImpl->removeTensor(tensor);
7291 }
7292
7300 void unmarkOutput(ITensor& tensor) noexcept
7301 {
7302 mImpl->unmarkOutput(tensor);
7303 }
7304
7319 IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept
7320 {
7321 return mImpl->addPluginV2(inputs, nbInputs, plugin);
7322 }
7323
7338 ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept
7339 {
7340 return mImpl->addSlice(input, start, size, stride);
7341 }
7342
7362 void setName(char const* name) noexcept
7363 {
7364 mImpl->setName(name);
7365 }
7366
7376 char const* getName() const noexcept
7377 {
7378 return mImpl->getName();
7379 }
7380
7394 IShapeLayer* addShape(ITensor& input) noexcept
7395 {
7396 return mImpl->addShape(input);
7397 }
7398
7412 bool hasImplicitBatchDimension() const noexcept
7413 {
7414 return mImpl->hasImplicitBatchDimension();
7415 }
7416
7430 bool markOutputForShapes(ITensor& tensor) noexcept
7431 {
7432 return mImpl->markOutputForShapes(tensor);
7433 }
7434
7442 bool unmarkOutputForShapes(ITensor& tensor) noexcept
7443 {
7444 return mImpl->unmarkOutputForShapes(tensor);
7445 }
7446
7461 {
7462 return mImpl->addParametricReLU(input, slope);
7463 }
7464
7483 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7484 {
7485 return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7486 }
7487
7502 IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept
7503 {
7504 return mImpl->addPoolingNd(input, type, windowSize);
7505 }
7506
7521 //
7525 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7526 {
7527 return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7528 }
7529
7561 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
7562 {
7563 return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);
7564 }
7565
7577 {
7578 return mImpl->addResize(input);
7579 }
7580
7591 {
7592 return mImpl->hasExplicitPrecision();
7593 }
7594
7606 ILoop* addLoop() noexcept
7607 {
7608 return mImpl->addLoop();
7609 }
7610
7646 ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept
7647 {
7648 return mImpl->addSelect(condition, thenInput, elseInput);
7649 }
7650
7663 IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept
7664 {
7665 return mImpl->addAssertion(condition, message);
7666 }
7667
7686 IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept
7687 {
7688 return mImpl->addFill(dimensions, op);
7689 }
7690
7703 TRT_DEPRECATED IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept
7704 {
7705 return mImpl->addPaddingNd(input, prePadding, postPadding);
7706 }
7707
7726 bool setWeightsName(Weights weights, char const* name) noexcept
7727 {
7728 return mImpl->setWeightsName(weights, name);
7729 }
7730
7742 //
7745 void setErrorRecorder(IErrorRecorder* recorder) noexcept
7746 {
7747 mImpl->setErrorRecorder(recorder);
7748 }
7749
7761 {
7762 return mImpl->getErrorRecorder();
7763 }
7764
7780 {
7781 return mImpl->addDequantize(input, scale);
7782 }
7783
7799 IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept
7800 {
7801 return mImpl->addScatter(data, indices, updates, mode);
7802 }
7803
7818 IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept
7819 {
7820 return mImpl->addQuantize(input, scale);
7821 }
7822
7834 {
7835 return mImpl->addIfConditional();
7836 }
7837
7847 IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept
7848 {
7849 return mImpl->addEinsum(inputs, nbInputs, equation);
7850 }
7851
7864 {
7865 return mImpl->addGridSample(input, grid);
7866 }
7867
7881 INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept
7882 {
7883 return mImpl->addNMS(boxes, scores, maxOutputBoxesPerClass);
7884 }
7885
7886protected:
7887 apiv::VNetworkDefinition* mImpl;
7888};
7889
7895enum class CalibrationAlgoType : int32_t
7896{
7901};
7902
7908template <>
7909constexpr inline int32_t EnumMax<CalibrationAlgoType>() noexcept
7910{
7911 return 4;
7912}
7913
7926{
7927public:
7933 virtual int32_t getBatchSize() const noexcept = 0;
7934
7948 virtual bool getBatch(void* bindings[], char const* names[], int32_t nbBindings) noexcept = 0;
7949
7964 virtual void const* readCalibrationCache(std::size_t& length) noexcept = 0;
7965
7974 virtual void writeCalibrationCache(void const* ptr, std::size_t length) noexcept = 0;
7975
7981 virtual CalibrationAlgoType getAlgorithm() noexcept = 0;
7982
7983 virtual ~IInt8Calibrator() noexcept = default;
7984};
7985
7991{
7992public:
7997 {
7999 }
8000
8001 virtual ~IInt8EntropyCalibrator() noexcept = default;
8002};
8003
8009{
8010public:
8015 {
8017 }
8018
8019 virtual ~IInt8EntropyCalibrator2() noexcept = default;
8020};
8021
8026{
8027public:
8032 {
8034 }
8035
8036 virtual ~IInt8MinMaxCalibrator() noexcept = default;
8037};
8038
8044{
8045public:
8050 {
8052 }
8053
8060 virtual double getQuantile() const noexcept = 0;
8061
8068 virtual double getRegressionCutoff() const noexcept = 0;
8069
8082 virtual void const* readHistogramCache(std::size_t& length) noexcept = 0;
8083
8092 virtual void writeHistogramCache(void const* ptr, std::size_t length) noexcept = 0;
8093
8094 virtual ~IInt8LegacyCalibrator() noexcept = default;
8095};
8096
8108{
8109public:
8114 {
8115 return mImpl->getTensorFormat();
8116 }
8117
8121 DataType getDataType() const noexcept
8122 {
8123 return mImpl->getDataType();
8124 }
8125
8129 Dims getStrides() const noexcept
8130 {
8131 return mImpl->getStrides();
8132 }
8133
8134protected:
8135 virtual ~IAlgorithmIOInfo() noexcept = default;
8136 apiv::VAlgorithmIOInfo* mImpl;
8137};
8138
8151{
8152public:
8156 int64_t getImplementation() const noexcept
8157 {
8158 return mImpl->getImplementation();
8159 }
8160
8164 int64_t getTactic() const noexcept
8165 {
8166 return mImpl->getTactic();
8167 }
8168
8169protected:
8170 virtual ~IAlgorithmVariant() noexcept = default;
8171 apiv::VAlgorithmVariant* mImpl;
8172};
8173
8183{
8184public:
8189 char const* getName() const noexcept
8190 {
8191 return mImpl->getName();
8192 }
8193
8200 Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
8201 {
8202 return mImpl->getDimensions(index, select);
8203 }
8204
8208 int32_t getNbInputs() const noexcept
8209 {
8210 return mImpl->getNbInputs();
8211 }
8212
8216 int32_t getNbOutputs() const noexcept
8217 {
8218 return mImpl->getNbOutputs();
8219 }
8220
8221protected:
8222 virtual ~IAlgorithmContext() noexcept = default;
8223 apiv::VAlgorithmContext* mImpl;
8224};
8225
8235class IAlgorithm : public INoCopy
8236{
8237public:
8248 TRT_DEPRECATED IAlgorithmIOInfo const& getAlgorithmIOInfo(int32_t index) const noexcept
8249 {
8250 return mImpl->getAlgorithmIOInfo(index);
8251 }
8252
8257 {
8258 return mImpl->getAlgorithmVariant();
8259 }
8260
8264 float getTimingMSec() const noexcept
8265 {
8266 return mImpl->getTimingMSec();
8267 }
8268
8272 std::size_t getWorkspaceSize() const noexcept
8273 {
8274 return mImpl->getWorkspaceSize();
8275 }
8276
8285 IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept
8286 {
8287 return mImpl->getAlgorithmIOInfoByIndex(index);
8288 }
8289
8290protected:
8291 virtual ~IAlgorithm() noexcept = default;
8292 apiv::VAlgorithm* mImpl;
8293}; // IAlgorithm
8294
8304{
8305public:
8320 virtual int32_t selectAlgorithms(IAlgorithmContext const& context, IAlgorithm const* const* choices,
8321 int32_t nbChoices, int32_t* selection) noexcept = 0;
8332 virtual void reportAlgorithms(IAlgorithmContext const* const* algoContexts, IAlgorithm const* const* algoChoices,
8333 int32_t nbAlgorithms) noexcept = 0;
8334
8335 virtual ~IAlgorithmSelector() noexcept = default;
8336};
8337
8344using QuantizationFlags = uint32_t;
8345
8353enum class QuantizationFlag : int32_t
8354{
8359};
8360
8366template <>
8367constexpr inline int32_t EnumMax<QuantizationFlag>() noexcept
8368{
8369 return 1;
8370}
8371
8378using BuilderFlags = uint32_t;
8379
8387enum class BuilderFlag : int32_t
8388{
8389 kFP16 = 0,
8390 kINT8 = 1,
8391 kDEBUG = 2,
8392 kGPU_FALLBACK = 3,
8393
8406
8407 kREFIT = 5,
8409
8413 kTF32 = 7,
8414
8416 kSPARSE_WEIGHTS = 8,
8417
8424 kSAFETY_SCOPE = 9,
8425
8428
8432
8436 kDIRECT_IO = 12,
8437
8440
8446};
8447
8453template <>
8454constexpr inline int32_t EnumMax<BuilderFlag>() noexcept
8455{
8456 return 15;
8457}
8458
8469class ITimingCache : public INoCopy
8470{
8471public:
8472 virtual ~ITimingCache() noexcept = default;
8473
8483 nvinfer1::IHostMemory* serialize() const noexcept
8484 {
8485 return mImpl->serialize();
8486 }
8487
8507 bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept
8508 {
8509 return mImpl->combine(inputCache, ignoreMismatch);
8510 }
8511
8517 bool reset() noexcept
8518 {
8519 return mImpl->reset();
8520 }
8521
8522protected:
8523 apiv::VTimingCache* mImpl;
8524};
8525
8533enum class MemoryPoolType : int32_t
8534{
8542 kWORKSPACE = 0,
8543
8551
8557 kDLA_LOCAL_DRAM = 2,
8558
8564 kDLA_GLOBAL_DRAM = 3,
8565};
8566
8572template <>
8573constexpr inline int32_t EnumMax<MemoryPoolType>() noexcept
8574{
8575 return 4;
8576}
8577
8586enum class PreviewFeature : int32_t
8587{
8595
8611};
8612namespace impl
8613{
8619template <>
8621{
8622 static constexpr int32_t kVALUE = 2;
8623};
8624} // namespace impl
8625
8634{
8635public:
8636 virtual ~IBuilderConfig() noexcept = default;
8637
8650 TRT_DEPRECATED virtual void setMinTimingIterations(int32_t minTiming) noexcept
8651 {
8652 mImpl->setMinTimingIterations(minTiming);
8653 }
8654
8664 TRT_DEPRECATED virtual int32_t getMinTimingIterations() const noexcept
8665 {
8666 return mImpl->getMinTimingIterations();
8667 }
8668
8677 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
8678 {
8679 mImpl->setAvgTimingIterations(avgTiming);
8680 }
8681
8689 int32_t getAvgTimingIterations() const noexcept
8690 {
8691 return mImpl->getAvgTimingIterations();
8692 }
8693
8702 void setEngineCapability(EngineCapability capability) noexcept
8703 {
8704 mImpl->setEngineCapability(capability);
8705 }
8706
8715 {
8716 return mImpl->getEngineCapability();
8717 }
8718
8724 void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept
8725 {
8726 mImpl->setInt8Calibrator(calibrator);
8727 }
8728
8733 {
8734 return mImpl->getInt8Calibrator();
8735 }
8736
8747 TRT_DEPRECATED void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept
8748 {
8749 mImpl->setMaxWorkspaceSize(workspaceSize);
8750 }
8751
8764 TRT_DEPRECATED std::size_t getMaxWorkspaceSize() const noexcept
8765 {
8766 return mImpl->getMaxWorkspaceSize();
8767 }
8768
8781 void setFlags(BuilderFlags builderFlags) noexcept
8782 {
8783 mImpl->setFlags(builderFlags);
8784 }
8785
8793 BuilderFlags getFlags() const noexcept
8794 {
8795 return mImpl->getFlags();
8796 }
8797
8805 void clearFlag(BuilderFlag builderFlag) noexcept
8806 {
8807 mImpl->clearFlag(builderFlag);
8808 }
8809
8817 void setFlag(BuilderFlag builderFlag) noexcept
8818 {
8819 mImpl->setFlag(builderFlag);
8820 }
8821
8829 bool getFlag(BuilderFlag builderFlag) const noexcept
8830 {
8831 return mImpl->getFlag(builderFlag);
8832 }
8833
8844 void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept
8845 {
8846 mImpl->setDeviceType(layer, deviceType);
8847 }
8848
8853 DeviceType getDeviceType(ILayer const* layer) const noexcept
8854 {
8855 return mImpl->getDeviceType(layer);
8856 }
8857
8863 bool isDeviceTypeSet(ILayer const* layer) const noexcept
8864 {
8865 return mImpl->isDeviceTypeSet(layer);
8866 }
8867
8873 void resetDeviceType(ILayer const* layer) noexcept
8874 {
8875 mImpl->resetDeviceType(layer);
8876 }
8877
8882 bool canRunOnDLA(ILayer const* layer) const noexcept
8883 {
8884 return mImpl->canRunOnDLA(layer);
8885 }
8886
8897 void setDLACore(int32_t dlaCore) noexcept
8898 {
8899 mImpl->setDLACore(dlaCore);
8900 }
8901
8906 int32_t getDLACore() const noexcept
8907 {
8908 return mImpl->getDLACore();
8909 }
8910
8916 void setDefaultDeviceType(DeviceType deviceType) noexcept
8917 {
8918 mImpl->setDefaultDeviceType(deviceType);
8919 }
8920
8927 {
8928 return mImpl->getDefaultDeviceType();
8929 }
8930
8936 void reset() noexcept
8937 {
8938 mImpl->reset();
8939 }
8940
8950 TRT_DEPRECATED void destroy() noexcept
8951 {
8952 delete this;
8953 }
8954
8962 void setProfileStream(const cudaStream_t stream) noexcept
8963 {
8964 return mImpl->setProfileStream(stream);
8965 }
8966
8974 cudaStream_t getProfileStream() const noexcept
8975 {
8976 return mImpl->getProfileStream();
8977 }
8978
8990 int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept
8991 {
8992 return mImpl->addOptimizationProfile(profile);
8993 }
8994
9003 int32_t getNbOptimizationProfiles() const noexcept
9004 {
9005 return mImpl->getNbOptimizationProfiles();
9006 }
9007
9016 {
9017 mImpl->setProfilingVerbosity(verbosity);
9018 }
9019
9029 {
9030 return mImpl->getProfilingVerbosity();
9031 }
9032
9038 {
9039 mImpl->setAlgorithmSelector(selector);
9040 }
9041
9046 {
9047 return mImpl->getAlgorithmSelector();
9048 }
9049
9060 bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept
9061 {
9062 return mImpl->setCalibrationProfile(profile);
9063 }
9064
9071 {
9072 return mImpl->getCalibrationProfile();
9073 }
9074
9088 {
9089 mImpl->setQuantizationFlags(flags);
9090 }
9091
9100 {
9101 return mImpl->getQuantizationFlags();
9102 }
9103
9112 {
9113 mImpl->clearQuantizationFlag(flag);
9114 }
9115
9124 {
9125 mImpl->setQuantizationFlag(flag);
9126 }
9127
9135 bool getQuantizationFlag(QuantizationFlag flag) const noexcept
9136 {
9137 return mImpl->getQuantizationFlag(flag);
9138 }
9139
9157 bool setTacticSources(TacticSources tacticSources) noexcept
9158 {
9159 return mImpl->setTacticSources(tacticSources);
9160 }
9161
9173 {
9174 return mImpl->getTacticSources();
9175 }
9176
9191 nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept
9192 {
9193 return mImpl->createTimingCache(blob, size);
9194 }
9195
9214 bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept
9215 {
9216 return mImpl->setTimingCache(cache, ignoreMismatch);
9217 }
9218
9225 {
9226 return mImpl->getTimingCache();
9227 }
9228
9256 void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
9257 {
9258 mImpl->setMemoryPoolLimit(pool, poolSize);
9259 }
9260
9275 std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
9276 {
9277 return mImpl->getMemoryPoolLimit(pool);
9278 }
9279
9293 void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
9294 {
9295 mImpl->setPreviewFeature(feature, enable);
9296 }
9297
9307 bool getPreviewFeature(PreviewFeature feature) const noexcept
9308 {
9309 return mImpl->getPreviewFeature(feature);
9310 }
9311
9312protected:
9313 apiv::VBuilderConfig* mImpl;
9314};
9315
9323
9333{
9339 kEXPLICIT_BATCH = 0,
9340
9344};
9345
9351template <>
9352constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>() noexcept
9353{
9354 return 2;
9355}
9356
9364class IBuilder : public INoCopy
9365{
9366public:
9367 virtual ~IBuilder() noexcept = default;
9368
9379 TRT_DEPRECATED void setMaxBatchSize(int32_t batchSize) noexcept
9380 {
9381 mImpl->setMaxBatchSize(batchSize);
9382 }
9383
9394 TRT_DEPRECATED int32_t getMaxBatchSize() const noexcept
9395 {
9396 return mImpl->getMaxBatchSize();
9397 }
9398
9402 bool platformHasFastFp16() const noexcept
9403 {
9404 return mImpl->platformHasFastFp16();
9405 }
9406
9410 bool platformHasFastInt8() const noexcept
9411 {
9412 return mImpl->platformHasFastInt8();
9413 }
9414
9422 TRT_DEPRECATED void destroy() noexcept
9423 {
9424 delete this;
9425 }
9426
9434 int32_t getMaxDLABatchSize() const noexcept
9435 {
9436 return mImpl->getMaxDLABatchSize();
9437 }
9438
9442 int32_t getNbDLACores() const noexcept
9443 {
9444 return mImpl->getNbDLACores();
9445 }
9446
9458 void setGpuAllocator(IGpuAllocator* allocator) noexcept
9459 {
9460 mImpl->setGpuAllocator(allocator);
9461 }
9462
9469 {
9470 return mImpl->createBuilderConfig();
9471 }
9472
9484 INetworkDefinition& network, IBuilderConfig& config) noexcept
9485 {
9486 return mImpl->buildEngineWithConfig(network, config);
9487 }
9488
9502 {
9503 return mImpl->createNetworkV2(flags);
9504 }
9505
9516 {
9517 return mImpl->createOptimizationProfile();
9518 }
9519
9531 //
9534 void setErrorRecorder(IErrorRecorder* recorder) noexcept
9535 {
9536 mImpl->setErrorRecorder(recorder);
9537 }
9538
9550 {
9551 return mImpl->getErrorRecorder();
9552 }
9553
9557 void reset() noexcept
9558 {
9559 mImpl->reset();
9560 }
9561
9565 bool platformHasTf32() const noexcept
9566 {
9567 return mImpl->platformHasTf32();
9568 }
9569
9585 {
9586 return mImpl->buildSerializedNetwork(network, config);
9587 }
9588
9608 bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
9609 {
9610 return mImpl->isNetworkSupported(network, config);
9611 }
9612
9618 ILogger* getLogger() const noexcept
9619 {
9620 return mImpl->getLogger();
9621 }
9622
9632 bool setMaxThreads(int32_t maxThreads) noexcept
9633 {
9634 return mImpl->setMaxThreads(maxThreads);
9635 }
9636
9646 int32_t getMaxThreads() const noexcept
9647 {
9648 return mImpl->getMaxThreads();
9649 }
9650
9651protected:
9652 apiv::VBuilder* mImpl;
9653};
9654
9655} // namespace nvinfer1
9656
9661extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept;
9662
9663namespace nvinfer1
9664{
9665namespace
9666{
9667
9675inline IBuilder* createInferBuilder(ILogger& logger) noexcept
9676{
9677 return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
9678}
9679
9680} // namespace
9681
9693 nvinfer1::EngineCapability capability) noexcept;
9694
9695} // namespace nvinfer1
9696
9697#endif // NV_INFER_H
#define TENSORRTAPI
Definition: NvInferRuntimeCommon.h:54
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeCommon.h:73
#define TRT_DEPRECATED
Definition: NvInferRuntimeCommon.h:40
#define TRT_DEPRECATED_ENUM
Definition: NvInferRuntimeCommon.h:41
Definition: NvInferRuntimeCommon.h:171
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeCommon.h:174
Descriptor for two-dimensional spatial data.
Definition: NvInferLegacyDims.h:70
An Activation layer in a network definition.
Definition: NvInfer.h:1593
void setBeta(float beta) noexcept
Set the beta parameter (must be finite).
Definition: NvInfer.h:1641
void setActivationType(ActivationType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1602
ActivationType getActivationType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1612
float getAlpha() const noexcept
Get the alpha parameter.
Definition: NvInfer.h:1650
virtual ~IActivationLayer() noexcept=default
float getBeta() const noexcept
Get the beta parameter.
Definition: NvInfer.h:1659
void setAlpha(float alpha) noexcept
Set the alpha parameter (must be finite).
Definition: NvInfer.h:1627
Describes the context and requirements, that could be fulfilled by one or more instances of IAlgorith...
Definition: NvInfer.h:8183
int32_t getNbOutputs() const noexcept
Return number of outputs of the algorithm.
Definition: NvInfer.h:8216
int32_t getNbInputs() const noexcept
Return number of inputs of the algorithm.
Definition: NvInfer.h:8208
char const * getName() const noexcept
Return name of the algorithm node. This is a unique identifier for the IAlgorithmContext.
Definition: NvInfer.h:8189
virtual ~IAlgorithmContext() noexcept=default
Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for input or output tensor.
Definition: NvInfer.h:8200
Describes a variation of execution of a layer. An algorithm is represented by IAlgorithmVariant and t...
Definition: NvInfer.h:8236
std::size_t getWorkspaceSize() const noexcept
The size of the GPU temporary memory in bytes which the algorithm uses at execution time.
Definition: NvInfer.h:8272
float getTimingMSec() const noexcept
The time in milliseconds to execute the algorithm.
Definition: NvInfer.h:8264
IAlgorithmIOInfo const * getAlgorithmIOInfoByIndex(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:8285
virtual ~IAlgorithm() noexcept=default
TRT_DEPRECATED IAlgorithmIOInfo const & getAlgorithmIOInfo(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:8248
IAlgorithmVariant const & getAlgorithmVariant() const noexcept
Returns the algorithm variant.
Definition: NvInfer.h:8256
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:8108
virtual ~IAlgorithmIOInfo() noexcept=default
Dims getStrides() const noexcept
Return strides of the input/output tensor of algorithm.
Definition: NvInfer.h:8129
DataType getDataType() const noexcept
Return DataType of the input/output of algorithm.
Definition: NvInfer.h:8121
TensorFormat getTensorFormat() const noexcept
Return TensorFormat of the input/output of algorithm.
Definition: NvInfer.h:8113
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
Definition: NvInfer.h:8304
virtual void reportAlgorithms(IAlgorithmContext const *const *algoContexts, IAlgorithm const *const *algoChoices, int32_t nbAlgorithms) noexcept=0
Called by TensorRT to report choices it made.
virtual int32_t selectAlgorithms(IAlgorithmContext const &context, IAlgorithm const *const *choices, int32_t nbChoices, int32_t *selection) noexcept=0
Select Algorithms for a layer from the given list of algorithm choices.
virtual ~IAlgorithmSelector() noexcept=default
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:8151
virtual ~IAlgorithmVariant() noexcept=default
int64_t getTactic() const noexcept
Return tactic of the algorithm.
Definition: NvInfer.h:8164
int64_t getImplementation() const noexcept
Return implementation of the algorithm.
Definition: NvInfer.h:8156
An assertion layer in a network.
Definition: NvInfer.h:5673
void setMessage(char const *message) noexcept
Set the message to print if the assertion fails.
Definition: NvInfer.h:5683
char const * getMessage() const noexcept
Return the assertion message.
Definition: NvInfer.h:5693
virtual ~IAssertionLayer() noexcept=default
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:8634
virtual TRT_DEPRECATED int32_t getMinTimingIterations() const noexcept
Query the number of minimization iterations.
Definition: NvInfer.h:8664
IOptimizationProfile const * getCalibrationProfile() noexcept
Get the current calibration profile.
Definition: NvInfer.h:9070
void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
Set the memory size for the memory pool.
Definition: NvInfer.h:9256
void setQuantizationFlag(QuantizationFlag flag) noexcept
Set a single quantization flag.
Definition: NvInfer.h:9123
nvinfer1::ITimingCache * createTimingCache(void const *blob, std::size_t size) const noexcept
Create timing cache.
Definition: NvInfer.h:9191
void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
Enable or disable a specific preview feature.
Definition: NvInfer.h:9293
void setInt8Calibrator(IInt8Calibrator *calibrator) noexcept
Set Int8 Calibration interface.
Definition: NvInfer.h:8724
bool getPreviewFeature(PreviewFeature feature) const noexcept
Get status of preview feature.
Definition: NvInfer.h:9307
void clearQuantizationFlag(QuantizationFlag flag) noexcept
clear a quantization flag.
Definition: NvInfer.h:9111
bool setTacticSources(TacticSources tacticSources) noexcept
Set tactic sources.
Definition: NvInfer.h:9157
bool getQuantizationFlag(QuantizationFlag flag) const noexcept
Returns true if the quantization flag is set.
Definition: NvInfer.h:9135
std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
Get the memory size limit of the memory pool.
Definition: NvInfer.h:9275
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInfer.h:8906
void setDeviceType(ILayer const *layer, DeviceType deviceType) noexcept
Set the device that this layer must execute on.
Definition: NvInfer.h:8844
void setEngineCapability(EngineCapability capability) noexcept
Configure the builder to target specified EngineCapability flow.
Definition: NvInfer.h:8702
bool getFlag(BuilderFlag builderFlag) const noexcept
Returns true if the build mode flag is set.
Definition: NvInfer.h:8829
void setQuantizationFlags(QuantizationFlags flags) noexcept
Set the quantization flags.
Definition: NvInfer.h:9087
bool setCalibrationProfile(IOptimizationProfile const *profile) noexcept
Add a calibration profile.
Definition: NvInfer.h:9060
virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
Set the number of averaging iterations used when timing layers.
Definition: NvInfer.h:8677
void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:9015
IAlgorithmSelector * getAlgorithmSelector() const noexcept
Get Algorithm Selector.
Definition: NvInfer.h:9045
int32_t getNbOptimizationProfiles() const noexcept
Get number of optimization profiles.
Definition: NvInfer.h:9003
QuantizationFlags getQuantizationFlags() const noexcept
Get the quantization flags.
Definition: NvInfer.h:9099
nvinfer1::ITimingCache const * getTimingCache() const noexcept
Get the pointer to the timing cache from current IBuilderConfig.
Definition: NvInfer.h:9224
void reset() noexcept
Resets the builder configuration to defaults.
Definition: NvInfer.h:8936
bool setTimingCache(ITimingCache const &cache, bool ignoreMismatch) noexcept
Attach a timing cache to IBuilderConfig.
Definition: NvInfer.h:9214
TRT_DEPRECATED void destroy() noexcept
Delete this IBuilderConfig.
Definition: NvInfer.h:8950
EngineCapability getEngineCapability() const noexcept
Query EngineCapability flow configured for the builder.
Definition: NvInfer.h:8714
void setAlgorithmSelector(IAlgorithmSelector *selector) noexcept
Set Algorithm Selector.
Definition: NvInfer.h:9037
TRT_DEPRECATED void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept
Set the maximum workspace size.
Definition: NvInfer.h:8747
TRT_DEPRECATED std::size_t getMaxWorkspaceSize() const noexcept
Get the maximum workspace size.
Definition: NvInfer.h:8764
DeviceType getDefaultDeviceType() const noexcept
Get the default DeviceType which was set by setDefaultDeviceType.
Definition: NvInfer.h:8926
BuilderFlags getFlags() const noexcept
Get the build mode flags for this builder config. Defaults to 0.
Definition: NvInfer.h:8793
void setFlags(BuilderFlags builderFlags) noexcept
Set the build mode flags to turn on builder options for this network.
Definition: NvInfer.h:8781
TacticSources getTacticSources() const noexcept
Get tactic sources.
Definition: NvInfer.h:9172
void resetDeviceType(ILayer const *layer) noexcept
reset the DeviceType for this layer
Definition: NvInfer.h:8873
void setDLACore(int32_t dlaCore) noexcept
Sets the DLA core used by the network. Defaults to -1.
Definition: NvInfer.h:8897
void clearFlag(BuilderFlag builderFlag) noexcept
clear a single build mode flag.
Definition: NvInfer.h:8805
int32_t addOptimizationProfile(IOptimizationProfile const *profile) noexcept
Add an optimization profile.
Definition: NvInfer.h:8990
apiv::VBuilderConfig * mImpl
Definition: NvInfer.h:9313
int32_t getAvgTimingIterations() const noexcept
Query the number of averaging iterations.
Definition: NvInfer.h:8689
void setDefaultDeviceType(DeviceType deviceType) noexcept
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
Definition: NvInfer.h:8916
void setFlag(BuilderFlag builderFlag) noexcept
Set a single build mode flag.
Definition: NvInfer.h:8817
virtual ~IBuilderConfig() noexcept=default
DeviceType getDeviceType(ILayer const *layer) const noexcept
Get the device that this layer executes on.
Definition: NvInfer.h:8853
bool canRunOnDLA(ILayer const *layer) const noexcept
Checks if a layer can run on DLA.
Definition: NvInfer.h:8882
cudaStream_t getProfileStream() const noexcept
Get the cuda stream that is used to profile this network.
Definition: NvInfer.h:8974
IInt8Calibrator * getInt8Calibrator() const noexcept
Get Int8 Calibration interface.
Definition: NvInfer.h:8732
ProfilingVerbosity getProfilingVerbosity() const noexcept
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:9028
bool isDeviceTypeSet(ILayer const *layer) const noexcept
whether the DeviceType has been explicitly set for this layer
Definition: NvInfer.h:8863
void setProfileStream(const cudaStream_t stream) noexcept
Set the cuda stream that is used to profile this network.
Definition: NvInfer.h:8962
Builds an engine from a network definition.
Definition: NvInfer.h:9365
int32_t getMaxDLABatchSize() const noexcept
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Definition: NvInfer.h:9434
int32_t getNbDLACores() const noexcept
Return the number of DLA engines available to this builder.
Definition: NvInfer.h:9442
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:9549
apiv::VBuilder * mImpl
Definition: NvInfer.h:9652
ILogger * getLogger() const noexcept
get the logger with which the builder was created
Definition: NvInfer.h:9618
bool isNetworkSupported(INetworkDefinition const &network, IBuilderConfig const &config) const noexcept
Checks that a network is within the scope of the IBuilderConfig settings.
Definition: NvInfer.h:9608
bool platformHasTf32() const noexcept
Determine whether the platform has TF32 support.
Definition: NvInfer.h:9565
int32_t getMaxThreads() const noexcept
get the maximum number of threads that can be used by the builder.
Definition: NvInfer.h:9646
TRT_DEPRECATED void destroy() noexcept
Destroy this object.
Definition: NvInfer.h:9422
nvinfer1::IOptimizationProfile * createOptimizationProfile() noexcept
Create a new optimization profile.
Definition: NvInfer.h:9515
bool platformHasFastFp16() const noexcept
Determine whether the platform has fast native fp16.
Definition: NvInfer.h:9402
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInfer.h:9458
nvinfer1::INetworkDefinition * createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept
Create a network definition object.
Definition: NvInfer.h:9501
nvinfer1::IBuilderConfig * createBuilderConfig() noexcept
Create a builder configuration object.
Definition: NvInfer.h:9468
void reset() noexcept
Resets the builder state to default values.
Definition: NvInfer.h:9557
bool setMaxThreads(int32_t maxThreads) noexcept
Set the maximum number of threads.
Definition: NvInfer.h:9632
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:9534
bool platformHasFastInt8() const noexcept
Determine whether the platform has fast native int8.
Definition: NvInfer.h:9410
nvinfer1::IHostMemory * buildSerializedNetwork(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds and serializes a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:9584
virtual ~IBuilder() noexcept=default
TRT_DEPRECATED int32_t getMaxBatchSize() const noexcept
Get the maximum batch size.
Definition: NvInfer.h:9394
TRT_DEPRECATED nvinfer1::ICudaEngine * buildEngineWithConfig(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds an engine for the given INetworkDefinition and given IBuilderConfig.
Definition: NvInfer.h:9483
A concatenation layer in a network definition.
Definition: NvInfer.h:2399
void setAxis(int32_t axis) noexcept
Set the axis along which concatenation occurs.
Definition: NvInfer.h:2413
int32_t getAxis() const noexcept
Get the axis along which concatenation occurs.
Definition: NvInfer.h:2423
virtual ~IConcatenationLayer() noexcept=default
Definition: NvInfer.h:5273
virtual ~IConditionLayer() noexcept=default
Layer that represents a constant value.
Definition: NvInfer.h:4646
void setWeights(Weights weights) noexcept
Set the weights for the layer.
Definition: NvInfer.h:4657
Weights getWeights() const noexcept
Get the weights for the layer.
Definition: NvInfer.h:4667
apiv::VConstantLayer * mImpl
Definition: NvInfer.h:4697
void setDimensions(Dims dimensions) noexcept
Set the dimensions for the layer.
Definition: NvInfer.h:4679
virtual ~IConstantLayer() noexcept=default
Dims getDimensions() const noexcept
Get the dimensions for the layer.
Definition: NvInfer.h:4691
A convolution layer in a network definition.
Definition: NvInfer.h:1036
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding of the convolution. If the padding is asymmetric, the pre-padding is returned.
Definition: NvInfer.h:1139
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride of the convolution.
Definition: NvInfer.h:1107
TRT_DEPRECATED DimsHW getDilation() const noexcept
Get the dilation for a convolution.
Definition: NvInfer.h:1246
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride of the convolution.
Definition: NvInfer.h:1364
void setKernelSizeNd(Dims kernelSize) noexcept
Set the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1339
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1273
Weights getBiasWeights() const noexcept
Get the bias weights for the convolution.
Definition: NvInfer.h:1218
int32_t getNbGroups() const noexcept
Get the number of groups of the convolution.
Definition: NvInfer.h:1169
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1314
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Get the stride of the convolution.
Definition: NvInfer.h:1097
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the convolution.
Definition: NvInfer.h:1404
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the convolution.
Definition: NvInfer.h:1374
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding of the convolution.
Definition: NvInfer.h:1127
Weights getKernelWeights() const noexcept
Get the kernel weights of the convolution.
Definition: NvInfer.h:1193
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding of the convolution.
Definition: NvInfer.h:1392
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1428
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the convolution.
Definition: NvInfer.h:1183
void setDilationNd(Dims dilation) noexcept
Set the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1418
Dims getPostPadding() const noexcept
Get the post-padding.
Definition: NvInfer.h:1300
TRT_DEPRECATED void setDilation(DimsHW dilation) noexcept
Set the dilation for a convolution.
Definition: NvInfer.h:1234
void setNbGroups(int32_t nbGroups) noexcept
Set the number of groups for a convolution.
Definition: NvInfer.h:1159
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1326
int32_t getNbOutputMaps() const noexcept
Get the number of output maps for the convolution.
Definition: NvInfer.h:1081
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding of the convolution.
Definition: NvInfer.h:1263
virtual ~IConvolutionLayer() noexcept=default
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the convolution.
Definition: NvInfer.h:1208
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1349
void setNbOutputMaps(int32_t nbOutputMaps) noexcept
Set the number of output maps for the convolution.
Definition: NvInfer.h:1071
TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
Set the HW kernel size of the convolution.
Definition: NvInfer.h:1047
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding of the convolution.
Definition: NvInfer.h:1290
TRT_DEPRECATED DimsHW getKernelSize() const noexcept
Get the HW kernel size of the convolution.
Definition: NvInfer.h:1059
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1383
A deconvolution layer in a network definition.
Definition: NvInfer.h:2441
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the deconvolution.
Definition: NvInfer.h:2619
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Set the stride of the deconvolution.
Definition: NvInfer.h:2504
void setNbOutputMaps(int32_t nbOutputMaps) noexcept
Set the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2478
Weights getKernelWeights() const noexcept
Get the kernel weights for the deconvolution.
Definition: NvInfer.h:2604
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding of the deconvolution.
Definition: NvInfer.h:2647
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding of the deconvolution.
Definition: NvInfer.h:2550
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2763
int32_t getNbGroups() const noexcept
Get the number of groups for a deconvolution.
Definition: NvInfer.h:2580
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2827
Weights getBiasWeights() const noexcept
Get the bias weights for the deconvolution.
Definition: NvInfer.h:2629
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the deconvolution.
Definition: NvInfer.h:2594
void setDilationNd(Dims dilation) noexcept
Set the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2817
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride of the deconvolution.
Definition: NvInfer.h:2516
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding of the deconvolution.
Definition: NvInfer.h:2536
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:2685
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2736
void setKernelSizeNd(Dims kernelSize) noexcept
Set the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2726
virtual ~IDeconvolutionLayer() noexcept=default
TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
Set the HW kernel size of the convolution.
Definition: NvInfer.h:2454
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2753
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2793
void setNbGroups(int32_t nbGroups) noexcept
Set the number of groups for a deconvolution.
Definition: NvInfer.h:2570
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:2699
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2781
TRT_DEPRECATED DimsHW getKernelSize() const noexcept
Get the HW kernel size of the deconvolution.
Definition: NvInfer.h:2466
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding of the deconvolution.
Definition: NvInfer.h:2675
int32_t getNbOutputMaps() const noexcept
Get the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2488
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:2657
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:2711
A Dequantize layer in a network definition.
Definition: NvInfer.h:6057
virtual ~IDequantizeLayer() noexcept=default
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:6067
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:6078
An Einsum layer in a network.
Definition: NvInfer.h:6124
bool setEquation(char const *equation) noexcept
Set the equation. The equation is a comma-separated list of subscript labels, where each label refers...
Definition: NvInfer.h:6135
virtual ~IEinsumLayer() noexcept=default
char const * getEquation() const noexcept
Return the equation.
Definition: NvInfer.h:6145
A elementwise layer in a network definition.
Definition: NvInfer.h:2902
virtual ~IElementWiseLayer() noexcept=default
apiv::VElementWiseLayer * mImpl
Definition: NvInfer.h:2931
ElementWiseOperation getOperation() const noexcept
Get the binary operation for the layer.
Definition: NvInfer.h:2925
void setOperation(ElementWiseOperation op) noexcept
Set the binary operation for the layer.
Definition: NvInfer.h:2913
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1689
Generate an output tensor with specified mode.
Definition: NvInfer.h:5755
FillOperation getOperation() const noexcept
Get the fill operation for the layer.
Definition: NvInfer.h:5801
void setOperation(FillOperation op) noexcept
Set the fill operation for the layer.
Definition: NvInfer.h:5791
void setDimensions(Dims dimensions) noexcept
Set the output tensor's dimensions.
Definition: NvInfer.h:5766
void setBeta(double beta) noexcept
Set the beta parameter.
Definition: NvInfer.h:5854
double getAlpha() const noexcept
Get the value of alpha parameter.
Definition: NvInfer.h:5835
void setAlpha(double alpha) noexcept
Set the alpha parameter.
Definition: NvInfer.h:5820
Dims getDimensions() const noexcept
Get the output tensor's dimensions.
Definition: NvInfer.h:5781
double getBeta() const noexcept
Get the value of beta parameter.
Definition: NvInfer.h:5869
virtual ~IFillLayer() noexcept=default
A fully connected layer in a network definition. This layer expects an input tensor of three or more ...
Definition: NvInfer.h:1485
virtual ~IFullyConnectedLayer() noexcept=default
void setNbOutputChannels(int32_t nbOutputs) noexcept
Set the number of output channels K from the fully connected layer.
Definition: NvInfer.h:1494
Weights getKernelWeights() const noexcept
Get the kernel weights.
Definition: NvInfer.h:1524
void setBiasWeights(Weights weights) noexcept
Set the bias weights.
Definition: NvInfer.h:1536
void setKernelWeights(Weights weights) noexcept
Set the kernel weights, given as a KxC matrix in row-major order.
Definition: NvInfer.h:1514
int32_t getNbOutputChannels() const noexcept
Get the number of output channels K from the fully connected layer.
Definition: NvInfer.h:1504
Weights getBiasWeights() const noexcept
Get the bias weights.
Definition: NvInfer.h:1546
A Gather layer in a network definition. Supports several kinds of gathering.
Definition: NvInfer.h:3037
void setGatherAxis(int32_t axis) noexcept
Set the axis used by GatherMode::kELEMENTS and GatherMode::kDEFAULT The axis must be less than the nu...
Definition: NvInfer.h:3048
void setNbElementWiseDims(int32_t elementWiseDims) noexcept
Set the number of leading dimensions of indices tensor to be handled elementwise. The gathering of in...
Definition: NvInfer.h:3080
apiv::VGatherLayer * mImpl
Definition: NvInfer.h:3116
int32_t getNbElementWiseDims() const noexcept
Get the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:3090
void setMode(GatherMode mode) noexcept
Set the gather mode.
Definition: NvInfer.h:3100
int32_t getGatherAxis() const noexcept
Get the axis to gather on.
Definition: NvInfer.h:3059
GatherMode getMode() const noexcept
Get the gather mode.
Definition: NvInfer.h:3110
virtual ~IGatherLayer() noexcept=default
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1362
A GridSample layer in a network definition.
Definition: NvInfer.h:6342
void setInterpolationMode(InterpolationMode mode) noexcept
Set the grid sample interpolation mode.
Definition: NvInfer.h:6349
bool setSampleMode(SampleMode mode) noexcept
Set the sample mode.
Definition: NvInfer.h:6395
void setAlignCorners(bool alignCorners) noexcept
Set the align corners mode.
Definition: NvInfer.h:6371
apiv::VGridSampleLayer * mImpl
Definition: NvInfer.h:6413
SampleMode getSampleMode() const noexcept
Get the sample mode.
Definition: NvInfer.h:6407
InterpolationMode getInterpolationMode() const noexcept
Get the grid sample interpolation mode.
Definition: NvInfer.h:6361
bool getAlignCorners() const noexcept
Get the align corners mode.
Definition: NvInfer.h:6383
virtual ~IGridSampleLayer() noexcept=default
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:144
A layer that represents the identity function.
Definition: NvInfer.h:4631
apiv::VIdentityLayer * mImpl
Definition: NvInfer.h:4633
virtual ~IIdentityLayer() noexcept=default
Definition: NvInfer.h:5256
IIfConditional * getConditional() const noexcept
Return pointer to the IIfConditional associated with this boundary layer.
Definition: NvInfer.h:5259
virtual ~IIfConditionalBoundaryLayer() noexcept=default
Definition: NvInfer.h:5326
IIfConditionalInputLayer * addInput(ITensor &input) noexcept
Add an If-conditional input.
Definition: NvInfer.h:5365
char const * getName() const noexcept
Return the name of the conditional.
Definition: NvInfer.h:5390
virtual ~IIfConditional() noexcept=default
IConditionLayer * setCondition(ITensor &condition) noexcept
Set the condition tensor for this If-Conditional construct.
Definition: NvInfer.h:5337
IIfConditionalOutputLayer * addOutput(ITensor &trueSubgraphOutput, ITensor &falseSubgraphOutput) noexcept
Add an If-conditional output.
Definition: NvInfer.h:5353
void setName(char const *name) noexcept
Set the name of the conditional.
Definition: NvInfer.h:5380
Definition: NvInfer.h:5297
virtual ~IIfConditionalInputLayer() noexcept=default
Definition: NvInfer.h:5286
virtual ~IIfConditionalOutputLayer() noexcept=default
Application-implemented interface for calibration.
Definition: NvInfer.h:7926
virtual int32_t getBatchSize() const noexcept=0
Get the batch size used for calibration batches.
Definition: NvInfer.h:8009
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8014
virtual ~IInt8EntropyCalibrator2() noexcept=default
Definition: NvInfer.h:7991
virtual ~IInt8EntropyCalibrator() noexcept=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7996
Definition: NvInfer.h:8044
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8049
virtual double getQuantile() const noexcept=0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
Definition: NvInfer.h:8026
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8031
virtual ~IInt8MinMaxCalibrator() noexcept=default
Definition: NvInfer.h:5518
virtual ~IIteratorLayer() noexcept=default
void setReverse(bool reverse) noexcept
Definition: NvInfer.h:5537
bool getReverse() const noexcept
True if and only if reversing input.
Definition: NvInfer.h:5543
int32_t getAxis() const noexcept
Get axis being iterated over.
Definition: NvInfer.h:5527
void setAxis(int32_t axis) noexcept
Set axis to iterate over.
Definition: NvInfer.h:5521
A LRN layer in a network definition.
Definition: NvInfer.h:2045
int32_t getWindowSize() const noexcept
Get the LRN window size.
Definition: NvInfer.h:2066
float getAlpha() const noexcept
Get the LRN alpha value.
Definition: NvInfer.h:2087
void setWindowSize(int32_t windowSize) noexcept
Set the LRN window size.
Definition: NvInfer.h:2056
void setK(float k) noexcept
Set the LRN K value.
Definition: NvInfer.h:2119
void setAlpha(float alpha) noexcept
Set the LRN alpha value.
Definition: NvInfer.h:2077
void setBeta(float beta) noexcept
Set the LRN beta value.
Definition: NvInfer.h:2098
virtual ~ILRNLayer() noexcept=default
float getBeta() const noexcept
Get the LRN beta value.
Definition: NvInfer.h:2108
float getK() const noexcept
Get the LRN K value.
Definition: NvInfer.h:2129
Base class for all layer classes in a network definition.
Definition: NvInfer.h:540
bool precisionIsSet() const noexcept
whether the computational precision has been set for this layer
Definition: NvInfer.h:680
void setPrecision(DataType dataType) noexcept
Set the computational precision of this layer.
Definition: NvInfer.h:656
void setName(char const *name) noexcept
Set the name of a layer.
Definition: NvInfer.h:561
void resetPrecision() noexcept
reset the computational precision for this layer
Definition: NvInfer.h:690
int32_t getNbInputs() const noexcept
Get the number of inputs of a layer.
Definition: NvInfer.h:579
DataType getOutputType(int32_t index) const noexcept
get the output type of this layer
Definition: NvInfer.h:742
DataType getPrecision() const noexcept
get the computational precision of this layer
Definition: NvInfer.h:668
char const * getName() const noexcept
Return the name of a layer.
Definition: NvInfer.h:571
int32_t getNbOutputs() const noexcept
Get the number of outputs of a layer.
Definition: NvInfer.h:600
bool outputTypeIsSet(int32_t index) const noexcept
whether the output type has been set for this layer
Definition: NvInfer.h:755
ITensor * getOutput(int32_t index) const noexcept
Get the layer output corresponding to the given index.
Definition: NvInfer.h:611
void setInput(int32_t index, ITensor &tensor) noexcept
Replace an input of this layer with a specific tensor.
Definition: NvInfer.h:628
void resetOutputType(int32_t index) noexcept
reset the output type for this layer
Definition: NvInfer.h:767
ITensor * getInput(int32_t index) const noexcept
Get the layer input corresponding to the given index.
Definition: NvInfer.h:592
void setOutputType(int32_t index, DataType dataType) noexcept
Set the output type of this layer.
Definition: NvInfer.h:728
LayerType getType() const noexcept
Return the type of a layer.
Definition: NvInfer.h:547
virtual ~ILayer() noexcept=default
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeCommon.h:1500
Definition: NvInfer.h:5237
virtual ~ILoopBoundaryLayer() noexcept=default
ILoop * getLoop() const noexcept
Return pointer to ILoop associated with this boundary layer.
Definition: NvInfer.h:5240
Definition: NvInfer.h:5559
void setName(char const *name) noexcept
Set the name of the loop.
Definition: NvInfer.h:5628
ITripLimitLayer * addTripLimit(ITensor &tensor, TripLimit limit) noexcept
Add a trip-count limiter, based on the given tensor.
Definition: NvInfer.h:5588
IIteratorLayer * addIterator(ITensor &tensor, int32_t axis=0, bool reverse=false) noexcept
Return layer that subscripts tensor by loop iteration.
Definition: NvInfer.h:5601
ILoopOutputLayer * addLoopOutput(ITensor &tensor, LoopOutput outputKind, int32_t axis=0) noexcept
Make an output for this loop, based on the given tensor.
Definition: NvInfer.h:5613
virtual ~ILoop() noexcept=default
char const * getName() const noexcept
Return the name of the loop.
Definition: NvInfer.h:5638
IRecurrenceLayer * addRecurrence(ITensor &initialValue) noexcept
Create a recurrence layer for this loop with initialValue as its first input.
Definition: NvInfer.h:5567
Definition: NvInfer.h:5447
virtual ~ILoopOutputLayer() noexcept=default
int32_t getAxis() const noexcept
Get axis being concatenated over.
Definition: NvInfer.h:5472
LoopOutput getLoopOutput() const noexcept
Definition: NvInfer.h:5449
void setAxis(int32_t axis) noexcept
Set where to insert the contenation axis. Ignored if getLoopOutput() is kLAST_VALUE.
Definition: NvInfer.h:5466
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:4524
apiv::VMatrixMultiplyLayer * mImpl
Definition: NvInfer.h:4550
virtual ~IMatrixMultiplyLayer() noexcept=default
MatrixOperation getOperation(int32_t index) const noexcept
Get the operation for an input tensor.
Definition: NvInfer.h:4544
void setOperation(int32_t index, MatrixOperation op) noexcept
Set the operation for an input tensor.
Definition: NvInfer.h:4532
A non-maximum suppression layer in a network definition.
Definition: NvInfer.h:6485
virtual ~INMSLayer() noexcept=default
void setTopKBoxLimit(int32_t limit) noexcept
Set the TopK box limit parameter for the layer.
Definition: NvInfer.h:6522
void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept
Set the bounding box format parameter for the layer.
Definition: NvInfer.h:6496
BoundingBoxFormat getBoundingBoxFormat() const noexcept
Get the bounding box format parameter for the layer.
Definition: NvInfer.h:6508
apiv::VNMSLayer * mImpl
Definition: NvInfer.h:6558
int32_t getTopKBoxLimit() const noexcept
Get the TopK box limit parameter for the layer.
Definition: NvInfer.h:6532
A network definition for input to the builder.
Definition: NvInfer.h:6582
IPluginV2Layer * addPluginV2(ITensor *const *inputs, int32_t nbInputs, IPluginV2 &plugin) noexcept
Add a plugin layer to the network using the IPluginV2 interface.
Definition: NvInfer.h:7319
IConvolutionLayer * addConvolutionNd(ITensor &input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension convolution layer to the network.
Definition: NvInfer.h:7482
IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7779
IConcatenationLayer * addConcatenation(ITensor *const *inputs, int32_t nbInputs) noexcept
Add a concatenation layer to the network.
Definition: NvInfer.h:6798
TRT_DEPRECATED IDeconvolutionLayer * addDeconvolution(ITensor &input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a deconvolution layer to the network.
Definition: NvInfer.h:6821
IShuffleLayer * addShuffle(ITensor &input) noexcept
Add a shuffle layer to the network.
Definition: NvInfer.h:6902
void setName(char const *name) noexcept
Sets the name of the network.
Definition: NvInfer.h:7362
ILRNLayer * addLRN(ITensor &input, int32_t window, float alpha, float beta, float k) noexcept
Add a LRN layer to the network.
Definition: NvInfer.h:6741
ITopKLayer * addTopK(ITensor &input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
Add a TopK layer to the network.
Definition: NvInfer.h:7076
IAssertionLayer * addAssertion(ITensor &condition, char const *message) noexcept
Add an assertion layer to the network.
Definition: NvInfer.h:7663
IScaleLayer * addScaleNd(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
Add a multi-dimension scale layer to the network.
Definition: NvInfer.h:7560
char const * getName() const noexcept
Returns the name associated with the network.
Definition: NvInfer.h:7376
TRT_DEPRECATED bool hasExplicitPrecision() const noexcept
True if network is an explicit precision network.
Definition: NvInfer.h:7590
IParametricReLULayer * addParametricReLU(ITensor &input, ITensor &slope) noexcept
Add a parametric ReLU layer to the network.
Definition: NvInfer.h:7460
ITensor * getOutput(int32_t index) const noexcept
Get the output tensor specified by the given index.
Definition: NvInfer.h:7003
ITensor * getInput(int32_t index) const noexcept
Get the input tensor specified by the given index.
Definition: NvInfer.h:6973
bool unmarkOutputForShapes(ITensor &tensor) noexcept
Undo markOutputForShapes.
Definition: NvInfer.h:7442
IFillLayer * addFill(Dims dimensions, FillOperation op) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7686
ILoop * addLoop() noexcept
Add a loop to the network.
Definition: NvInfer.h:7606
IDeconvolutionLayer * addDeconvolutionNd(ITensor &input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension deconvolution layer to the network.
Definition: NvInfer.h:7524
IActivationLayer * addActivation(ITensor &input, ActivationType type) noexcept
Add an activation layer to the network.
Definition: NvInfer.h:6703
virtual ~INetworkDefinition() noexcept=default
INMSLayer * addNMS(ITensor &boxes, ITensor &scores, ITensor &maxOutputBoxesPerClass) noexcept
Add a non-maximum suppression layer to the network.
Definition: NvInfer.h:7881
ILayer * getLayer(int32_t index) const noexcept
Get the layer specified by the given index.
Definition: NvInfer.h:6945
TRT_DEPRECATED IRNNv2Layer * addRNNv2(ITensor &input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
Add an layerCount deep RNN layer to the network with hiddenSize internal states that can take a batch...
Definition: NvInfer.h:7258
TRT_DEPRECATED IFullyConnectedLayer * addFullyConnected(ITensor &input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
Add a fully connected layer to the network.
Definition: NvInfer.h:6683
IIfConditional * addIfConditional() noexcept
Add an If-conditional layer to the network.
Definition: NvInfer.h:7833
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:7760
int32_t getNbInputs() const noexcept
Get the number of inputs in the network.
Definition: NvInfer.h:6957
bool hasImplicitBatchDimension() const noexcept
Query whether the network was created with an implicit batch dimension.
Definition: NvInfer.h:7412
IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
Add a reduce layer to the network.
Definition: NvInfer.h:7042
IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation) noexcept
Add a unary layer to the network.
Definition: NvInfer.h:6871
IGridSampleLayer * addGridSample(ITensor &input, ITensor &grid) noexcept
Add a GridSample layer to the network.
Definition: NvInfer.h:7863
void removeTensor(ITensor &tensor) noexcept
remove a tensor from the network definition.
Definition: NvInfer.h:7288
ISelectLayer * addSelect(ITensor &condition, ITensor &thenInput, ITensor &elseInput) noexcept
Add a select layer to the network.
Definition: NvInfer.h:7646
IScatterLayer * addScatter(ITensor &data, ITensor &indices, ITensor &updates, ScatterMode mode) noexcept
Add a Scatter layer to the network with specified mode and axis=0.
Definition: NvInfer.h:7799
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInfer.h:6931
apiv::VNetworkDefinition * mImpl
Definition: NvInfer.h:7887
IPoolingLayer * addPoolingNd(ITensor &input, PoolingType type, Dims windowSize) noexcept
Add a multi-dimension pooling layer to the network.
Definition: NvInfer.h:7502
bool markOutputForShapes(ITensor &tensor) noexcept
Enable tensor's value to be computed by IExecutionContext::getShapeBinding.
Definition: NvInfer.h:7430
IOneHotLayer * addOneHot(ITensor &indices, ITensor &values, ITensor &depth, int32_t axis) noexcept
Add a OneHot layer to the network.
Definition: NvInfer.h:6919
TRT_DEPRECATED IPaddingLayer * addPadding(ITensor &input, DimsHW prePadding, DimsHW postPadding) noexcept
Add a padding layer to the network.
Definition: NvInfer.h:6888
IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
Add a Scale layer to the network.
Definition: NvInfer.h:6768
void unmarkOutput(ITensor &tensor) noexcept
unmark a tensor as a network output.
Definition: NvInfer.h:7300
IIdentityLayer * addIdentity(ITensor &input) noexcept
Add an identity layer.
Definition: NvInfer.h:7273
IGatherLayer * addGatherV2(ITensor &data, ITensor &indices, GatherMode mode) noexcept
Add gather with specified mode, axis=0 and nbElementWiseDims=0.
Definition: NvInfer.h:7108
TRT_DEPRECATED IConvolutionLayer * addConvolution(ITensor &input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a convolution layer to the network.
Definition: NvInfer.h:6660
IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7818
IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op) noexcept
Add an elementwise layer to the network.
Definition: NvInfer.h:6849
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:7745
ISliceLayer * addSlice(ITensor &input, Dims start, Dims size, Dims stride) noexcept
Add a slice layer to the network.
Definition: NvInfer.h:7338
IConstantLayer * addConstant(Dims dimensions, Weights weights) noexcept
Add a constant layer to the network.
Definition: NvInfer.h:7189
IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds) noexcept
Add a RaggedSoftMax layer to the network.
Definition: NvInfer.h:7126
IShapeLayer * addShape(ITensor &input) noexcept
Add a shape layer to the network.
Definition: NvInfer.h:7394
IGatherLayer * addGather(ITensor &data, ITensor &indices, int32_t axis) noexcept
Add gather with mode GatherMode::kDEFAULT and specified axis and nbElementWiseDims=0.
Definition: NvInfer.h:7092
TRT_DEPRECATED IPoolingLayer * addPooling(ITensor &input, PoolingType type, DimsHW windowSize) noexcept
Add a pooling layer to the network.
Definition: NvInfer.h:6722
TRT_DEPRECATED void destroy() noexcept
Destroy this INetworkDefinition object.
Definition: NvInfer.h:7015
IResizeLayer * addResize(ITensor &input) noexcept
Add a resize layer to the network.
Definition: NvInfer.h:7576
IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1) noexcept
Add a MatrixMultiply layer to the network.
Definition: NvInfer.h:7147
ISoftMaxLayer * addSoftMax(ITensor &input) noexcept
Add a SoftMax layer to the network.
Definition: NvInfer.h:6781
IEinsumLayer * addEinsum(ITensor *const *inputs, int32_t nbInputs, char const *equation) noexcept
Add an Einsum layer to the network.
Definition: NvInfer.h:7847
void markOutput(ITensor &tensor) noexcept
Mark a tensor as a network output.
Definition: NvInfer.h:6637
INonZeroLayer * addNonZero(ITensor &input) noexcept
Add a nonzero layer to the network.
Definition: NvInfer.h:7162
int32_t getNbOutputs() const noexcept
Get the number of outputs in the network.
Definition: NvInfer.h:6987
TRT_DEPRECATED IPaddingLayer * addPaddingNd(ITensor &input, Dims prePadding, Dims postPadding) noexcept
Add a padding layer to the network. Only 2D padding is currently supported.
Definition: NvInfer.h:7703
bool setWeightsName(Weights weights, char const *name) noexcept
Associate a name with all current uses of the given weights.
Definition: NvInfer.h:7726
Forward declaration of IEngineInspector for use by other interfaces.
Definition: NvInferRuntime.h:43
Definition: NvInfer.h:4576
virtual ~INonZeroLayer() noexcept=default
A OneHot layer in a network definition.
Definition: NvInfer.h:6307
apiv::VOneHotLayer * mImpl
Definition: NvInfer.h:6328
void setAxis(int32_t axis) noexcept
Set the axis parameter.
Definition: NvInfer.h:6314
int32_t getAxis() const noexcept
Get the value of the axis parameter.
Definition: NvInfer.h:6322
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:1133
Layer that represents a padding operation.
Definition: NvInfer.h:3835
Dims getPostPaddingNd() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3936
TRT_DEPRECATED DimsHW getPrePadding() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3858
TRT_DEPRECATED void setPostPadding(DimsHW padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3872
virtual ~IPaddingLayer() noexcept=default
TRT_DEPRECATED void setPrePadding(DimsHW padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3846
Dims getPrePaddingNd() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3910
TRT_DEPRECATED DimsHW getPostPadding() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3884
void setPostPaddingNd(Dims padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3924
void setPrePaddingNd(Dims padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3898
apiv::VPaddingLayer * mImpl
Definition: NvInfer.h:3942
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:4711
apiv::VParametricReLULayer * mImpl
Definition: NvInfer.h:4713
virtual ~IParametricReLULayer() noexcept=default
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:1234
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:394
Layer type for pluginV2.
Definition: NvInfer.h:3606
virtual ~IPluginV2Layer() noexcept=default
apiv::VPluginV2Layer * mImpl
Definition: NvInfer.h:3619
IPluginV2 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:3613
A Pooling layer in a network definition.
Definition: NvInfer.h:1707
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride for pooling.
Definition: NvInfer.h:1780
PoolingType getPoolingType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1726
void setWindowSizeNd(Dims windowSize) noexcept
Set the multi-dimension window size for pooling.
Definition: NvInfer.h:1959
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1946
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:1922
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Set the stride for pooling.
Definition: NvInfer.h:1768
bool getAverageCountExcludesPadding() const noexcept
Get whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1866
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1894
void setPoolingType(PoolingType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1716
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1935
TRT_DEPRECATED void setWindowSize(DimsHW windowSize) noexcept
Set the window size for pooling.
Definition: NvInfer.h:1740
Dims getWindowSizeNd() const noexcept
Get the multi-dimension window size for pooling.
Definition: NvInfer.h:1969
void setAverageCountExcludesPadding(bool exclusive) noexcept
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1855
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding for pooling.
Definition: NvInfer.h:2013
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding for pooling.
Definition: NvInfer.h:1884
float getBlendFactor() const noexcept
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1838
Dims getStrideNd() const noexcept
Get the multi-dimension stride for pooling.
Definition: NvInfer.h:1994
virtual ~IPoolingLayer() noexcept=default
Dims getPaddingNd() const noexcept
Get the multi-dimension padding for pooling.
Definition: NvInfer.h:2025
TRT_DEPRECATED DimsHW getWindowSize() const noexcept
Get the window size for pooling.
Definition: NvInfer.h:1752
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding for pooling.
Definition: NvInfer.h:1810
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride for pooling.
Definition: NvInfer.h:1984
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding for pooling.
Definition: NvInfer.h:1912
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding for pooling.
Definition: NvInfer.h:1796
void setBlendFactor(float blendFactor) noexcept
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1825
A Quantize layer in a network definition.
Definition: NvInfer.h:5971
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5992
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5981
virtual ~IQuantizeLayer() noexcept=default
An RNN layer in a network definition, version 2.
Definition: NvInfer.h:3316
void setDirection(RNNDirection op) noexcept
Set the direction of the RNN layer.
Definition: NvInfer.h:3416
void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept
Set the bias parameters for an individual gate in the RNN.
Definition: NvInfer.h:3520
void setCellState(ITensor &cell) noexcept
Set the initial cell state of the LSTM with the provided cell ITensor.
Definition: NvInfer.h:3576
int32_t getDataLength() const noexcept
Get the maximum data length of the RNN.
Definition: NvInfer.h:3330
void setInputMode(RNNInputMode op) noexcept
Set the input mode of the RNN layer.
Definition: NvInfer.h:3391
Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
Get the bias parameters for an individual gate in the RNN.
Definition: NvInfer.h:3530
void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept
Set the weight parameters for an individual gate in the RNN.
Definition: NvInfer.h:3485
void setOperation(RNNOperation op) noexcept
Set the operation of the RNN layer.
Definition: NvInfer.h:3371
Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
Get the weight parameters for an individual gate in the RNN.
Definition: NvInfer.h:3495
RNNDirection getDirection() const noexcept
Get the direction of the RNN layer.
Definition: NvInfer.h:3426
void setSequenceLengths(ITensor &seqLengths) noexcept
Specify individual sequence lengths in the batch with the ITensor pointed to by seqLengths.
Definition: NvInfer.h:3349
RNNInputMode getInputMode() const noexcept
Get the input mode of the RNN layer.
Definition: NvInfer.h:3401
ITensor * getHiddenState() const noexcept
Get the initial hidden state of the RNN.
Definition: NvInfer.h:3557
ITensor * getCellState() const noexcept
Get the initial cell state of the RNN.
Definition: NvInfer.h:3586
int32_t getMaxSeqLength() const noexcept
Get the maximum sequence length of the RNN.
Definition: NvInfer.h:3326
int32_t getLayerCount() const noexcept
Get the layer count of the RNN.
Definition: NvInfer.h:3318
apiv::VRNNv2Layer * mImpl
Definition: NvInfer.h:3592
ITensor * getSequenceLengths() const noexcept
Get the sequence lengths specified for the RNN.
Definition: NvInfer.h:3361
RNNOperation getOperation() const noexcept
Get the operation of the RNN layer.
Definition: NvInfer.h:3381
virtual ~IRNNv2Layer() noexcept=default
int32_t getHiddenSize() const noexcept
Get the hidden size of the RNN.
Definition: NvInfer.h:3322
void setHiddenState(ITensor &hidden) noexcept
Set the initial hidden state of the RNN with the provided hidden ITensor.
Definition: NvInfer.h:3547
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:4597
apiv::VRaggedSoftMaxLayer * mImpl
Definition: NvInfer.h:4599
virtual ~IRaggedSoftMaxLayer() noexcept=default
Definition: NvInfer.h:5402
virtual ~IRecurrenceLayer() noexcept=default
Layer that represents a reduction across a non-bool tensor.
Definition: NvInfer.h:3757
void setKeepDimensions(bool keepDimensions) noexcept
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:3804
void setOperation(ReduceOperation op) noexcept
Set the reduce operation for the layer.
Definition: NvInfer.h:3764
ReduceOperation getOperation() const noexcept
Get the reduce operation for the layer.
Definition: NvInfer.h:3774
virtual ~IReduceLayer() noexcept=default
uint32_t getReduceAxes() const noexcept
Get the axes over which to reduce for the layer.
Definition: NvInfer.h:3794
void setReduceAxes(uint32_t reduceAxes) noexcept
Set the axes over which to reduce.
Definition: NvInfer.h:3784
apiv::VReduceLayer * mImpl
Definition: NvInfer.h:3820
bool getKeepDimensions() const noexcept
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:3814
A resize layer in a network definition.
Definition: NvInfer.h:4903
void setSelectorForSinglePixel(ResizeSelector selector) noexcept
Set coordinate selector function when resized to single pixel.
Definition: NvInfer.h:5092
void setOutputDimensions(Dims dimensions) noexcept
Set the output dimensions.
Definition: NvInfer.h:4923
void setNearestRounding(ResizeRoundMode value) noexcept
Set rounding mode for nearest neighbor resize.
Definition: NvInfer.h:5116
virtual ~IResizeLayer() noexcept=default
int32_t getScales(int32_t size, float *scales) const noexcept
Copies resize scales to scales[0, ..., nbScales-1], where nbScales is the number of scales that were ...
Definition: NvInfer.h:4982
void setCubicCoeff(float A) noexcept
Set the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:5148
ResizeMode getResizeMode() const noexcept
Get resize mode for an input tensor.
Definition: NvInfer.h:5004
void setScales(float const *scales, int32_t nbScales) noexcept
Set the resize scales.
Definition: NvInfer.h:4963
void setResizeMode(ResizeMode resizeMode) noexcept
Set resize mode for an input tensor.
Definition: NvInfer.h:4994
float getCubicCoeff() const noexcept
Get the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:5158
ResizeSelector getSelectorForSinglePixel() const noexcept
Get the coordinate selector function when resized to single pixel.
Definition: NvInfer.h:5102
TRT_DEPRECATED bool getAlignCorners() const noexcept
True if align corners has been set.
Definition: NvInfer.h:5032
void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept
Set coordinate transformation function.
Definition: NvInfer.h:5067
void setExcludeOutside(bool excludeFlag) noexcept
Set the state for excluding outside pixels.
Definition: NvInfer.h:5171
Dims getOutputDimensions() const noexcept
Get the output dimensions.
Definition: NvInfer.h:4933
ResizeRoundMode getNearestRounding() const noexcept
Get rounding mode for nearest neighbor resize.
Definition: NvInfer.h:5126
bool getExcludeOutside() const noexcept
Get the state for excluding outside pixels.
Definition: NvInfer.h:5181
TRT_DEPRECATED void setAlignCorners(bool alignCorners) noexcept
Set whether to align corners while resizing.
Definition: NvInfer.h:5020
ResizeCoordinateTransformation getCoordinateTransformation() const noexcept
Get coordinate transformation function.
Definition: NvInfer.h:5077
A Scale layer in a network definition.
Definition: NvInfer.h:2189
Weights getScale() const noexcept
Get the scale value.
Definition: NvInfer.h:2246
Weights getPower() const noexcept
Get the power value.
Definition: NvInfer.h:2266
void setScale(Weights scale) noexcept
Set the scale value.
Definition: NvInfer.h:2236
void setPower(Weights power) noexcept
Set the power value.
Definition: NvInfer.h:2256
ScaleMode getMode() const noexcept
Get the scale mode.
Definition: NvInfer.h:2206
void setShift(Weights shift) noexcept
Set the shift value.
Definition: NvInfer.h:2216
void setChannelAxis(int32_t channelAxis) noexcept
Set the channel axis.
Definition: NvInfer.h:2302
Weights getShift() const noexcept
Get the shift value.
Definition: NvInfer.h:2226
virtual ~IScaleLayer() noexcept=default
void setMode(ScaleMode mode) noexcept
Set the scale mode.
Definition: NvInfer.h:2196
int32_t getChannelAxis() const noexcept
Get the channel axis.
Definition: NvInfer.h:2281
A scatter layer in a network definition. Supports several kinds of scattering.
Definition: NvInfer.h:6234
void setMode(ScatterMode mode) noexcept
Set the scatter mode.
Definition: NvInfer.h:6241
apiv::VScatterLayer * mImpl
Definition: NvInfer.h:6275
void setAxis(int32_t axis) noexcept
Set the axis used by ScatterMode::kELEMENTS.
Definition: NvInfer.h:6261
int32_t getAxis() const noexcept
Get the axis.
Definition: NvInfer.h:6269
ScatterMode getMode() const noexcept
Get the scatter mode.
Definition: NvInfer.h:6251
virtual ~IScatterLayer() noexcept=default
Definition: NvInfer.h:5652
virtual ~ISelectLayer() noexcept=default
Layer type for getting shape of a tensor.
Definition: NvInfer.h:4353
virtual ~IShapeLayer() noexcept=default
apiv::VShapeLayer * mImpl
Definition: NvInfer.h:4355
Layer type for shuffling data.
Definition: NvInfer.h:3970
apiv::VShuffleLayer * mImpl
Definition: NvInfer.h:4125
void setReshapeDimensions(Dims dimensions) noexcept
Set the reshaped dimensions.
Definition: NvInfer.h:4018
void setFirstTranspose(Permutation permutation) noexcept
Set the permutation applied by the first transpose operation.
Definition: NvInfer.h:3981
void setSecondTranspose(Permutation permutation) noexcept
Set the permutation applied by the second transpose operation.
Definition: NvInfer.h:4078
Dims getReshapeDimensions() const noexcept
Get the reshaped dimensions.
Definition: NvInfer.h:4031
Permutation getFirstTranspose() const noexcept
Get the permutation applied by the first transpose operation.
Definition: NvInfer.h:3993
virtual ~IShuffleLayer() noexcept=default
Permutation getSecondTranspose() const noexcept
Get the permutation applied by the second transpose operation.
Definition: NvInfer.h:4090
bool getZeroIsPlaceholder() const noexcept
Get meaning of 0 in reshape dimensions.
Definition: NvInfer.h:4119
void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
Set meaning of 0 in reshape dimensions.
Definition: NvInfer.h:4106
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:4203
void setMode(SliceMode mode) noexcept
Set the slice mode.
Definition: NvInfer.h:4297
apiv::VSliceLayer * mImpl
Definition: NvInfer.h:4336
virtual ~ISliceLayer() noexcept=default
void setStride(Dims stride) noexcept
Set the stride for computing the output slice data.
Definition: NvInfer.h:4272
void setStart(Dims start) noexcept
Set the start offset that the slice layer uses to create the output slice.
Definition: NvInfer.h:4214
Dims getStart() const noexcept
Get the start offset for the slice layer.
Definition: NvInfer.h:4229
void setSize(Dims size) noexcept
Set the dimensions of the output slice.
Definition: NvInfer.h:4243
Dims getSize() const noexcept
Get dimensions of the output slice.
Definition: NvInfer.h:4258
SliceMode getMode() const noexcept
Get the slice mode.
Definition: NvInfer.h:4307
Dims getStride() const noexcept
Get the stride for the output slice.
Definition: NvInfer.h:4287
A Softmax layer in a network definition.
Definition: NvInfer.h:2334
void setAxes(uint32_t axes) noexcept
Set the axis along which softmax is computed. Currently, only one axis can be set.
Definition: NvInfer.h:2366
uint32_t getAxes() const noexcept
Get the axis along which softmax occurs.
Definition: NvInfer.h:2376
virtual ~ISoftMaxLayer() noexcept=default
A tensor in a network definition.
Definition: NvInfer.h:172
bool setDynamicRange(float min, float max) noexcept
Set dynamic range for the tensor.
Definition: NvInfer.h:274
void setAllowedFormats(TensorFormats formats) noexcept
Set allowed formats for this tensor. By default all formats are allowed. Shape tensors (for which isS...
Definition: NvInfer.h:410
TensorLocation getLocation() const noexcept
Get the storage location of a tensor.
Definition: NvInfer.h:338
void resetDynamicRange() noexcept
Undo effect of setDynamicRange.
Definition: NvInfer.h:371
void setName(char const *name) noexcept
Set the tensor name.
Definition: NvInfer.h:188
bool isExecutionTensor() const noexcept
Whether the tensor is an execution tensor.
Definition: NvInfer.h:481
void setType(DataType type) noexcept
Set the data type of a tensor.
Definition: NvInfer.h:247
bool dynamicRangeIsSet() const noexcept
Query whether dynamic range is set.
Definition: NvInfer.h:363
void setLocation(TensorLocation location) noexcept
Set the storage location of a tensor.
Definition: NvInfer.h:353
char const * getName() const noexcept
Get the tensor name.
Definition: NvInfer.h:200
bool isShapeTensor() const noexcept
Whether the tensor is a shape tensor.
Definition: NvInfer.h:458
float getDynamicRangeMax() const noexcept
Get maximum of dynamic range.
Definition: NvInfer.h:391
bool isNetworkInput() const noexcept
Whether the tensor is a network input.
Definition: NvInfer.h:282
bool isNetworkOutput() const noexcept
Whether the tensor is a network output.
Definition: NvInfer.h:290
bool getBroadcastAcrossBatch() const noexcept
Check if tensor is broadcast across the batch.
Definition: NvInfer.h:328
void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
Set whether to enable broadcast of tensor across the batch.
Definition: NvInfer.h:312
DataType getType() const noexcept
Get the data type of a tensor.
Definition: NvInfer.h:259
apiv::VTensor * mImpl
Definition: NvInfer.h:528
float getDynamicRangeMin() const noexcept
Get minimum of dynamic range.
Definition: NvInfer.h:381
virtual ~ITensor() noexcept=default
void setDimensionName(int32_t index, char const *name) noexcept
Name a dimension of an input tensor.
Definition: NvInfer.h:507
char const * getDimensionName(int32_t index) const noexcept
Get the name of an input dimension.
Definition: NvInfer.h:522
void setDimensions(Dims dimensions) noexcept
Set the dimensions of a tensor.
Definition: NvInfer.h:219
Dims getDimensions() const noexcept
Get the dimensions of a tensor.
Definition: NvInfer.h:232
TensorFormats getAllowedFormats() const noexcept
Get a bitmask of TensorFormat values that the tensor supports. For a shape tensor,...
Definition: NvInfer.h:423
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:8470
bool combine(ITimingCache const &inputCache, bool ignoreMismatch) noexcept
Combine input timing cache into local instance.
Definition: NvInfer.h:8507
virtual ~ITimingCache() noexcept=default
apiv::VTimingCache * mImpl
Definition: NvInfer.h:8523
bool reset() noexcept
Empty the timing cache.
Definition: NvInfer.h:8517
Layer that represents a TopK reduction.
Definition: NvInfer.h:4389
void setK(int32_t k) noexcept
Set the k value for the layer.
Definition: NvInfer.h:4418
void setReduceAxes(uint32_t reduceAxes) noexcept
Set which axes to reduce for the layer.
Definition: NvInfer.h:4438
TopKOperation getOperation() const noexcept
Get the operation for the layer.
Definition: NvInfer.h:4406
apiv::VTopKLayer * mImpl
Definition: NvInfer.h:4454
void setOperation(TopKOperation op) noexcept
Set the operation for the layer.
Definition: NvInfer.h:4396
int32_t getK() const noexcept
Get the k value for the layer.
Definition: NvInfer.h:4428
uint32_t getReduceAxes() const noexcept
Get the axes to reduce for the layer.
Definition: NvInfer.h:4448
virtual ~ITopKLayer() noexcept=default
Definition: NvInfer.h:5505
TripLimit getTripLimit() const noexcept
Definition: NvInfer.h:5507
virtual ~ITripLimitLayer() noexcept=default
Layer that represents an unary operation.
Definition: NvInfer.h:3682
void setOperation(UnaryOperation op) noexcept
Set the unary operation for the layer.
Definition: NvInfer.h:3691
apiv::VUnaryLayer * mImpl
Definition: NvInfer.h:3707
UnaryOperation getOperation() const noexcept
Get the unary operation for the layer.
Definition: NvInfer.h:3701
virtual ~IUnaryLayer() noexcept=default
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:126
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:1345
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4808
@ kFORMULA
Use formula to map the original index.
@ kUPPER
Select the upper left pixel.
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:69
nvinfer1::IPluginRegistry * getBuilderPluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for the given capability or nullptr if no registry exists.
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8534
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2145
@ kUNIFORM
Identical coefficients across all elements of the tensor.
@ kCHANNEL
Per-channel coefficients.
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:8344
constexpr int32_t EnumMax< RNNDirection >() noexcept
Definition: NvInfer.h:3237
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:6423
@ kCENTER_SIZES
(x_center, y_center, width, height) where (x_center, y_center) is the center point of the box
@ kCORNER_PAIRS
(x1, y1, x2, y2) where (x1, y1) and (x2, y2) are any pair of diagonal corners
constexpr int32_t EnumMax< BuilderFlag >() noexcept
Definition: NvInfer.h:8454
constexpr int32_t EnumMax< LayerType >() noexcept
Definition: NvInfer.h:107
constexpr int32_t EnumMax< RNNGateType >() noexcept
Definition: NvInfer.h:3298
constexpr int32_t EnumMax< CalibrationAlgoType >() noexcept
Definition: NvInfer.h:7909
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3637
@ kCOSH
Hyperbolic cosine.
@ kACOSH
Inverse hyperbolic cosine.
@ kERF
Gauss error function.
@ kACOS
Inverse cosine.
@ kABS
Absolute value.
@ kSINH
Hyperbolic sine.
@ kROUND
Round to nearest even for float datatype.
@ kATANH
Inverse hyperbolic tangent.
@ kASINH
Inverse hyperbolic sine.
@ kSIGN
Sign, If input > 0, output 1; if input < 0, output -1; if input == 0, output 0.
@ kEXP
Exponentiation.
@ kATAN
Inverse tangent.
constexpr int32_t EnumMax< ReduceOperation >() noexcept
Definition: NvInfer.h:3744
constexpr int32_t EnumMax< TripLimit >() noexcept
Definition: NvInfer.h:5229
constexpr int32_t EnumMax< RNNInputMode >() noexcept
Definition: NvInfer.h:3269
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3258
@ kSKIP
No operation is performed on the first recurrent layer.
@ kLINEAR
Perform the normal matrix multiplication in the first recurrent layer.
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:126
@ kSELU
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
@ kTANH
TanH activation.
@ kSCALED_TANH
Scaled tanh activation: alpha*tanh(beta*x)
@ kRELU
Rectified linear activation.
@ kELU
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
@ kLEAKY_RELU
LeakyRelu activation: x>=0 ? x : alpha * x.
@ kSOFTSIGN
Softsign activation: x / (1+|x|)
@ kHARD_SIGMOID
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
@ kTHRESHOLDED_RELU
Thresholded ReLU activation: x>alpha ? x : 0.
@ kSIGMOID
Sigmoid activation.
@ kCLIP
Clip activation: max(alpha, min(beta, x))
@ kSOFTPLUS
Parametric softplus activation: alpha*log(exp(beta*x)+1)
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5712
@ kLINSPACE
Generate evenly spaced numbers over a specified interval.
@ kRANDOM_UNIFORM
Generate a tensor with random values drawn from a uniform distribution.
@ kRANDOM_NORMAL
Generate a tensor with random values drawn from a normal distribution.
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4838
@ kHALF_UP
Round half up.
@ kHALF_DOWN
Round half down.
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3282
@ kUPDATE
Update gate (z).
@ kHIDDEN
Hidden gate (h).
@ kFORGET
Forget gate (f).
@ kOUTPUT
Output gate (o).
@ kRESET
Reset gate (r).
@ kINPUT
Input gate (i).
@ kCELL
Cell gate (c).
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:1000
@ kSAME_LOWER
Use SAME padding, with prePadding >= postPadding.
@ kEXPLICIT_ROUND_DOWN
Use explicit padding, rounding output size down.
@ kCAFFE_ROUND_DOWN
Use CAFFE padding, rounding output size down, uses prePadding value.
@ kEXPLICIT_ROUND_UP
Use explicit padding, rounding output size up.
@ kCAFFE_ROUND_UP
Use CAFFE padding, rounding output size up, uses prePadding value.
@ kSAME_UPPER
Use SAME padding, with prePadding <= postPadding.
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:5217
@ kWHILE
Tensor is a scalar of type kBOOL. Loop terminates when value is false.
@ kCOUNT
Tensor is scalar of type kINT32 that contains the trip count.
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:9322
PreviewFeature
Define preview features.
Definition: NvInfer.h:8587
@ kDISABLE_EXTERNAL_TACTIC_SOURCES_FOR_CORE_0805
constexpr int32_t EnumMax< GatherMode >() noexcept
Definition: NvInfer.h:2953
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:117
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:8378
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:598
constexpr int32_t EnumMax< ScaleMode >() noexcept
Definition: NvInfer.h:2157
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7896
LayerType
The type values of layer classes.
Definition: NvInfer.h:53
@ kGRID_SAMPLE
Grid sample layer.
@ kRAGGED_SOFTMAX
Ragged softmax layer.
@ kDECONVOLUTION
Deconvolution layer.
@ kREDUCE
Reduce layer.
@ kASSERTION
Assertion layer.
@ kTOPK
TopK layer.
@ kRESIZE
Resize Layer.
@ kPADDING
Padding layer.
@ kMATRIX_MULTIPLY
Matrix multiply layer.
@ kCONDITION
Condition layer.
@ kCONDITIONAL_INPUT
Conditional Input layer.
@ kIDENTITY
Identity layer.
@ kQUANTIZE
Quantize layer.
@ kSCATTER
Scatter layer.
@ kCONVOLUTION
Convolution layer.
@ kPARAMETRIC_RELU
Parametric ReLU layer.
@ kCONCATENATION
Concatenation layer.
@ kONE_HOT
OneHot layer.
@ kSLICE
Slice layer.
@ kEINSUM
Einsum layer.
@ kSOFTMAX
SoftMax layer.
@ kSHAPE
Shape layer.
@ kFULLY_CONNECTED
Fully connected layer.
@ kRECURRENCE
Loop Recurrence layer.
@ kDEQUANTIZE
Dequantize layer.
@ kSHUFFLE
Shuffle layer.
@ kITERATOR
Loop Iterator layer.
@ kPOOLING
Pooling layer.
@ kTRIP_LIMIT
Loop Trip limit layer.
@ kSCALE
Scale layer.
@ kGATHER
Gather layer.
@ kUNARY
UnaryOp operation Layer.
@ kACTIVATION
Activation layer.
@ kELEMENTWISE
Elementwise layer.
@ kSELECT
Select layer.
@ kPLUGIN_V2
PluginV2 layer.
@ kLOOP_OUTPUT
Loop output layer.
@ kCONDITIONAL_OUTPUT
Conditional Output layer.
@ kCONSTANT
Constant layer.
@ kNON_ZERO
NonZero layer.
@ kRNN_V2
RNNv2 layer.
@ kFILL
Fill layer.
@ kPLUGIN
Plugin layer.
constexpr int32_t EnumMax< QuantizationFlag >() noexcept
Definition: NvInfer.h:8367
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:4135
@ kCLAMP
Out of bounds indices are clamped to bounds.
@ kSTRICT_BOUNDS
Fail with error when the coordinates are out of bounds.
@ kWRAP
Coordinates wrap around periodically.
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2941
@ kDEFAULT
Similar to ONNX Gather.
@ kELEMENT
Similar to ONNX GatherElements.
@ kND
Similar to ONNX GatherND.
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:118
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:1357
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:9333
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2851
@ kSUB
Subtract the second element from the first.
@ kSUM
Sum of the two elements.
@ kPROD
Product of the two elements.
@ kFLOOR_DIV
Floor division of the first element by the second.
@ kEQUAL
Check if two elements are equal.
@ kAND
Logical AND of two elements.
@ kOR
Logical OR of two elements.
@ kMIN
Minimum of the two elements.
@ kPOW
The first element to the power of the second element.
@ kLESS
Check if element in first tensor is less than corresponding element in second tensor.
@ kGREATER
Check if element in first tensor is greater than corresponding element in second tensor.
@ kXOR
Logical XOR of two elements.
@ kDIV
Divide the first element by the second.
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:8354
constexpr int32_t EnumMax< SampleMode >() noexcept
Definition: NvInfer.h:4155
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3226
@ kBIDIRECTION
Network iterates from first to last and vice versa and outputs concatenated.
@ kUNIDIRECTION
Network iterations from first input to last input.
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:4723
@ kNEAREST
ND (0 < N <= 8) nearest neighbor resizing.
@ kCUBIC
Supports bicubic (2D) interpolation.
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:8388
@ kDEBUG
Enable debugging of layers via synchronizing after every layer.
@ kGPU_FALLBACK
Enable layers marked to execute on GPU if layer cannot execute on DLA.
@ kSPARSE_WEIGHTS
Allow the builder to examine weights and use optimized functions when weights have suitable sparsity.
@ kFP16
Enable FP16 layer selection, with FP32 fallback.
@ kINT8
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
@ kDISABLE_TIMING_CACHE
Disable reuse of timing information across identical layers.
@ kREFIT
Enable building a refittable engine.
@ kOBEY_PRECISION_CONSTRAINTS
Require that layers execute in specified precisions. Build fails otherwise.
@ kREJECT_EMPTY_ALGORITHMS
Fail if IAlgorithmSelector::selectAlgorithms returns an empty set of algorithms.
constexpr int32_t EnumMax< TopKOperation >() noexcept
Definition: NvInfer.h:4376
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:201
constexpr int32_t EnumMax< MemoryPoolType >() noexcept
Definition: NvInfer.h:8573
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4365
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3730
constexpr int32_t EnumMax< LoopOutput >() noexcept
Definition: NvInfer.h:5210
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:3200
@ kGRU
Three-gate network consisting of Gated Recurrent Units.
@ kLSTM
Four-gate LSTM network w/o peephole connections.
constexpr int32_t EnumMax< NetworkDefinitionCreationFlag >() noexcept
Definition: NvInfer.h:9352
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:6161
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4465
@ kTRANSPOSE
Like kNONE, but transpose the matrix dimensions.
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4754
constexpr int32_t EnumMax< UnaryOperation >() noexcept
Definition: NvInfer.h:3669
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:5193
@ kLAST_VALUE
Output value is value of tensor for last iteration.
@ kCONCATENATE
Output value is concatenation of values of tensor for each iteration, in forward order.
@ kREVERSE
Output value is concatenation of values of tensor for each iteration, in reverse order.
constexpr int32_t EnumMax< BoundingBoxFormat >() noexcept
Definition: NvInfer.h:6436
constexpr int32_t EnumMax< MatrixOperation >() noexcept
Definition: NvInfer.h:4493