TensorRT 8.4.1
NvInfer.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
4 *
5 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
6 * property and proprietary rights in and to this material, related
7 * documentation and any modifications thereto. Any use, reproduction,
8 * disclosure or distribution of this material and related documentation
9 * without an express license agreement from NVIDIA CORPORATION or
10 * its affiliates is strictly prohibited.
11 */
12
13#ifndef NV_INFER_H
14#define NV_INFER_H
15
16#include "NvInferLegacyDims.h"
17#include "NvInferRuntime.h"
18
27//
30
36
42namespace nvinfer1
43{
44
52enum class LayerType : int32_t
53{
54 kCONVOLUTION = 0,
56 kACTIVATION = 2,
57 kPOOLING = 3,
58 kLRN = 4,
59 kSCALE = 5,
60 kSOFTMAX = 6,
61 kDECONVOLUTION = 7,
62 kCONCATENATION = 8,
63 kELEMENTWISE = 9,
64 kPLUGIN = 10,
65 kUNARY = 11,
66 kPADDING = 12,
67 kSHUFFLE = 13,
68 kREDUCE = 14,
69 kTOPK = 15,
70 kGATHER = 16,
71 kMATRIX_MULTIPLY = 17,
72 kRAGGED_SOFTMAX = 18,
73 kCONSTANT = 19,
74 kRNN_V2 = 20,
75 kIDENTITY = 21,
76 kPLUGIN_V2 = 22,
77 kSLICE = 23,
78 kSHAPE = 24,
79 kPARAMETRIC_RELU = 25,
80 kRESIZE = 26,
81 kTRIP_LIMIT = 27,
82 kRECURRENCE = 28,
83 kITERATOR = 29,
84 kLOOP_OUTPUT = 30,
85 kSELECT = 31,
86 kFILL = 32,
87 kQUANTIZE = 33,
88 kDEQUANTIZE = 34,
89 kCONDITION = 35,
92 kSCATTER = 38,
93 kEINSUM = 39,
94 kASSERTION = 40,
95};
96
102template <>
103constexpr inline int32_t EnumMax<LayerType>() noexcept
104{
105 return 41;
106}
107
114using TensorFormats = uint32_t;
115
121enum class ActivationType : int32_t
122{
123 kRELU = 0,
124 kSIGMOID = 1,
125 kTANH = 2,
126 kLEAKY_RELU = 3,
127 kELU = 4,
128 kSELU = 5,
129 kSOFTSIGN = 6,
130 kSOFTPLUS = 7,
131 kCLIP = 8,
132 kHARD_SIGMOID = 9,
133 kSCALED_TANH = 10,
135};
136
137namespace impl
138{
144template <>
146{
147 static constexpr int32_t kVALUE = 12;
148};
149} // namespace impl
150
166class ITensor : public INoCopy
167{
168public:
181 void setName(char const* name) noexcept
182 {
183 mImpl->setName(name);
184 }
185
193 char const* getName() const noexcept
194 {
195 return mImpl->getName();
196 }
197
212 void setDimensions(Dims dimensions) noexcept
213 {
214 mImpl->setDimensions(dimensions);
215 }
216
225 Dims getDimensions() const noexcept
226 {
227 return mImpl->getDimensions();
228 }
229
240 void setType(DataType type) noexcept
241 {
242 mImpl->setType(type);
243 }
244
252 DataType getType() const noexcept
253 {
254 return mImpl->getType();
255 }
256
267 bool setDynamicRange(float min, float max) noexcept
268 {
269 return mImpl->setDynamicRange(min, max);
270 }
271
275 bool isNetworkInput() const noexcept
276 {
277 return mImpl->isNetworkInput();
278 }
279
283 bool isNetworkOutput() const noexcept
284 {
285 return mImpl->isNetworkOutput();
286 }
287
305 void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
306 {
307 mImpl->setBroadcastAcrossBatch(broadcastAcrossBatch);
308 }
309
321 bool getBroadcastAcrossBatch() const noexcept
322 {
323 return mImpl->getBroadcastAcrossBatch();
324 }
325
332 {
333 return mImpl->getLocation();
334 }
335
346 void setLocation(TensorLocation location) noexcept
347 {
348 mImpl->setLocation(location);
349 }
350
356 bool dynamicRangeIsSet() const noexcept
357 {
358 return mImpl->dynamicRangeIsSet();
359 }
360
364 void resetDynamicRange() noexcept
365 {
366 mImpl->resetDynamicRange();
367 }
368
374 float getDynamicRangeMin() const noexcept
375 {
376 return mImpl->getDynamicRangeMin();
377 }
378
384 float getDynamicRangeMax() const noexcept
385 {
386 return mImpl->getDynamicRangeMax();
387 }
388
403 void setAllowedFormats(TensorFormats formats) noexcept
404 {
405 mImpl->setAllowedFormats(formats);
406 }
407
417 {
418 return mImpl->getAllowedFormats();
419 }
420
451 bool isShapeTensor() const noexcept
452 {
453 return mImpl->isShapeTensor();
454 }
455
474 bool isExecutionTensor() const noexcept
475 {
476 return mImpl->isExecutionTensor();
477 }
478
479protected:
480 apiv::VTensor* mImpl;
481 virtual ~ITensor() noexcept = default;
482};
483
491class ILayer : public INoCopy
492{
493public:
499 LayerType getType() const noexcept
500 {
501 return mLayer->getType();
502 }
503
511 void setName(char const* name) noexcept
512 {
513 mLayer->setName(name);
514 }
515
519
522 char const* getName() const noexcept
523 {
524 return mLayer->getName();
525 }
526
530 int32_t getNbInputs() const noexcept
531 {
532 return mLayer->getNbInputs();
533 }
534
543 ITensor* getInput(int32_t index) const noexcept
544 {
545 return mLayer->getInput(index);
546 }
547
551 int32_t getNbOutputs() const noexcept
552 {
553 return mLayer->getNbOutputs();
554 }
555
562 ITensor* getOutput(int32_t index) const noexcept
563 {
564 return mLayer->getOutput(index);
565 }
566
579 void setInput(int32_t index, ITensor& tensor) noexcept
580 {
581 return mLayer->setInput(index, tensor);
582 }
583
607 void setPrecision(DataType dataType) noexcept
608 {
609 mLayer->setPrecision(dataType);
610 }
611
619 DataType getPrecision() const noexcept
620 {
621 return mLayer->getPrecision();
622 }
623
631 bool precisionIsSet() const noexcept
632 {
633 return mLayer->precisionIsSet();
634 }
635
641 void resetPrecision() noexcept
642 {
643 mLayer->resetPrecision();
644 }
645
679 void setOutputType(int32_t index, DataType dataType) noexcept
680 {
681 mLayer->setOutputType(index, dataType);
682 }
683
693 DataType getOutputType(int32_t index) const noexcept
694 {
695 return mLayer->getOutputType(index);
696 }
697
706 bool outputTypeIsSet(int32_t index) const noexcept
707 {
708 return mLayer->outputTypeIsSet(index);
709 }
710
718 void resetOutputType(int32_t index) noexcept
719 {
720 return mLayer->resetOutputType(index);
721 }
722
723protected:
724 virtual ~ILayer() noexcept = default;
725 apiv::VLayer* mLayer;
726};
727
950enum class PaddingMode : int32_t
951{
954 kSAME_UPPER = 2,
955 kSAME_LOWER = 3,
957 kCAFFE_ROUND_UP = 5
958};
959
960namespace impl
961{
967template <>
969{
970 static constexpr int32_t kVALUE = 6;
971};
972} // namespace impl
973
987{
988public:
998 TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
999 {
1000 mImpl->setKernelSize(kernelSize);
1001 }
1002
1011 {
1012 return mImpl->getKernelSize();
1013 }
1014
1022 void setNbOutputMaps(int32_t nbOutputMaps) noexcept
1023 {
1024 mImpl->setNbOutputMaps(nbOutputMaps);
1025 }
1026
1032 int32_t getNbOutputMaps() const noexcept
1033 {
1034 return mImpl->getNbOutputMaps();
1035 }
1036
1048 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
1049 {
1050 mImpl->setStride(stride);
1051 }
1052
1059 {
1060 return mImpl->getStride();
1061 }
1062
1078 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
1079 {
1080 return mImpl->setPadding(padding);
1081 }
1082
1091 {
1092 return mImpl->getPadding();
1093 }
1094
1110 void setNbGroups(int32_t nbGroups) noexcept
1111 {
1112 mImpl->setNbGroups(nbGroups);
1113 }
1114
1120 int32_t getNbGroups() const noexcept
1121 {
1122 return mImpl->getNbGroups();
1123 }
1124
1134 void setKernelWeights(Weights weights) noexcept
1135 {
1136 mImpl->setKernelWeights(weights);
1137 }
1138
1144 Weights getKernelWeights() const noexcept
1145 {
1146 return mImpl->getKernelWeights();
1147 }
1148
1159 void setBiasWeights(Weights weights) noexcept
1160 {
1161 mImpl->setBiasWeights(weights);
1162 }
1163
1169 Weights getBiasWeights() const noexcept
1170 {
1171 return mImpl->getBiasWeights();
1172 }
1173
1185 TRT_DEPRECATED void setDilation(DimsHW dilation) noexcept
1186 {
1187 return mImpl->setDilation(dilation);
1188 }
1189
1198 {
1199 return mImpl->getDilation();
1200 }
1201
1214 void setPrePadding(Dims padding) noexcept
1215 {
1216 mImpl->setPrePadding(padding);
1217 }
1218
1224 Dims getPrePadding() const noexcept
1225 {
1226 return mImpl->getPrePadding();
1227 }
1228
1241 void setPostPadding(Dims padding) noexcept
1242 {
1243 mImpl->setPostPadding(padding);
1244 }
1245
1251 Dims getPostPadding() const noexcept
1252 {
1253 return mImpl->getPostPadding();
1254 }
1255
1265 void setPaddingMode(PaddingMode paddingMode) noexcept
1266 {
1267 mImpl->setPaddingMode(paddingMode);
1268 }
1269
1278 {
1279 return mImpl->getPaddingMode();
1280 }
1281
1290 void setKernelSizeNd(Dims kernelSize) noexcept
1291 {
1292 mImpl->setKernelSizeNd(kernelSize);
1293 }
1294
1300 Dims getKernelSizeNd() const noexcept
1301 {
1302 return mImpl->getKernelSizeNd();
1303 }
1304
1315 void setStrideNd(Dims stride) noexcept
1316 {
1317 mImpl->setStrideNd(stride);
1318 }
1319
1325 Dims getStrideNd() const noexcept
1326 {
1327 return mImpl->getStrideNd();
1328 }
1329
1343 void setPaddingNd(Dims padding) noexcept
1344 {
1345 mImpl->setPaddingNd(padding);
1346 }
1347
1355 Dims getPaddingNd() const noexcept
1356 {
1357 return mImpl->getPaddingNd();
1358 }
1359
1369 void setDilationNd(Dims dilation) noexcept
1370 {
1371 mImpl->setDilationNd(dilation);
1372 }
1373
1379 Dims getDilationNd() const noexcept
1380 {
1381 return mImpl->getDilationNd();
1382 }
1383
1405 using ILayer::setInput;
1406
1407protected:
1408 virtual ~IConvolutionLayer() noexcept = default;
1409 apiv::VConvolutionLayer* mImpl;
1410};
1411
1444{
1445public:
1453 void setNbOutputChannels(int32_t nbOutputs) noexcept
1454 {
1455 mImpl->setNbOutputChannels(nbOutputs);
1456 }
1457
1463 int32_t getNbOutputChannels() const noexcept
1464 {
1465 return mImpl->getNbOutputChannels();
1466 }
1467
1473 void setKernelWeights(Weights weights) noexcept
1474 {
1475 mImpl->setKernelWeights(weights);
1476 }
1477
1483 Weights getKernelWeights() const noexcept
1484 {
1485 return mImpl->getKernelWeights();
1486 }
1487
1495 void setBiasWeights(Weights weights) noexcept
1496 {
1497 mImpl->setBiasWeights(weights);
1498 }
1499
1505 Weights getBiasWeights() const noexcept
1506 {
1507 return mImpl->getBiasWeights();
1508 }
1509
1531 using ILayer::setInput;
1532
1533protected:
1534 virtual ~IFullyConnectedLayer() noexcept = default;
1535 apiv::VFullyConnectedLayer* mImpl;
1536};
1537
1552{
1553public:
1562 {
1563 mImpl->setActivationType(type);
1564 }
1565
1572 {
1573 return mImpl->getActivationType();
1574 }
1575
1586 void setAlpha(float alpha) noexcept
1587 {
1588 mImpl->setAlpha(alpha);
1589 }
1590
1600 void setBeta(float beta) noexcept
1601 {
1602 mImpl->setBeta(beta);
1603 }
1604
1609 float getAlpha() const noexcept
1610 {
1611 return mImpl->getAlpha();
1612 }
1613
1618 float getBeta() const noexcept
1619 {
1620 return mImpl->getBeta();
1621 }
1622
1623protected:
1624 virtual ~IActivationLayer() noexcept = default;
1625 apiv::VActivationLayer* mImpl;
1626};
1627
1633enum class PoolingType : int32_t
1634{
1635 kMAX = 0, // Maximum over elements
1636 kAVERAGE = 1, // Average over elements. If the tensor is padded, the count includes the padding
1637 kMAX_AVERAGE_BLEND = 2 // Blending between max and average pooling: (1-blendFactor)*maxPool + blendFactor*avgPool
1638};
1639
1640namespace impl
1641{
1647template <>
1649{
1650 static constexpr int32_t kVALUE = 3;
1651};
1652} // namespace impl
1653
1665class IPoolingLayer : public ILayer
1666{
1667public:
1675 void setPoolingType(PoolingType type) noexcept
1676 {
1677 mImpl->setPoolingType(type);
1678 }
1679
1686 {
1687 return mImpl->getPoolingType();
1688 }
1689
1699 TRT_DEPRECATED void setWindowSize(DimsHW windowSize) noexcept
1700 {
1701 mImpl->setWindowSize(windowSize);
1702 }
1703
1712 {
1713 return mImpl->getWindowSize();
1714 }
1715
1727 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
1728 {
1729 mImpl->setStride(stride);
1730 }
1731
1740 {
1741 return mImpl->getStride();
1742 }
1743
1755 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
1756 {
1757 mImpl->setPadding(padding);
1758 }
1759
1770 {
1771 return mImpl->getPadding();
1772 }
1773
1784 void setBlendFactor(float blendFactor) noexcept
1785 {
1786 mImpl->setBlendFactor(blendFactor);
1787 }
1788
1797 float getBlendFactor() const noexcept
1798 {
1799 return mImpl->getBlendFactor();
1800 }
1801
1814 void setAverageCountExcludesPadding(bool exclusive) noexcept
1815 {
1816 mImpl->setAverageCountExcludesPadding(exclusive);
1817 }
1818
1826 {
1827 return mImpl->getAverageCountExcludesPadding();
1828 }
1829
1843 void setPrePadding(Dims padding) noexcept
1844 {
1845 mImpl->setPrePadding(padding);
1846 }
1847
1853 Dims getPrePadding() const noexcept
1854 {
1855 return mImpl->getPrePadding();
1856 }
1857
1871 void setPostPadding(Dims padding) noexcept
1872 {
1873 mImpl->setPostPadding(padding);
1874 }
1875
1881 Dims getPostPadding() const noexcept
1882 {
1883 return mImpl->getPostPadding();
1884 }
1885
1894 void setPaddingMode(PaddingMode paddingMode) noexcept
1895 {
1896 mImpl->setPaddingMode(paddingMode);
1897 }
1898
1906 {
1907 return mImpl->getPaddingMode();
1908 }
1909
1918 void setWindowSizeNd(Dims windowSize) noexcept
1919 {
1920 mImpl->setWindowSizeNd(windowSize);
1921 }
1922
1928 Dims getWindowSizeNd() const noexcept
1929 {
1930 return mImpl->getWindowSizeNd();
1931 }
1932
1943 void setStrideNd(Dims stride) noexcept
1944 {
1945 mImpl->setStrideNd(stride);
1946 }
1947
1953 Dims getStrideNd() const noexcept
1954 {
1955 return mImpl->getStrideNd();
1956 }
1957
1972 void setPaddingNd(Dims padding) noexcept
1973 {
1974 mImpl->setPaddingNd(padding);
1975 }
1976
1984 Dims getPaddingNd() const noexcept
1985 {
1986 return mImpl->getPaddingNd();
1987 }
1988
1989protected:
1990 virtual ~IPoolingLayer() noexcept = default;
1991 apiv::VPoolingLayer* mImpl;
1992};
1993
2003class ILRNLayer : public ILayer
2004{
2005public:
2015 void setWindowSize(int32_t windowSize) noexcept
2016 {
2017 mImpl->setWindowSize(windowSize);
2018 }
2019
2025 int32_t getWindowSize() const noexcept
2026 {
2027 return mImpl->getWindowSize();
2028 }
2029
2036 void setAlpha(float alpha) noexcept
2037 {
2038 mImpl->setAlpha(alpha);
2039 }
2040
2046 float getAlpha() const noexcept
2047 {
2048 return mImpl->getAlpha();
2049 }
2050
2057 void setBeta(float beta) noexcept
2058 {
2059 mImpl->setBeta(beta);
2060 }
2061
2067 float getBeta() const noexcept
2068 {
2069 return mImpl->getBeta();
2070 }
2071
2078 void setK(float k) noexcept
2079 {
2080 mImpl->setK(k);
2081 }
2082
2088 float getK() const noexcept
2089 {
2090 return mImpl->getK();
2091 }
2092
2093protected:
2094 virtual ~ILRNLayer() noexcept = default;
2095 apiv::VLRNLayer* mImpl;
2096};
2097
2103enum class ScaleMode : int32_t
2104{
2105 kUNIFORM = 0,
2106 kCHANNEL = 1,
2107 kELEMENTWISE = 2
2108};
2109
2115template <>
2116constexpr inline int32_t EnumMax<ScaleMode>() noexcept
2117{
2118 return 3;
2119}
2120
2147class IScaleLayer : public ILayer
2148{
2149public:
2155 void setMode(ScaleMode mode) noexcept
2156 {
2157 mImpl->setMode(mode);
2158 }
2159
2165 ScaleMode getMode() const noexcept
2166 {
2167 return mImpl->getMode();
2168 }
2169
2175 void setShift(Weights shift) noexcept
2176 {
2177 mImpl->setShift(shift);
2178 }
2179
2185 Weights getShift() const noexcept
2186 {
2187 return mImpl->getShift();
2188 }
2189
2195 void setScale(Weights scale) noexcept
2196 {
2197 mImpl->setScale(scale);
2198 }
2199
2205 Weights getScale() const noexcept
2206 {
2207 return mImpl->getScale();
2208 }
2209
2215 void setPower(Weights power) noexcept
2216 {
2217 mImpl->setPower(power);
2218 }
2219
2225 Weights getPower() const noexcept
2226 {
2227 return mImpl->getPower();
2228 }
2229
2240 int32_t getChannelAxis() const noexcept
2241 {
2242 return mImpl->getChannelAxis();
2243 }
2244
2261 void setChannelAxis(int32_t channelAxis) noexcept
2262 {
2263 mImpl->setChannelAxis(channelAxis);
2264 }
2265
2266protected:
2267 virtual ~IScaleLayer() noexcept = default;
2268 apiv::VScaleLayer* mImpl;
2269};
2270
2292class ISoftMaxLayer : public ILayer
2293{
2294public:
2325 void setAxes(uint32_t axes) noexcept
2326 {
2327 mImpl->setAxes(axes);
2328 }
2329
2335 uint32_t getAxes() const noexcept
2336 {
2337 return mImpl->getAxes();
2338 }
2339
2340protected:
2341 virtual ~ISoftMaxLayer() noexcept = default;
2342 apiv::VSoftMaxLayer* mImpl;
2343};
2344
2358{
2359public:
2370 void setAxis(int32_t axis) noexcept
2371 {
2372 mImpl->setAxis(axis);
2373 }
2374
2380 int32_t getAxis() const noexcept
2381 {
2382 return mImpl->getAxis();
2383 }
2384
2385protected:
2386 virtual ~IConcatenationLayer() noexcept = default;
2387 apiv::VConcatenationLayer* mImpl;
2388};
2389
2398{
2399public:
2411 TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
2412 {
2413 mImpl->setKernelSize(kernelSize);
2414 }
2415
2424 {
2425 return mImpl->getKernelSize();
2426 }
2427
2435 void setNbOutputMaps(int32_t nbOutputMaps) noexcept
2436 {
2437 mImpl->setNbOutputMaps(nbOutputMaps);
2438 }
2439
2445 int32_t getNbOutputMaps() const noexcept
2446 {
2447 return mImpl->getNbOutputMaps();
2448 }
2449
2461 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
2462 {
2463 mImpl->setStride(stride);
2464 }
2465
2474 {
2475 return mImpl->getStride();
2476 }
2477
2493 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
2494 {
2495 mImpl->setPadding(padding);
2496 }
2497
2508 {
2509 return mImpl->getPadding();
2510 }
2511
2527 void setNbGroups(int32_t nbGroups) noexcept
2528 {
2529 mImpl->setNbGroups(nbGroups);
2530 }
2531
2537 int32_t getNbGroups() const noexcept
2538 {
2539 return mImpl->getNbGroups();
2540 }
2541
2551 void setKernelWeights(Weights weights) noexcept
2552 {
2553 mImpl->setKernelWeights(weights);
2554 }
2555
2561 Weights getKernelWeights() const noexcept
2562 {
2563 return mImpl->getKernelWeights();
2564 }
2565
2576 void setBiasWeights(Weights weights) noexcept
2577 {
2578 mImpl->setBiasWeights(weights);
2579 }
2580
2586 Weights getBiasWeights() const noexcept
2587 {
2588 return mImpl->getBiasWeights();
2589 }
2590
2604 void setPrePadding(Dims padding) noexcept
2605 {
2606 mImpl->setPrePadding(padding);
2607 }
2608
2614 Dims getPrePadding() const noexcept
2615 {
2616 return mImpl->getPrePadding();
2617 }
2618
2632 void setPostPadding(Dims padding) noexcept
2633 {
2634 mImpl->setPostPadding(padding);
2635 }
2636
2642 Dims getPostPadding() const noexcept
2643 {
2644 return mImpl->getPostPadding();
2645 }
2646
2656 void setPaddingMode(PaddingMode paddingMode) noexcept
2657 {
2658 mImpl->setPaddingMode(paddingMode);
2659 }
2660
2669 {
2670 return mImpl->getPaddingMode();
2671 }
2672
2683 void setKernelSizeNd(Dims kernelSize) noexcept
2684 {
2685 mImpl->setKernelSizeNd(kernelSize);
2686 }
2687
2693 Dims getKernelSizeNd() const noexcept
2694 {
2695 return mImpl->getKernelSizeNd();
2696 }
2697
2710 void setStrideNd(Dims stride) noexcept
2711 {
2712 mImpl->setStrideNd(stride);
2713 }
2714
2720 Dims getStrideNd() const noexcept
2721 {
2722 return mImpl->getStrideNd();
2723 }
2724
2738 void setPaddingNd(Dims padding) noexcept
2739 {
2740 mImpl->setPaddingNd(padding);
2741 }
2742
2750 Dims getPaddingNd() const noexcept
2751 {
2752 return mImpl->getPaddingNd();
2753 }
2754
2776 using ILayer::setInput;
2777
2784 void setDilationNd(Dims dilation) noexcept
2785 {
2786 mImpl->setDilationNd(dilation);
2787 }
2788
2794 Dims getDilationNd() const noexcept
2795 {
2796 return mImpl->getDilationNd();
2797 }
2798
2799protected:
2800 virtual ~IDeconvolutionLayer() noexcept = default;
2801 apiv::VDeconvolutionLayer* mImpl;
2802};
2803
2817enum class ElementWiseOperation : int32_t
2818{
2819 kSUM = 0,
2820 kPROD = 1,
2821 kMAX = 2,
2822 kMIN = 3,
2823 kSUB = 4,
2824 kDIV = 5,
2825 kPOW = 6,
2826 kFLOOR_DIV = 7,
2827 kAND = 8,
2828 kOR = 9,
2829 kXOR = 10,
2830 kEQUAL = 11,
2831 kGREATER = 12,
2832 kLESS = 13
2833};
2834
2835namespace impl
2836{
2842template <>
2844{
2845 static constexpr int32_t kVALUE = 14;
2846};
2847} // namespace impl
2848
2869{
2870public:
2881 {
2882 return mImpl->setOperation(op);
2883 }
2884
2893 {
2894 return mImpl->getOperation();
2895 }
2896
2897protected:
2898 apiv::VElementWiseLayer* mImpl;
2899 virtual ~IElementWiseLayer() noexcept = default;
2900};
2901
2907enum class GatherMode : int32_t
2908{
2909 kDEFAULT = 0,
2910 kELEMENT = 1,
2911 kND = 2
2912};
2913
2919template <>
2920constexpr inline int32_t EnumMax<GatherMode>() noexcept
2921{
2922 return 3;
2923}
2924
3003class IGatherLayer : public ILayer
3004{
3005public:
3015 void setGatherAxis(int32_t axis) noexcept
3016 {
3017 mImpl->setGatherAxis(axis);
3018 }
3019
3026 int32_t getGatherAxis() const noexcept
3027 {
3028 return mImpl->getGatherAxis();
3029 }
3030
3047 void setNbElementWiseDims(int32_t elementWiseDims) noexcept
3048 {
3049 mImpl->setNbElementWiseDims(elementWiseDims);
3050 }
3051
3057 int32_t getNbElementWiseDims() const noexcept
3058 {
3059 return mImpl->getNbElementWiseDims();
3060 }
3061
3067 void setMode(GatherMode mode) noexcept
3068 {
3069 mImpl->setMode(mode);
3070 }
3071
3077 GatherMode getMode() const noexcept
3078 {
3079 return mImpl->getMode();
3080 }
3081
3082protected:
3083 apiv::VGatherLayer* mImpl;
3084 virtual ~IGatherLayer() noexcept = default;
3085};
3086
3166enum class RNNOperation : int32_t
3167{
3168 kRELU = 0,
3169 kTANH = 1,
3170 kLSTM = 2,
3171 kGRU = 3
3172};
3173
3179template <>
3180constexpr inline int32_t EnumMax<RNNOperation>() noexcept
3181{
3182 return 4;
3183}
3184
3192enum class RNNDirection : int32_t
3193{
3194 kUNIDIRECTION = 0,
3195 kBIDIRECTION = 1
3196};
3197
3203template <>
3204constexpr inline int32_t EnumMax<RNNDirection>() noexcept
3205{
3206 return 2;
3207}
3208
3224enum class RNNInputMode : int32_t
3225{
3226 kLINEAR = 0,
3227 kSKIP = 1
3228};
3229
3235template <>
3236constexpr inline int32_t EnumMax<RNNInputMode>() noexcept
3237{
3238 return 2;
3239}
3240
3248enum class RNNGateType : int32_t
3249{
3250 kINPUT = 0,
3251 kOUTPUT = 1,
3252 kFORGET = 2,
3253 kUPDATE = 3,
3254 kRESET = 4,
3255 kCELL = 5,
3256 kHIDDEN = 6
3257};
3258
3264template <>
3265constexpr inline int32_t EnumMax<RNNGateType>() noexcept
3266{
3267 return 7;
3268}
3269
3282{
3283public:
3284 int32_t getLayerCount() const noexcept
3285 {
3286 return mImpl->getLayerCount();
3287 }
3288 int32_t getHiddenSize() const noexcept
3289 {
3290 return mImpl->getHiddenSize();
3291 }
3292 int32_t getMaxSeqLength() const noexcept
3293 {
3294 return mImpl->getMaxSeqLength();
3295 }
3296 int32_t getDataLength() const noexcept
3297 {
3298 return mImpl->getDataLength();
3299 }
3300
3315 void setSequenceLengths(ITensor& seqLengths) noexcept
3316 {
3317 return mImpl->setSequenceLengths(seqLengths);
3318 }
3319
3327 ITensor* getSequenceLengths() const noexcept
3328 {
3329 return mImpl->getSequenceLengths();
3330 }
3331
3337 void setOperation(RNNOperation op) noexcept
3338 {
3339 mImpl->setOperation(op);
3340 }
3341
3348 {
3349 return mImpl->getOperation();
3350 }
3351
3357 void setInputMode(RNNInputMode op) noexcept
3358 {
3359 mImpl->setInputMode(op);
3360 }
3361
3368 {
3369 return mImpl->getInputMode();
3370 }
3371
3382 void setDirection(RNNDirection op) noexcept
3383 {
3384 mImpl->setDirection(op);
3385 }
3386
3393 {
3394 return mImpl->getDirection();
3395 }
3396
3451 void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept
3452 {
3453 mImpl->setWeightsForGate(layerIndex, gate, isW, weights);
3454 }
3455
3461 Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
3462 {
3463 return mImpl->getWeightsForGate(layerIndex, gate, isW);
3464 }
3465
3486 void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept
3487 {
3488 mImpl->setBiasForGate(layerIndex, gate, isW, bias);
3489 }
3490
3496 Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
3497 {
3498 return mImpl->getBiasForGate(layerIndex, gate, isW);
3499 }
3500
3513 void setHiddenState(ITensor& hidden) noexcept
3514 {
3515 mImpl->setHiddenState(hidden);
3516 }
3517
3523 ITensor* getHiddenState() const noexcept
3524 {
3525 return mImpl->getHiddenState();
3526 }
3527
3542 void setCellState(ITensor& cell) noexcept
3543 {
3544 mImpl->setCellState(cell);
3545 }
3546
3552 ITensor* getCellState() const noexcept
3553 {
3554 return mImpl->getCellState();
3555 }
3556
3557protected:
3558 apiv::VRNNv2Layer* mImpl;
3559 virtual ~IRNNv2Layer() noexcept = default;
3560};
3561
3572{
3573public:
3580 {
3581 return mImpl->getPlugin();
3582 }
3583
3584protected:
3585 apiv::VPluginV2Layer* mImpl;
3586 virtual ~IPluginV2Layer() noexcept = default;
3587};
3588
3602enum class UnaryOperation : int32_t
3603{
3604 kEXP = 0,
3605 kLOG = 1,
3606 kSQRT = 2,
3607 kRECIP = 3,
3608 kABS = 4,
3609 kNEG = 5,
3610 kSIN = 6,
3611 kCOS = 7,
3612 kTAN = 8,
3613 kSINH = 9,
3614 kCOSH = 10,
3615 kASIN = 11,
3616 kACOS = 12,
3617 kATAN = 13,
3618 kASINH = 14,
3619 kACOSH = 15,
3620 kATANH = 16,
3621 kCEIL = 17,
3622 kFLOOR = 18,
3623 kERF = 19,
3624 kNOT = 20,
3625 kSIGN = 21,
3626 kROUND = 22
3627};
3628
3634template <>
3635constexpr inline int32_t EnumMax<UnaryOperation>() noexcept
3636{
3637 return 23;
3638}
3639
3647class IUnaryLayer : public ILayer
3648{
3649public:
3658 {
3659 mImpl->setOperation(op);
3660 }
3661
3668 {
3669 return mImpl->getOperation();
3670 }
3671
3672protected:
3673 apiv::VUnaryLayer* mImpl;
3674 virtual ~IUnaryLayer() noexcept = default;
3675};
3676
3695enum class ReduceOperation : int32_t
3696{
3697 kSUM = 0,
3698 kPROD = 1,
3699 kMAX = 2,
3700 kMIN = 3,
3701 kAVG = 4
3702};
3703
3709template <>
3710constexpr inline int32_t EnumMax<ReduceOperation>() noexcept
3711{
3712 return 5;
3713}
3714
3722class IReduceLayer : public ILayer
3723{
3724public:
3731 {
3732 mImpl->setOperation(op);
3733 }
3734
3741 {
3742 return mImpl->getOperation();
3743 }
3744
3750 void setReduceAxes(uint32_t reduceAxes) noexcept
3751 {
3752 mImpl->setReduceAxes(reduceAxes);
3753 }
3754
3760 uint32_t getReduceAxes() const noexcept
3761 {
3762 return mImpl->getReduceAxes();
3763 }
3764
3770 void setKeepDimensions(bool keepDimensions) noexcept
3771 {
3772 mImpl->setKeepDimensions(keepDimensions);
3773 }
3774
3780 bool getKeepDimensions() const noexcept
3781 {
3782 return mImpl->getKeepDimensions();
3783 }
3784
3785protected:
3786 apiv::VReduceLayer* mImpl;
3787 virtual ~IReduceLayer() noexcept = default;
3788};
3789
3800class IPaddingLayer : public ILayer
3801{
3802public:
3812 TRT_DEPRECATED void setPrePadding(DimsHW padding) noexcept
3813 {
3814 mImpl->setPrePadding(padding);
3815 }
3816
3825 {
3826 return mImpl->getPrePadding();
3827 }
3828
3838 TRT_DEPRECATED void setPostPadding(DimsHW padding) noexcept
3839 {
3840 mImpl->setPostPadding(padding);
3841 }
3842
3851 {
3852 return mImpl->getPostPadding();
3853 }
3854
3864 void setPrePaddingNd(Dims padding) noexcept
3865 {
3866 mImpl->setPrePaddingNd(padding);
3867 }
3868
3876 Dims getPrePaddingNd() const noexcept
3877 {
3878 return mImpl->getPrePaddingNd();
3879 }
3880
3890 void setPostPaddingNd(Dims padding) noexcept
3891 {
3892 mImpl->setPostPaddingNd(padding);
3893 }
3894
3902 Dims getPostPaddingNd() const noexcept
3903 {
3904 return mImpl->getPostPaddingNd();
3905 }
3906
3907protected:
3908 apiv::VPaddingLayer* mImpl;
3909 virtual ~IPaddingLayer() noexcept = default;
3910};
3911
3913{
3920 int32_t order[Dims::MAX_DIMS];
3921};
3922
3935class IShuffleLayer : public ILayer
3936{
3937public:
3947 void setFirstTranspose(Permutation permutation) noexcept
3948 {
3949 mImpl->setFirstTranspose(permutation);
3950 }
3951
3960 {
3961 return mImpl->getFirstTranspose();
3962 }
3963
3984 void setReshapeDimensions(Dims dimensions) noexcept
3985 {
3986 mImpl->setReshapeDimensions(dimensions);
3987 }
3988
3998 {
3999 return mImpl->getReshapeDimensions();
4000 }
4001
4007 //
4030 using ILayer::setInput;
4031
4044 void setSecondTranspose(Permutation permutation) noexcept
4045 {
4046 mImpl->setSecondTranspose(permutation);
4047 }
4048
4057 {
4058 return mImpl->getSecondTranspose();
4059 }
4060
4072 void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
4073 {
4074 return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);
4075 }
4076
4085 bool getZeroIsPlaceholder() const noexcept
4086 {
4087 return mImpl->getZeroIsPlaceholder();
4088 }
4089
4090protected:
4091 apiv::VShuffleLayer* mImpl;
4092 virtual ~IShuffleLayer() noexcept = default;
4093};
4094
4100enum class SliceMode : int32_t
4101{
4102 kDEFAULT = 0,
4103 kWRAP = 1,
4104 kCLAMP = 2,
4105 kFILL = 3,
4106 kREFLECT = 4,
4109};
4110
4116template <>
4117constexpr inline int32_t EnumMax<SliceMode>() noexcept
4118{
4119 return 5;
4120}
4121
4164class ISliceLayer : public ILayer
4165{
4166public:
4176 void setStart(Dims start) noexcept
4177 {
4178 mImpl->setStart(start);
4179 }
4180
4191 Dims getStart() const noexcept
4192 {
4193 return mImpl->getStart();
4194 }
4195
4205 void setSize(Dims size) noexcept
4206 {
4207 return mImpl->setSize(size);
4208 }
4209
4220 Dims getSize() const noexcept
4221 {
4222 return mImpl->getSize();
4223 }
4224
4234 void setStride(Dims stride) noexcept
4235 {
4236 mImpl->setStride(stride);
4237 }
4238
4249 Dims getStride() const noexcept
4250 {
4251 return mImpl->getStride();
4252 }
4253
4259 void setMode(SliceMode mode) noexcept
4260 {
4261 mImpl->setMode(mode);
4262 }
4263
4269 SliceMode getMode() const noexcept
4270 {
4271 return mImpl->getMode();
4272 }
4273
4295 using ILayer::setInput;
4296
4297protected:
4298 apiv::VSliceLayer* mImpl;
4299 virtual ~ISliceLayer() noexcept = default;
4300};
4301
4314class IShapeLayer : public ILayer
4315{
4316protected:
4317 apiv::VShapeLayer* mImpl;
4318 virtual ~IShapeLayer() noexcept = default;
4319};
4320
4326enum class TopKOperation : int32_t
4327{
4328 kMAX = 0,
4329 kMIN = 1,
4330};
4331
4337template <>
4338constexpr inline int32_t EnumMax<TopKOperation>() noexcept
4339{
4340 return 2;
4341}
4342
4350class ITopKLayer : public ILayer
4351{
4352public:
4358 void setOperation(TopKOperation op) noexcept
4359 {
4360 mImpl->setOperation(op);
4361 }
4362
4369 {
4370 return mImpl->getOperation();
4371 }
4372
4380 void setK(int32_t k) noexcept
4381 {
4382 mImpl->setK(k);
4383 }
4384
4390 int32_t getK() const noexcept
4391 {
4392 return mImpl->getK();
4393 }
4394
4400 void setReduceAxes(uint32_t reduceAxes) noexcept
4401 {
4402 mImpl->setReduceAxes(reduceAxes);
4403 }
4404
4410 uint32_t getReduceAxes() const noexcept
4411 {
4412 return mImpl->getReduceAxes();
4413 }
4414
4415protected:
4416 apiv::VTopKLayer* mImpl;
4417 virtual ~ITopKLayer() noexcept = default;
4418};
4419
4426enum class MatrixOperation : int32_t
4427{
4431 kNONE,
4432
4434 kTRANSPOSE,
4435
4446 kVECTOR
4447};
4448
4454template <>
4455constexpr inline int32_t EnumMax<MatrixOperation>() noexcept
4456{
4457 return 3;
4458}
4459
4486{
4487public:
4494 void setOperation(int32_t index, MatrixOperation op) noexcept
4495 {
4496 mImpl->setOperation(index, op);
4497 }
4498
4506 MatrixOperation getOperation(int32_t index) const noexcept
4507 {
4508 return mImpl->getOperation(index);
4509 }
4510
4511protected:
4512 apiv::VMatrixMultiplyLayer* mImpl;
4513 virtual ~IMatrixMultiplyLayer() noexcept = default;
4514};
4515
4531{
4532protected:
4533 apiv::VRaggedSoftMaxLayer* mImpl;
4534 virtual ~IRaggedSoftMaxLayer() noexcept = default;
4535};
4536
4561{
4562protected:
4563 apiv::VIdentityLayer* mImpl;
4564 virtual ~IIdentityLayer() noexcept = default;
4565};
4566
4576{
4577public:
4587 void setWeights(Weights weights) noexcept
4588 {
4589 mImpl->setWeights(weights);
4590 }
4591
4597 Weights getWeights() const noexcept
4598 {
4599 return mImpl->getWeights();
4600 }
4601
4609 void setDimensions(Dims dimensions) noexcept
4610 {
4611 mImpl->setDimensions(dimensions);
4612 }
4613
4621 Dims getDimensions() const noexcept
4622 {
4623 return mImpl->getDimensions();
4624 }
4625
4626protected:
4627 apiv::VConstantLayer* mImpl;
4628 virtual ~IConstantLayer() noexcept = default;
4629};
4630
4641{
4642protected:
4643 apiv::VParametricReLULayer* mImpl;
4644 virtual ~IParametricReLULayer() noexcept = default;
4645};
4646
4652enum class ResizeMode : int32_t
4653{
4654 kNEAREST = 0,
4655 kLINEAR = 1
4656};
4657
4658namespace impl
4659{
4665template <>
4667{
4668 static constexpr int32_t kVALUE = 2;
4669};
4670} // namespace impl
4671
4680{
4693 kALIGN_CORNERS = 0,
4694
4701 kASYMMETRIC = 1,
4702
4709 kHALF_PIXEL = 2,
4710};
4711
4712namespace impl
4713{
4719template <>
4721{
4722 static constexpr int32_t kVALUE = 3;
4723};
4724} // namespace impl
4725
4733enum class ResizeSelector : int32_t
4734{
4736 kFORMULA = 0,
4737
4739 kUPPER = 1,
4740};
4741
4742namespace impl
4743{
4749template <>
4751{
4752 static constexpr int32_t kVALUE = 2;
4753};
4754} // namespace impl
4755
4763enum class ResizeRoundMode : int32_t
4764{
4766 kHALF_UP = 0,
4767
4769 kHALF_DOWN = 1,
4770
4772 kFLOOR = 2,
4773
4775 kCEIL = 3,
4776};
4777
4778namespace impl
4779{
4785template <>
4787{
4788 static constexpr int32_t kVALUE = 4;
4789};
4790} // namespace impl
4791
4828class IResizeLayer : public ILayer
4829{
4830public:
4849 void setOutputDimensions(Dims dimensions) noexcept
4850 {
4851 return mImpl->setOutputDimensions(dimensions);
4852 }
4853
4859 Dims getOutputDimensions() const noexcept
4860 {
4861 return mImpl->getOutputDimensions();
4862 }
4863
4889 void setScales(float const* scales, int32_t nbScales) noexcept
4890 {
4891 mImpl->setScales(scales, nbScales);
4892 }
4893
4908 int32_t getScales(int32_t size, float* scales) const noexcept
4909 {
4910 return mImpl->getScales(size, scales);
4911 }
4912
4920 void setResizeMode(ResizeMode resizeMode) noexcept
4921 {
4922 mImpl->setResizeMode(resizeMode);
4923 }
4924
4930 ResizeMode getResizeMode() const noexcept
4931 {
4932 return mImpl->getResizeMode();
4933 }
4934
4946 TRT_DEPRECATED void setAlignCorners(bool alignCorners) noexcept
4947 {
4948 mImpl->setAlignCorners(alignCorners);
4949 }
4950
4958 TRT_DEPRECATED bool getAlignCorners() const noexcept
4959 {
4960 return mImpl->getAlignCorners();
4961 }
4962
4982 using ILayer::setInput;
4983
4994 {
4995 mImpl->setCoordinateTransformation(coordTransform);
4996 }
4997
5004 {
5005 return mImpl->getCoordinateTransformation();
5006 }
5007
5019 {
5020 mImpl->setSelectorForSinglePixel(selector);
5021 }
5022
5029 {
5030 return mImpl->getSelectorForSinglePixel();
5031 }
5032
5043 {
5044 mImpl->setNearestRounding(value);
5045 }
5046
5053 {
5054 return mImpl->getNearestRounding();
5055 }
5056
5057protected:
5058 virtual ~IResizeLayer() noexcept = default;
5059 apiv::VResizeLayer* mImpl;
5060};
5061
5063enum class LoopOutput : int32_t
5064{
5066 kLAST_VALUE = 0,
5067
5069 kCONCATENATE = 1,
5070
5072 kREVERSE = 2
5073};
5074
5080template <>
5081constexpr inline int32_t EnumMax<LoopOutput>() noexcept
5082{
5083 return 3;
5084}
5085
5087enum class TripLimit : int32_t
5088{
5089
5090 kCOUNT = 0,
5091 kWHILE = 1
5092};
5093
5099template <>
5100constexpr inline int32_t EnumMax<TripLimit>() noexcept
5101{
5102 return 2;
5103}
5104
5105class ILoop;
5106
5108{
5109public:
5111 ILoop* getLoop() const noexcept
5112 {
5113 return mBoundary->getLoop();
5114 }
5115
5116protected:
5117 virtual ~ILoopBoundaryLayer() noexcept = default;
5118 apiv::VLoopBoundaryLayer* mBoundary;
5119};
5120
5127{
5128public:
5131 {
5132 return mBoundary->getConditional();
5133 }
5134
5135protected:
5136 virtual ~IIfConditionalBoundaryLayer() noexcept = default;
5137 apiv::VConditionalBoundaryLayer* mBoundary;
5138};
5139
5144{
5145public:
5146protected:
5147 virtual ~IConditionLayer() noexcept = default;
5148 apiv::VConditionLayer* mImpl;
5149};
5150
5157{
5158public:
5159protected:
5160 virtual ~IIfConditionalOutputLayer() noexcept = default;
5161 apiv::VConditionalOutputLayer* mImpl;
5162};
5163
5168{
5169public:
5170protected:
5171 virtual ~IIfConditionalInputLayer() noexcept = default;
5172 apiv::VConditionalInputLayer* mImpl;
5173};
5174
5197{
5198public:
5209 {
5210 return mImpl->setCondition(condition);
5211 }
5212
5224 IIfConditionalOutputLayer* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput) noexcept
5225 {
5226 return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);
5227 }
5228
5237 {
5238 return mImpl->addInput(input);
5239 }
5240
5249 void setName(char const* name) noexcept
5250 {
5251 mImpl->setName(name);
5252 }
5253
5259 char const* getName() const noexcept
5260 {
5261 return mImpl->getName();
5262 }
5263
5264protected:
5265 virtual ~IIfConditional() noexcept = default;
5266 apiv::VIfConditional* mImpl;
5267};
5268
5269
5271{
5272public:
5278 //
5291 using ILayer::setInput;
5292
5293protected:
5294 virtual ~IRecurrenceLayer() noexcept = default;
5295 apiv::VRecurrenceLayer* mImpl;
5296};
5297
5316{
5317public:
5318 LoopOutput getLoopOutput() const noexcept
5319 {
5320 return mImpl->getLoopOutput();
5321 }
5322
5335 void setAxis(int32_t axis) noexcept
5336 {
5337 mImpl->setAxis(axis);
5338 }
5339
5341 int32_t getAxis() const noexcept
5342 {
5343 return mImpl->getAxis();
5344 }
5345
5351 //
5366 using ILayer::setInput;
5367
5368protected:
5369 virtual ~ILoopOutputLayer() noexcept = default;
5370 apiv::VLoopOutputLayer* mImpl;
5371};
5372
5374{
5375public:
5376 TripLimit getTripLimit() const noexcept
5377 {
5378 return mImpl->getTripLimit();
5379 }
5380
5381protected:
5382 virtual ~ITripLimitLayer() noexcept = default;
5383 apiv::VTripLimitLayer* mImpl;
5384};
5385
5387{
5388public:
5390 void setAxis(int32_t axis) noexcept
5391 {
5392 mImpl->setAxis(axis);
5393 }
5394
5396 int32_t getAxis() const noexcept
5397 {
5398 return mImpl->getAxis();
5399 }
5400
5406 void setReverse(bool reverse) noexcept
5407 {
5408 mImpl->setReverse(reverse);
5409 }
5410
5412 bool getReverse() const noexcept
5413 {
5414 return mImpl->getReverse();
5415 }
5416
5417protected:
5418 virtual ~IIteratorLayer() noexcept = default;
5419 apiv::VIteratorLayer* mImpl;
5420};
5421
5427class ILoop : public INoCopy
5428{
5429public:
5436 IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept
5437 {
5438 return mImpl->addRecurrence(initialValue);
5439 }
5440
5458 {
5459 return mImpl->addTripLimit(tensor, limit);
5460 }
5461
5470 IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept
5471 {
5472 return mImpl->addIterator(tensor, axis, reverse);
5473 }
5474
5482 ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept
5483 {
5484 return mImpl->addLoopOutput(tensor, outputKind, axis);
5485 }
5486
5495 void setName(char const* name) noexcept
5496 {
5497 mImpl->setName(name);
5498 }
5499
5505 char const* getName() const noexcept
5506 {
5507 return mImpl->getName();
5508 }
5509
5510protected:
5511 virtual ~ILoop() noexcept = default;
5512 apiv::VLoop* mImpl;
5513};
5514
5518class ISelectLayer : public ILayer
5519{
5520protected:
5521 virtual ~ISelectLayer() noexcept = default;
5522 apiv::VSelectLayer* mImpl;
5523};
5524
5540{
5541public:
5550 void setMessage(char const* message) noexcept
5551 {
5552 mImpl->setMessage(message);
5553 }
5554
5560 char const* getMessage() const noexcept
5561 {
5562 return mImpl->getMessage();
5563 }
5564
5565protected:
5566 virtual ~IAssertionLayer() noexcept = default;
5567
5568 apiv::VAssertionLayer* mImpl;
5569};
5570
5578enum class FillOperation : int32_t
5579{
5580 kLINSPACE = 0,
5581 kRANDOM_UNIFORM = 1
5582};
5583
5589template <>
5590constexpr inline int32_t EnumMax<FillOperation>() noexcept
5591{
5592 return 2;
5593}
5594
5620class IFillLayer : public ILayer
5621{
5622public:
5631 //
5632 void setDimensions(Dims dimensions) noexcept
5633 {
5634 mImpl->setDimensions(dimensions);
5635 }
5636
5647 Dims getDimensions() const noexcept
5648 {
5649 return mImpl->getDimensions();
5650 }
5651
5657 void setOperation(FillOperation op) noexcept
5658 {
5659 mImpl->setOperation(op);
5660 }
5661
5668 {
5669 return mImpl->getOperation();
5670 }
5671
5684 //
5685 void setAlpha(double alpha) noexcept
5686 {
5687 mImpl->setAlpha(alpha);
5688 }
5689
5700 double getAlpha() const noexcept
5701 {
5702 return mImpl->getAlpha();
5703 }
5704
5718 void setBeta(double beta) noexcept
5719 {
5720 mImpl->setBeta(beta);
5721 }
5722
5733 double getBeta() const noexcept
5734 {
5735 return mImpl->getBeta();
5736 }
5737
5764 using ILayer::setInput;
5765
5766protected:
5767 virtual ~IFillLayer() noexcept = default;
5768 apiv::VFillLayer* mImpl;
5769};
5770
5829{
5830public:
5839 int32_t getAxis() const noexcept
5840 {
5841 return mImpl->getAxis();
5842 }
5850 void setAxis(int32_t axis) noexcept
5851 {
5852 mImpl->setAxis(axis);
5853 }
5854
5855protected:
5856 virtual ~IQuantizeLayer() noexcept = default;
5857 apiv::VQuantizeLayer* mImpl;
5858};
5859
5916{
5917public:
5926 int32_t getAxis() const noexcept
5927 {
5928 return mImpl->getAxis();
5929 }
5937 void setAxis(int32_t axis) noexcept
5938 {
5939 mImpl->setAxis(axis);
5940 }
5941
5942protected:
5943 virtual ~IDequantizeLayer() noexcept = default;
5944 apiv::VDequantizeLayer* mImpl;
5945};
5946
5982class IEinsumLayer : public ILayer
5983{
5984public:
5994 bool setEquation(char const* equation) noexcept
5995 {
5996 return mImpl->setEquation(equation);
5997 }
5998
6004 char const* getEquation() const noexcept
6005 {
6006 return mImpl->getEquation();
6007 }
6008
6009protected:
6010 virtual ~IEinsumLayer() noexcept = default;
6011 apiv::VEinsumLayer* mImpl;
6012};
6013
6019enum class ScatterMode : int32_t
6020{
6021 kELEMENT = 0,
6022 kND = 1,
6023};
6024
6030template <>
6031constexpr inline int32_t EnumMax<ScatterMode>() noexcept
6032{
6033 return 2;
6034}
6035
6092class IScatterLayer : public ILayer
6093{
6094public:
6100 void setMode(ScatterMode mode) noexcept
6101 {
6102 mImpl->setMode(mode);
6103 }
6104
6110 ScatterMode getMode() const noexcept
6111 {
6112 return mImpl->getMode();
6113 }
6114
6120 void setAxis(int32_t axis) noexcept
6121 {
6122 mImpl->setAxis(axis);
6123 }
6124
6128 int32_t getAxis() const noexcept
6129 {
6130 return mImpl->getAxis();
6131 }
6132
6133protected:
6134 apiv::VScatterLayer* mImpl;
6135 virtual ~IScatterLayer() noexcept = default;
6136}; // class IScatterLayer
6137
6158{
6159public:
6160 virtual ~INetworkDefinition() noexcept = default;
6161
6197 ITensor* addInput(char const* name, DataType type, Dims dimensions) noexcept
6198 {
6199 return mImpl->addInput(name, type, dimensions);
6200 }
6201
6211 void markOutput(ITensor& tensor) noexcept
6212 {
6213 mImpl->markOutput(tensor);
6214 }
6215
6235 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6236 {
6237 return mImpl->addConvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6238 }
6239
6258 ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
6259 {
6260 return mImpl->addFullyConnected(input, nbOutputs, kernelWeights, biasWeights);
6261 }
6262
6278 {
6279 return mImpl->addActivation(input, type);
6280 }
6281
6297 {
6298 return mImpl->addPooling(input, type, windowSize);
6299 }
6300
6315 ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept
6316 {
6317 return mImpl->addLRN(input, window, alpha, beta, k);
6318 }
6319
6342 IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
6343 {
6344 return mImpl->addScale(input, mode, shift, scale, power);
6345 }
6346
6356 {
6357 return mImpl->addSoftMax(input);
6358 }
6359
6372 IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept
6373 {
6374 return mImpl->addConcatenation(inputs, nbInputs);
6375 }
6376
6396 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6397 {
6398 return mImpl->addDeconvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6399 }
6400
6424 {
6425 return mImpl->addElementWise(input1, input2, op);
6426 }
6427
6445 IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept
6446 {
6447 return mImpl->addUnary(input, operation);
6448 }
6449
6462 TRT_DEPRECATED IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept
6463 {
6464 return mImpl->addPadding(input, prePadding, postPadding);
6465 }
6466
6477 {
6478 return mImpl->addShuffle(input);
6479 }
6480
6488 int32_t getNbLayers() const noexcept
6489 {
6490 return mImpl->getNbLayers();
6491 }
6492
6502 ILayer* getLayer(int32_t index) const noexcept
6503 {
6504 return mImpl->getLayer(index);
6505 }
6506
6514 int32_t getNbInputs() const noexcept
6515 {
6516 return mImpl->getNbInputs();
6517 }
6518
6530 ITensor* getInput(int32_t index) const noexcept
6531 {
6532 return mImpl->getInput(index);
6533 }
6534
6544 int32_t getNbOutputs() const noexcept
6545 {
6546 return mImpl->getNbOutputs();
6547 }
6548
6560 ITensor* getOutput(int32_t index) const noexcept
6561 {
6562 return mImpl->getOutput(index);
6563 }
6564
6572 TRT_DEPRECATED void destroy() noexcept
6573 {
6574 delete this;
6575 }
6576
6600 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
6601 {
6602 return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);
6603 }
6604
6633 ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
6634 {
6635 return mImpl->addTopK(input, op, k, reduceAxes);
6636 }
6637
6649 IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept
6650 {
6651 return mImpl->addGather(data, indices, axis);
6652 }
6653
6666 {
6667 return mImpl->addGatherV2(data, indices, mode);
6668 }
6669
6684 {
6685 return mImpl->addRaggedSoftMax(input, bounds);
6686 }
6687
6705 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
6706 {
6707 return mImpl->addMatrixMultiply(input0, op0, input1, op1);
6708 }
6709
6730 IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept
6731 {
6732 return mImpl->addConstant(dimensions, weights);
6733 }
6734
6800 ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
6801 {
6802 return mImpl->addRNNv2(input, layerCount, hiddenSize, maxSeqLen, op);
6803 }
6804
6815 {
6816 return mImpl->addIdentity(input);
6817 }
6818
6829 void removeTensor(ITensor& tensor) noexcept
6830 {
6831 mImpl->removeTensor(tensor);
6832 }
6833
6841 void unmarkOutput(ITensor& tensor) noexcept
6842 {
6843 mImpl->unmarkOutput(tensor);
6844 }
6845
6860 IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept
6861 {
6862 return mImpl->addPluginV2(inputs, nbInputs, plugin);
6863 }
6864
6879 ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept
6880 {
6881 return mImpl->addSlice(input, start, size, stride);
6882 }
6883
6901 void setName(char const* name) noexcept
6902 {
6903 mImpl->setName(name);
6904 }
6905
6915 char const* getName() const noexcept
6916 {
6917 return mImpl->getName();
6918 }
6919
6933 IShapeLayer* addShape(ITensor& input) noexcept
6934 {
6935 return mImpl->addShape(input);
6936 }
6937
6951 bool hasImplicitBatchDimension() const noexcept
6952 {
6953 return mImpl->hasImplicitBatchDimension();
6954 }
6955
6969 bool markOutputForShapes(ITensor& tensor) noexcept
6970 {
6971 return mImpl->markOutputForShapes(tensor);
6972 }
6973
6981 bool unmarkOutputForShapes(ITensor& tensor) noexcept
6982 {
6983 return mImpl->unmarkOutputForShapes(tensor);
6984 }
6985
7000 {
7001 return mImpl->addParametricReLU(input, slope);
7002 }
7003
7022 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7023 {
7024 return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7025 }
7026
7041 IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept
7042 {
7043 return mImpl->addPoolingNd(input, type, windowSize);
7044 }
7045
7060 //
7064 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7065 {
7066 return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7067 }
7068
7100 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
7101 {
7102 return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);
7103 }
7104
7116 {
7117 return mImpl->addResize(input);
7118 }
7119
7133 {
7134 return mImpl->hasExplicitPrecision();
7135 }
7136
7148 ILoop* addLoop() noexcept
7149 {
7150 return mImpl->addLoop();
7151 }
7152
7188 ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept
7189 {
7190 return mImpl->addSelect(condition, thenInput, elseInput);
7191 }
7192
7205 IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept
7206 {
7207 return mImpl->addAssertion(condition, message);
7208 }
7209
7223 IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept
7224 {
7225 return mImpl->addFill(dimensions, op);
7226 }
7227
7240 TRT_DEPRECATED IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept
7241 {
7242 return mImpl->addPaddingNd(input, prePadding, postPadding);
7243 }
7244
7260 bool setWeightsName(Weights weights, char const* name) noexcept
7261 {
7262 return mImpl->setWeightsName(weights, name);
7263 }
7264
7276 //
7279 void setErrorRecorder(IErrorRecorder* recorder) noexcept
7280 {
7281 mImpl->setErrorRecorder(recorder);
7282 }
7283
7295 {
7296 return mImpl->getErrorRecorder();
7297 }
7298
7314 {
7315 return mImpl->addDequantize(input, scale);
7316 }
7317
7333 IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept
7334 {
7335 return mImpl->addScatter(data, indices, updates, mode);
7336 }
7337
7352 IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept
7353 {
7354 return mImpl->addQuantize(input, scale);
7355 }
7356
7368 {
7369 return mImpl->addIfConditional();
7370 }
7371
7381 IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept
7382 {
7383 return mImpl->addEinsum(inputs, nbInputs, equation);
7384 }
7385
7386protected:
7387 apiv::VNetworkDefinition* mImpl;
7388};
7389
7395enum class CalibrationAlgoType : int32_t
7396{
7401};
7402
7408template <>
7409constexpr inline int32_t EnumMax<CalibrationAlgoType>() noexcept
7410{
7411 return 4;
7412}
7413
7426{
7427public:
7433 virtual int32_t getBatchSize() const noexcept = 0;
7434
7448 virtual bool getBatch(void* bindings[], char const* names[], int32_t nbBindings) noexcept = 0;
7449
7464 virtual void const* readCalibrationCache(std::size_t& length) noexcept = 0;
7465
7474 virtual void writeCalibrationCache(void const* ptr, std::size_t length) noexcept = 0;
7475
7481 virtual CalibrationAlgoType getAlgorithm() noexcept = 0;
7482
7483 virtual ~IInt8Calibrator() noexcept = default;
7484};
7485
7491{
7492public:
7497 {
7499 }
7500
7501 virtual ~IInt8EntropyCalibrator() noexcept = default;
7502};
7503
7509{
7510public:
7515 {
7517 }
7518
7519 virtual ~IInt8EntropyCalibrator2() noexcept = default;
7520};
7521
7526{
7527public:
7532 {
7534 }
7535
7536 virtual ~IInt8MinMaxCalibrator() noexcept = default;
7537};
7538
7544{
7545public:
7550 {
7552 }
7553
7560 virtual double getQuantile() const noexcept = 0;
7561
7568 virtual double getRegressionCutoff() const noexcept = 0;
7569
7582 virtual void const* readHistogramCache(std::size_t& length) noexcept = 0;
7583
7592 virtual void writeHistogramCache(void const* ptr, std::size_t length) noexcept = 0;
7593
7594 virtual ~IInt8LegacyCalibrator() noexcept = default;
7595};
7596
7608{
7609public:
7614 {
7615 return mImpl->getTensorFormat();
7616 }
7617
7621 DataType getDataType() const noexcept
7622 {
7623 return mImpl->getDataType();
7624 }
7625
7629 Dims getStrides() const noexcept
7630 {
7631 return mImpl->getStrides();
7632 }
7633
7634protected:
7635 virtual ~IAlgorithmIOInfo() noexcept = default;
7636 apiv::VAlgorithmIOInfo* mImpl;
7637};
7638
7651{
7652public:
7656 int64_t getImplementation() const noexcept
7657 {
7658 return mImpl->getImplementation();
7659 }
7660
7664 int64_t getTactic() const noexcept
7665 {
7666 return mImpl->getTactic();
7667 }
7668
7669protected:
7670 virtual ~IAlgorithmVariant() noexcept = default;
7671 apiv::VAlgorithmVariant* mImpl;
7672};
7673
7683{
7684public:
7689 char const* getName() const noexcept
7690 {
7691 return mImpl->getName();
7692 }
7693
7700 Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
7701 {
7702 return mImpl->getDimensions(index, select);
7703 }
7704
7708 int32_t getNbInputs() const noexcept
7709 {
7710 return mImpl->getNbInputs();
7711 }
7712
7716 int32_t getNbOutputs() const noexcept
7717 {
7718 return mImpl->getNbOutputs();
7719 }
7720
7721protected:
7722 virtual ~IAlgorithmContext() noexcept = default;
7723 apiv::VAlgorithmContext* mImpl;
7724};
7725
7735class IAlgorithm : public INoCopy
7736{
7737public:
7748 TRT_DEPRECATED IAlgorithmIOInfo const& getAlgorithmIOInfo(int32_t index) const noexcept
7749 {
7750 return mImpl->getAlgorithmIOInfo(index);
7751 }
7752
7757 {
7758 return mImpl->getAlgorithmVariant();
7759 }
7760
7764 float getTimingMSec() const noexcept
7765 {
7766 return mImpl->getTimingMSec();
7767 }
7768
7772 std::size_t getWorkspaceSize() const noexcept
7773 {
7774 return mImpl->getWorkspaceSize();
7775 }
7776
7785 IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept
7786 {
7787 return mImpl->getAlgorithmIOInfoByIndex(index);
7788 }
7789
7790protected:
7791 virtual ~IAlgorithm() noexcept = default;
7792 apiv::VAlgorithm* mImpl;
7793}; // IAlgorithm
7794
7804{
7805public:
7820 virtual int32_t selectAlgorithms(IAlgorithmContext const& context, IAlgorithm const* const* choices,
7821 int32_t nbChoices, int32_t* selection) noexcept = 0;
7832 virtual void reportAlgorithms(IAlgorithmContext const* const* algoContexts, IAlgorithm const* const* algoChoices,
7833 int32_t nbAlgorithms) noexcept = 0;
7834
7835 virtual ~IAlgorithmSelector() noexcept = default;
7836};
7837
7844using QuantizationFlags = uint32_t;
7845
7853enum class QuantizationFlag : int32_t
7854{
7859};
7860
7866template <>
7867constexpr inline int32_t EnumMax<QuantizationFlag>() noexcept
7868{
7869 return 1;
7870}
7871
7878using BuilderFlags = uint32_t;
7879
7887enum class BuilderFlag : int32_t
7888{
7889 kFP16 = 0,
7890 kINT8 = 1,
7891 kDEBUG = 2,
7892 kGPU_FALLBACK = 3,
7893
7905
7906 kREFIT = 5,
7908
7912 kTF32 = 7,
7913
7915 kSPARSE_WEIGHTS = 8,
7916
7923 kSAFETY_SCOPE = 9,
7924
7927
7931
7935 kDIRECT_IO = 12,
7936
7939};
7940
7946template <>
7947constexpr inline int32_t EnumMax<BuilderFlag>() noexcept
7948{
7949 return 14;
7950}
7951
7962class ITimingCache : public INoCopy
7963{
7964public:
7965 virtual ~ITimingCache() noexcept = default;
7966
7976 nvinfer1::IHostMemory* serialize() const noexcept
7977 {
7978 return mImpl->serialize();
7979 }
7980
8000 bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept
8001 {
8002 return mImpl->combine(inputCache, ignoreMismatch);
8003 }
8004
8010 bool reset() noexcept
8011 {
8012 return mImpl->reset();
8013 }
8014
8015protected:
8016 apiv::VTimingCache* mImpl;
8017};
8018
8026enum class MemoryPoolType : int32_t
8027{
8035 kWORKSPACE = 0,
8036
8044
8050 kDLA_LOCAL_DRAM = 2,
8051
8057 kDLA_GLOBAL_DRAM = 3,
8058};
8059
8065template <>
8066constexpr inline int32_t EnumMax<MemoryPoolType>() noexcept
8067{
8068 return 4;
8069}
8070
8079{
8080public:
8081 virtual ~IBuilderConfig() noexcept = default;
8082
8095 TRT_DEPRECATED virtual void setMinTimingIterations(int32_t minTiming) noexcept
8096 {
8097 mImpl->setMinTimingIterations(minTiming);
8098 }
8099
8109 TRT_DEPRECATED virtual int32_t getMinTimingIterations() const noexcept
8110 {
8111 return mImpl->getMinTimingIterations();
8112 }
8113
8122 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
8123 {
8124 mImpl->setAvgTimingIterations(avgTiming);
8125 }
8126
8134 int32_t getAvgTimingIterations() const noexcept
8135 {
8136 return mImpl->getAvgTimingIterations();
8137 }
8138
8147 void setEngineCapability(EngineCapability capability) noexcept
8148 {
8149 mImpl->setEngineCapability(capability);
8150 }
8151
8160 {
8161 return mImpl->getEngineCapability();
8162 }
8163
8169 void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept
8170 {
8171 mImpl->setInt8Calibrator(calibrator);
8172 }
8173
8178 {
8179 return mImpl->getInt8Calibrator();
8180 }
8181
8192 TRT_DEPRECATED void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept
8193 {
8194 mImpl->setMaxWorkspaceSize(workspaceSize);
8195 }
8196
8209 TRT_DEPRECATED std::size_t getMaxWorkspaceSize() const noexcept
8210 {
8211 return mImpl->getMaxWorkspaceSize();
8212 }
8213
8226 void setFlags(BuilderFlags builderFlags) noexcept
8227 {
8228 mImpl->setFlags(builderFlags);
8229 }
8230
8238 BuilderFlags getFlags() const noexcept
8239 {
8240 return mImpl->getFlags();
8241 }
8242
8250 void clearFlag(BuilderFlag builderFlag) noexcept
8251 {
8252 mImpl->clearFlag(builderFlag);
8253 }
8254
8262 void setFlag(BuilderFlag builderFlag) noexcept
8263 {
8264 mImpl->setFlag(builderFlag);
8265 }
8266
8274 bool getFlag(BuilderFlag builderFlag) const noexcept
8275 {
8276 return mImpl->getFlag(builderFlag);
8277 }
8278
8289 void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept
8290 {
8291 mImpl->setDeviceType(layer, deviceType);
8292 }
8293
8298 DeviceType getDeviceType(ILayer const* layer) const noexcept
8299 {
8300 return mImpl->getDeviceType(layer);
8301 }
8302
8308 bool isDeviceTypeSet(ILayer const* layer) const noexcept
8309 {
8310 return mImpl->isDeviceTypeSet(layer);
8311 }
8312
8318 void resetDeviceType(ILayer const* layer) noexcept
8319 {
8320 mImpl->resetDeviceType(layer);
8321 }
8322
8327 bool canRunOnDLA(ILayer const* layer) const noexcept
8328 {
8329 return mImpl->canRunOnDLA(layer);
8330 }
8331
8342 void setDLACore(int32_t dlaCore) noexcept
8343 {
8344 mImpl->setDLACore(dlaCore);
8345 }
8346
8351 int32_t getDLACore() const noexcept
8352 {
8353 return mImpl->getDLACore();
8354 }
8355
8361 void setDefaultDeviceType(DeviceType deviceType) noexcept
8362 {
8363 mImpl->setDefaultDeviceType(deviceType);
8364 }
8365
8372 {
8373 return mImpl->getDefaultDeviceType();
8374 }
8375
8381 void reset() noexcept
8382 {
8383 mImpl->reset();
8384 }
8385
8395 TRT_DEPRECATED void destroy() noexcept
8396 {
8397 delete this;
8398 }
8399
8407 void setProfileStream(const cudaStream_t stream) noexcept
8408 {
8409 return mImpl->setProfileStream(stream);
8410 }
8411
8419 cudaStream_t getProfileStream() const noexcept
8420 {
8421 return mImpl->getProfileStream();
8422 }
8423
8435 int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept
8436 {
8437 return mImpl->addOptimizationProfile(profile);
8438 }
8439
8448 int32_t getNbOptimizationProfiles() const noexcept
8449 {
8450 return mImpl->getNbOptimizationProfiles();
8451 }
8452
8461 {
8462 mImpl->setProfilingVerbosity(verbosity);
8463 }
8464
8474 {
8475 return mImpl->getProfilingVerbosity();
8476 }
8477
8483 {
8484 mImpl->setAlgorithmSelector(selector);
8485 }
8486
8491 {
8492 return mImpl->getAlgorithmSelector();
8493 }
8494
8505 bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept
8506 {
8507 return mImpl->setCalibrationProfile(profile);
8508 }
8509
8516 {
8517 return mImpl->getCalibrationProfile();
8518 }
8519
8533 {
8534 mImpl->setQuantizationFlags(flags);
8535 }
8536
8545 {
8546 return mImpl->getQuantizationFlags();
8547 }
8548
8557 {
8558 mImpl->clearQuantizationFlag(flag);
8559 }
8560
8569 {
8570 mImpl->setQuantizationFlag(flag);
8571 }
8572
8580 bool getQuantizationFlag(QuantizationFlag flag) const noexcept
8581 {
8582 return mImpl->getQuantizationFlag(flag);
8583 }
8584
8605 bool setTacticSources(TacticSources tacticSources) noexcept
8606 {
8607 return mImpl->setTacticSources(tacticSources);
8608 }
8609
8621 {
8622 return mImpl->getTacticSources();
8623 }
8624
8639 nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept
8640 {
8641 return mImpl->createTimingCache(blob, size);
8642 }
8643
8662 bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept
8663 {
8664 return mImpl->setTimingCache(cache, ignoreMismatch);
8665 }
8666
8673 {
8674 return mImpl->getTimingCache();
8675 }
8676
8704 void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
8705 {
8706 mImpl->setMemoryPoolLimit(pool, poolSize);
8707 }
8708
8723 std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
8724 {
8725 return mImpl->getMemoryPoolLimit(pool);
8726 }
8727
8728protected:
8729 apiv::VBuilderConfig* mImpl;
8730};
8731
8739
8749{
8755 kEXPLICIT_BATCH = 0,
8756
8760};
8761
8767template <>
8768constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>() noexcept
8769{
8770 return 2;
8771}
8772
8780class IBuilder : public INoCopy
8781{
8782public:
8783 virtual ~IBuilder() noexcept = default;
8784
8795 TRT_DEPRECATED void setMaxBatchSize(int32_t batchSize) noexcept
8796 {
8797 mImpl->setMaxBatchSize(batchSize);
8798 }
8799
8810 TRT_DEPRECATED int32_t getMaxBatchSize() const noexcept
8811 {
8812 return mImpl->getMaxBatchSize();
8813 }
8814
8818 bool platformHasFastFp16() const noexcept
8819 {
8820 return mImpl->platformHasFastFp16();
8821 }
8822
8826 bool platformHasFastInt8() const noexcept
8827 {
8828 return mImpl->platformHasFastInt8();
8829 }
8830
8838 TRT_DEPRECATED void destroy() noexcept
8839 {
8840 delete this;
8841 }
8842
8850 int32_t getMaxDLABatchSize() const noexcept
8851 {
8852 return mImpl->getMaxDLABatchSize();
8853 }
8854
8858 int32_t getNbDLACores() const noexcept
8859 {
8860 return mImpl->getNbDLACores();
8861 }
8862
8874 void setGpuAllocator(IGpuAllocator* allocator) noexcept
8875 {
8876 mImpl->setGpuAllocator(allocator);
8877 }
8878
8885 {
8886 return mImpl->createBuilderConfig();
8887 }
8888
8900 INetworkDefinition& network, IBuilderConfig& config) noexcept
8901 {
8902 return mImpl->buildEngineWithConfig(network, config);
8903 }
8904
8918 {
8919 return mImpl->createNetworkV2(flags);
8920 }
8921
8932 {
8933 return mImpl->createOptimizationProfile();
8934 }
8935
8947 //
8950 void setErrorRecorder(IErrorRecorder* recorder) noexcept
8951 {
8952 mImpl->setErrorRecorder(recorder);
8953 }
8954
8966 {
8967 return mImpl->getErrorRecorder();
8968 }
8969
8973 void reset() noexcept
8974 {
8975 mImpl->reset();
8976 }
8977
8981 bool platformHasTf32() const noexcept
8982 {
8983 return mImpl->platformHasTf32();
8984 }
8985
9001 {
9002 return mImpl->buildSerializedNetwork(network, config);
9003 }
9004
9024 bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
9025 {
9026 return mImpl->isNetworkSupported(network, config);
9027 }
9028
9034 ILogger* getLogger() const noexcept
9035 {
9036 return mImpl->getLogger();
9037 }
9038
9048 bool setMaxThreads(int32_t maxThreads) noexcept
9049 {
9050 return mImpl->setMaxThreads(maxThreads);
9051 }
9052
9062 int32_t getMaxThreads() const noexcept
9063 {
9064 return mImpl->getMaxThreads();
9065 }
9066
9067protected:
9068 apiv::VBuilder* mImpl;
9069};
9070
9071} // namespace nvinfer1
9072
9077extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept;
9078
9079namespace nvinfer1
9080{
9081namespace
9082{
9083
9091inline IBuilder* createInferBuilder(ILogger& logger) noexcept
9092{
9093 return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
9094}
9095
9096} // namespace
9097
9109 nvinfer1::EngineCapability capability) noexcept;
9110
9111} // namespace nvinfer1
9112
9113#endif // NV_INFER_H
#define TENSORRTAPI
Definition: NvInferRuntimeCommon.h:54
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeCommon.h:73
#define TRT_DEPRECATED
Definition: NvInferRuntimeCommon.h:40
#define TRT_DEPRECATED_ENUM
Definition: NvInferRuntimeCommon.h:41
Definition: NvInferRuntimeCommon.h:153
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeCommon.h:156
Descriptor for two-dimensional spatial data.
Definition: NvInferLegacyDims.h:64
An Activation layer in a network definition.
Definition: NvInfer.h:1552
void setBeta(float beta) noexcept
Set the beta parameter (must be finite).
Definition: NvInfer.h:1600
void setActivationType(ActivationType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1561
ActivationType getActivationType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1571
float getAlpha() const noexcept
Get the alpha parameter.
Definition: NvInfer.h:1609
virtual ~IActivationLayer() noexcept=default
float getBeta() const noexcept
Get the beta parameter.
Definition: NvInfer.h:1618
void setAlpha(float alpha) noexcept
Set the alpha parameter (must be finite).
Definition: NvInfer.h:1586
Describes the context and requirements, that could be fulfilled by one or more instances of IAlgorith...
Definition: NvInfer.h:7683
int32_t getNbOutputs() const noexcept
Return number of outputs of the algorithm.
Definition: NvInfer.h:7716
int32_t getNbInputs() const noexcept
Return number of inputs of the algorithm.
Definition: NvInfer.h:7708
char const * getName() const noexcept
Return name of the algorithm node. This is a unique identifier for the IAlgorithmContext.
Definition: NvInfer.h:7689
virtual ~IAlgorithmContext() noexcept=default
Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for input or output tensor.
Definition: NvInfer.h:7700
Describes a variation of execution of a layer. An algorithm is represented by IAlgorithmVariant and t...
Definition: NvInfer.h:7736
std::size_t getWorkspaceSize() const noexcept
The size of the GPU temporary memory in bytes which the algorithm uses at execution time.
Definition: NvInfer.h:7772
float getTimingMSec() const noexcept
The time in milliseconds to execute the algorithm.
Definition: NvInfer.h:7764
IAlgorithmIOInfo const * getAlgorithmIOInfoByIndex(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:7785
virtual ~IAlgorithm() noexcept=default
TRT_DEPRECATED IAlgorithmIOInfo const & getAlgorithmIOInfo(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:7748
IAlgorithmVariant const & getAlgorithmVariant() const noexcept
Returns the algorithm variant.
Definition: NvInfer.h:7756
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:7608
virtual ~IAlgorithmIOInfo() noexcept=default
Dims getStrides() const noexcept
Return strides of the input/output tensor of algorithm.
Definition: NvInfer.h:7629
DataType getDataType() const noexcept
Return DataType of the input/output of algorithm.
Definition: NvInfer.h:7621
TensorFormat getTensorFormat() const noexcept
Return TensorFormat of the input/output of algorithm.
Definition: NvInfer.h:7613
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
Definition: NvInfer.h:7804
virtual void reportAlgorithms(IAlgorithmContext const *const *algoContexts, IAlgorithm const *const *algoChoices, int32_t nbAlgorithms) noexcept=0
Called by TensorRT to report choices it made.
virtual int32_t selectAlgorithms(IAlgorithmContext const &context, IAlgorithm const *const *choices, int32_t nbChoices, int32_t *selection) noexcept=0
Select Algorithms for a layer from the given list of algorithm choices.
virtual ~IAlgorithmSelector() noexcept=default
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:7651
virtual ~IAlgorithmVariant() noexcept=default
int64_t getTactic() const noexcept
Return tactic of the algorithm.
Definition: NvInfer.h:7664
int64_t getImplementation() const noexcept
Return implementation of the algorithm.
Definition: NvInfer.h:7656
An assertion layer in a network.
Definition: NvInfer.h:5540
void setMessage(char const *message) noexcept
Set the message to print if the assertion fails.
Definition: NvInfer.h:5550
char const * getMessage() const noexcept
Return the assertion message.
Definition: NvInfer.h:5560
virtual ~IAssertionLayer() noexcept=default
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:8079
virtual TRT_DEPRECATED int32_t getMinTimingIterations() const noexcept
Query the number of minimization iterations.
Definition: NvInfer.h:8109
IOptimizationProfile const * getCalibrationProfile() noexcept
Get the current calibration profile.
Definition: NvInfer.h:8515
void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
Set the memory size for the memory pool.
Definition: NvInfer.h:8704
void setQuantizationFlag(QuantizationFlag flag) noexcept
Set a single quantization flag.
Definition: NvInfer.h:8568
nvinfer1::ITimingCache * createTimingCache(void const *blob, std::size_t size) const noexcept
Create timing cache.
Definition: NvInfer.h:8639
void setInt8Calibrator(IInt8Calibrator *calibrator) noexcept
Set Int8 Calibration interface.
Definition: NvInfer.h:8169
void clearQuantizationFlag(QuantizationFlag flag) noexcept
clear a quantization flag.
Definition: NvInfer.h:8556
bool setTacticSources(TacticSources tacticSources) noexcept
Set tactic sources.
Definition: NvInfer.h:8605
bool getQuantizationFlag(QuantizationFlag flag) const noexcept
Returns true if the quantization flag is set.
Definition: NvInfer.h:8580
std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
Get the memory size limit of the memory pool.
Definition: NvInfer.h:8723
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInfer.h:8351
void setDeviceType(ILayer const *layer, DeviceType deviceType) noexcept
Set the device that this layer must execute on.
Definition: NvInfer.h:8289
void setEngineCapability(EngineCapability capability) noexcept
Configure the builder to target specified EngineCapability flow.
Definition: NvInfer.h:8147
bool getFlag(BuilderFlag builderFlag) const noexcept
Returns true if the build mode flag is set.
Definition: NvInfer.h:8274
void setQuantizationFlags(QuantizationFlags flags) noexcept
Set the quantization flags.
Definition: NvInfer.h:8532
bool setCalibrationProfile(IOptimizationProfile const *profile) noexcept
Add a calibration profile.
Definition: NvInfer.h:8505
virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
Set the number of averaging iterations used when timing layers.
Definition: NvInfer.h:8122
void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:8460
IAlgorithmSelector * getAlgorithmSelector() const noexcept
Get Algorithm Selector.
Definition: NvInfer.h:8490
int32_t getNbOptimizationProfiles() const noexcept
Get number of optimization profiles.
Definition: NvInfer.h:8448
QuantizationFlags getQuantizationFlags() const noexcept
Get the quantization flags.
Definition: NvInfer.h:8544
nvinfer1::ITimingCache const * getTimingCache() const noexcept
Get the pointer to the timing cache from current IBuilderConfig.
Definition: NvInfer.h:8672
void reset() noexcept
Resets the builder configuration to defaults.
Definition: NvInfer.h:8381
bool setTimingCache(ITimingCache const &cache, bool ignoreMismatch) noexcept
Attach a timing cache to IBuilderConfig.
Definition: NvInfer.h:8662
TRT_DEPRECATED void destroy() noexcept
Delete this IBuilderConfig.
Definition: NvInfer.h:8395
EngineCapability getEngineCapability() const noexcept
Query EngineCapability flow configured for the builder.
Definition: NvInfer.h:8159
void setAlgorithmSelector(IAlgorithmSelector *selector) noexcept
Set Algorithm Selector.
Definition: NvInfer.h:8482
TRT_DEPRECATED void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept
Set the maximum workspace size.
Definition: NvInfer.h:8192
TRT_DEPRECATED std::size_t getMaxWorkspaceSize() const noexcept
Get the maximum workspace size.
Definition: NvInfer.h:8209
DeviceType getDefaultDeviceType() const noexcept
Get the default DeviceType which was set by setDefaultDeviceType.
Definition: NvInfer.h:8371
BuilderFlags getFlags() const noexcept
Get the build mode flags for this builder config. Defaults to 0.
Definition: NvInfer.h:8238
void setFlags(BuilderFlags builderFlags) noexcept
Set the build mode flags to turn on builder options for this network.
Definition: NvInfer.h:8226
TacticSources getTacticSources() const noexcept
Get tactic sources.
Definition: NvInfer.h:8620
void resetDeviceType(ILayer const *layer) noexcept
reset the DeviceType for this layer
Definition: NvInfer.h:8318
void setDLACore(int32_t dlaCore) noexcept
Sets the DLA core used by the network. Defaults to -1.
Definition: NvInfer.h:8342
void clearFlag(BuilderFlag builderFlag) noexcept
clear a single build mode flag.
Definition: NvInfer.h:8250
int32_t addOptimizationProfile(IOptimizationProfile const *profile) noexcept
Add an optimization profile.
Definition: NvInfer.h:8435
apiv::VBuilderConfig * mImpl
Definition: NvInfer.h:8729
int32_t getAvgTimingIterations() const noexcept
Query the number of averaging iterations.
Definition: NvInfer.h:8134
void setDefaultDeviceType(DeviceType deviceType) noexcept
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
Definition: NvInfer.h:8361
void setFlag(BuilderFlag builderFlag) noexcept
Set a single build mode flag.
Definition: NvInfer.h:8262
virtual ~IBuilderConfig() noexcept=default
DeviceType getDeviceType(ILayer const *layer) const noexcept
Get the device that this layer executes on.
Definition: NvInfer.h:8298
bool canRunOnDLA(ILayer const *layer) const noexcept
Checks if a layer can run on DLA.
Definition: NvInfer.h:8327
cudaStream_t getProfileStream() const noexcept
Get the cuda stream that is used to profile this network.
Definition: NvInfer.h:8419
IInt8Calibrator * getInt8Calibrator() const noexcept
Get Int8 Calibration interface.
Definition: NvInfer.h:8177
ProfilingVerbosity getProfilingVerbosity() const noexcept
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:8473
bool isDeviceTypeSet(ILayer const *layer) const noexcept
whether the DeviceType has been explicitly set for this layer
Definition: NvInfer.h:8308
void setProfileStream(const cudaStream_t stream) noexcept
Set the cuda stream that is used to profile this network.
Definition: NvInfer.h:8407
Builds an engine from a network definition.
Definition: NvInfer.h:8781
int32_t getMaxDLABatchSize() const noexcept
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Definition: NvInfer.h:8850
int32_t getNbDLACores() const noexcept
Return the number of DLA engines available to this builder.
Definition: NvInfer.h:8858
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:8965
apiv::VBuilder * mImpl
Definition: NvInfer.h:9068
ILogger * getLogger() const noexcept
get the logger with which the builder was created
Definition: NvInfer.h:9034
bool isNetworkSupported(INetworkDefinition const &network, IBuilderConfig const &config) const noexcept
Checks that a network is within the scope of the IBuilderConfig settings.
Definition: NvInfer.h:9024
bool platformHasTf32() const noexcept
Determine whether the platform has TF32 support.
Definition: NvInfer.h:8981
int32_t getMaxThreads() const noexcept
get the maximum number of threads that can be used by the builder.
Definition: NvInfer.h:9062
TRT_DEPRECATED void destroy() noexcept
Destroy this object.
Definition: NvInfer.h:8838
nvinfer1::IOptimizationProfile * createOptimizationProfile() noexcept
Create a new optimization profile.
Definition: NvInfer.h:8931
bool platformHasFastFp16() const noexcept
Determine whether the platform has fast native fp16.
Definition: NvInfer.h:8818
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInfer.h:8874
nvinfer1::INetworkDefinition * createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept
Create a network definition object.
Definition: NvInfer.h:8917
nvinfer1::IBuilderConfig * createBuilderConfig() noexcept
Create a builder configuration object.
Definition: NvInfer.h:8884
void reset() noexcept
Resets the builder state to default values.
Definition: NvInfer.h:8973
bool setMaxThreads(int32_t maxThreads) noexcept
Set the maximum number of threads.
Definition: NvInfer.h:9048
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:8950
bool platformHasFastInt8() const noexcept
Determine whether the platform has fast native int8.
Definition: NvInfer.h:8826
nvinfer1::IHostMemory * buildSerializedNetwork(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds and serializes a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:9000
virtual ~IBuilder() noexcept=default
TRT_DEPRECATED int32_t getMaxBatchSize() const noexcept
Get the maximum batch size.
Definition: NvInfer.h:8810
TRT_DEPRECATED nvinfer1::ICudaEngine * buildEngineWithConfig(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds an engine for the given INetworkDefinition and given IBuilderConfig.
Definition: NvInfer.h:8899
A concatenation layer in a network definition.
Definition: NvInfer.h:2358
void setAxis(int32_t axis) noexcept
Set the axis along which concatenation occurs.
Definition: NvInfer.h:2370
int32_t getAxis() const noexcept
Get the axis along which concatenation occurs.
Definition: NvInfer.h:2380
virtual ~IConcatenationLayer() noexcept=default
Definition: NvInfer.h:5144
virtual ~IConditionLayer() noexcept=default
Layer that represents a constant value.
Definition: NvInfer.h:4576
void setWeights(Weights weights) noexcept
Set the weights for the layer.
Definition: NvInfer.h:4587
Weights getWeights() const noexcept
Get the weights for the layer.
Definition: NvInfer.h:4597
apiv::VConstantLayer * mImpl
Definition: NvInfer.h:4627
void setDimensions(Dims dimensions) noexcept
Set the dimensions for the layer.
Definition: NvInfer.h:4609
virtual ~IConstantLayer() noexcept=default
Dims getDimensions() const noexcept
Get the dimensions for the layer.
Definition: NvInfer.h:4621
A convolution layer in a network definition.
Definition: NvInfer.h:987
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding of the convolution. If the padding is asymmetric, the pre-padding is returned.
Definition: NvInfer.h:1090
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride of the convolution.
Definition: NvInfer.h:1058
TRT_DEPRECATED DimsHW getDilation() const noexcept
Get the dilation for a convolution.
Definition: NvInfer.h:1197
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride of the convolution.
Definition: NvInfer.h:1315
void setKernelSizeNd(Dims kernelSize) noexcept
Set the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1290
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1224
Weights getBiasWeights() const noexcept
Get the bias weights for the convolution.
Definition: NvInfer.h:1169
int32_t getNbGroups() const noexcept
Get the number of groups of the convolution.
Definition: NvInfer.h:1120
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1265
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Get the stride of the convolution.
Definition: NvInfer.h:1048
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the convolution.
Definition: NvInfer.h:1355
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the convolution.
Definition: NvInfer.h:1325
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding of the convolution.
Definition: NvInfer.h:1078
Weights getKernelWeights() const noexcept
Get the kernel weights of the convolution.
Definition: NvInfer.h:1144
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding of the convolution.
Definition: NvInfer.h:1343
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1379
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the convolution.
Definition: NvInfer.h:1134
void setDilationNd(Dims dilation) noexcept
Set the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1369
Dims getPostPadding() const noexcept
Get the post-padding.
Definition: NvInfer.h:1251
TRT_DEPRECATED void setDilation(DimsHW dilation) noexcept
Set the dilation for a convolution.
Definition: NvInfer.h:1185
void setNbGroups(int32_t nbGroups) noexcept
Set the number of groups for a convolution.
Definition: NvInfer.h:1110
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1277
int32_t getNbOutputMaps() const noexcept
Get the number of output maps for the convolution.
Definition: NvInfer.h:1032
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding of the convolution.
Definition: NvInfer.h:1214
virtual ~IConvolutionLayer() noexcept=default
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the convolution.
Definition: NvInfer.h:1159
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1300
void setNbOutputMaps(int32_t nbOutputMaps) noexcept
Set the number of output maps for the convolution.
Definition: NvInfer.h:1022
TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
Set the HW kernel size of the convolution.
Definition: NvInfer.h:998
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding of the convolution.
Definition: NvInfer.h:1241
TRT_DEPRECATED DimsHW getKernelSize() const noexcept
Get the HW kernel size of the convolution.
Definition: NvInfer.h:1010
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1343
A deconvolution layer in a network definition.
Definition: NvInfer.h:2398
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the deconvolution.
Definition: NvInfer.h:2576
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Set the stride of the deconvolution.
Definition: NvInfer.h:2461
void setNbOutputMaps(int32_t nbOutputMaps) noexcept
Set the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2435
Weights getKernelWeights() const noexcept
Get the kernel weights for the deconvolution.
Definition: NvInfer.h:2561
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding of the deconvolution.
Definition: NvInfer.h:2604
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding of the deconvolution.
Definition: NvInfer.h:2507
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2720
int32_t getNbGroups() const noexcept
Get the number of groups for a deconvolution.
Definition: NvInfer.h:2537
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2794
Weights getBiasWeights() const noexcept
Get the bias weights for the deconvolution.
Definition: NvInfer.h:2586
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the deconvolution.
Definition: NvInfer.h:2551
void setDilationNd(Dims dilation) noexcept
Set the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2784
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride of the deconvolution.
Definition: NvInfer.h:2473
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding of the deconvolution.
Definition: NvInfer.h:2493
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:2642
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2693
void setKernelSizeNd(Dims kernelSize) noexcept
Set the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2683
virtual ~IDeconvolutionLayer() noexcept=default
TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
Set the HW kernel size of the convolution.
Definition: NvInfer.h:2411
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2710
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2750
void setNbGroups(int32_t nbGroups) noexcept
Set the number of groups for a deconvolution.
Definition: NvInfer.h:2527
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:2656
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2738
TRT_DEPRECATED DimsHW getKernelSize() const noexcept
Get the HW kernel size of the deconvolution.
Definition: NvInfer.h:2423
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding of the deconvolution.
Definition: NvInfer.h:2632
int32_t getNbOutputMaps() const noexcept
Get the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2445
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:2614
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:2668
A Dequantize layer in a network definition.
Definition: NvInfer.h:5916
virtual ~IDequantizeLayer() noexcept=default
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5926
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5937
An Einsum layer in a network.
Definition: NvInfer.h:5983
bool setEquation(char const *equation) noexcept
Set the equation. The equation is a comma-separated list of subscript labels, where each label refers...
Definition: NvInfer.h:5994
virtual ~IEinsumLayer() noexcept=default
char const * getEquation() const noexcept
Return the equation.
Definition: NvInfer.h:6004
A elementwise layer in a network definition.
Definition: NvInfer.h:2869
virtual ~IElementWiseLayer() noexcept=default
apiv::VElementWiseLayer * mImpl
Definition: NvInfer.h:2898
ElementWiseOperation getOperation() const noexcept
Get the binary operation for the layer.
Definition: NvInfer.h:2892
void setOperation(ElementWiseOperation op) noexcept
Set the binary operation for the layer.
Definition: NvInfer.h:2880
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1665
Generate an output tensor with specified mode.
Definition: NvInfer.h:5621
FillOperation getOperation() const noexcept
Get the fill operation for the layer.
Definition: NvInfer.h:5667
void setOperation(FillOperation op) noexcept
Set the fill operation for the layer.
Definition: NvInfer.h:5657
void setDimensions(Dims dimensions) noexcept
Set the output tensor's dimensions.
Definition: NvInfer.h:5632
void setBeta(double beta) noexcept
Set the beta parameter.
Definition: NvInfer.h:5718
double getAlpha() const noexcept
Get the value of alpha parameter.
Definition: NvInfer.h:5700
void setAlpha(double alpha) noexcept
Set the alpha parameter.
Definition: NvInfer.h:5685
Dims getDimensions() const noexcept
Get the output tensor's dimensions.
Definition: NvInfer.h:5647
double getBeta() const noexcept
Get the value of beta parameter.
Definition: NvInfer.h:5733
virtual ~IFillLayer() noexcept=default
A fully connected layer in a network definition. This layer expects an input tensor of three or more ...
Definition: NvInfer.h:1444
virtual ~IFullyConnectedLayer() noexcept=default
void setNbOutputChannels(int32_t nbOutputs) noexcept
Set the number of output channels K from the fully connected layer.
Definition: NvInfer.h:1453
Weights getKernelWeights() const noexcept
Get the kernel weights.
Definition: NvInfer.h:1483
void setBiasWeights(Weights weights) noexcept
Set the bias weights.
Definition: NvInfer.h:1495
void setKernelWeights(Weights weights) noexcept
Set the kernel weights, given as a KxC matrix in row-major order.
Definition: NvInfer.h:1473
int32_t getNbOutputChannels() const noexcept
Get the number of output channels K from the fully connected layer.
Definition: NvInfer.h:1463
Weights getBiasWeights() const noexcept
Get the bias weights.
Definition: NvInfer.h:1505
A Gather layer in a network definition. Supports several kinds of gathering.
Definition: NvInfer.h:3004
void setGatherAxis(int32_t axis) noexcept
Set the axis used by GatherMode::kELEMENTS and GatherMode::kDEFAULT The axis must be less than the nu...
Definition: NvInfer.h:3015
void setNbElementWiseDims(int32_t elementWiseDims) noexcept
Set the number of leading dimensions of indices tensor to be handled elementwise. The gathering of in...
Definition: NvInfer.h:3047
apiv::VGatherLayer * mImpl
Definition: NvInfer.h:3083
int32_t getNbElementWiseDims() const noexcept
Get the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:3057
void setMode(GatherMode mode) noexcept
Set the gather mode.
Definition: NvInfer.h:3067
int32_t getGatherAxis() const noexcept
Get the axis to gather on.
Definition: NvInfer.h:3026
GatherMode getMode() const noexcept
Get the gather mode.
Definition: NvInfer.h:3077
virtual ~IGatherLayer() noexcept=default
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1338
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:144
A layer that represents the identity function.
Definition: NvInfer.h:4561
apiv::VIdentityLayer * mImpl
Definition: NvInfer.h:4563
virtual ~IIdentityLayer() noexcept=default
Definition: NvInfer.h:5127
IIfConditional * getConditional() const noexcept
Return pointer to the IIfConditional associated with this boundary layer.
Definition: NvInfer.h:5130
virtual ~IIfConditionalBoundaryLayer() noexcept=default
Definition: NvInfer.h:5197
IIfConditionalInputLayer * addInput(ITensor &input) noexcept
Add an If-conditional input.
Definition: NvInfer.h:5236
char const * getName() const noexcept
Return the name of the conditional.
Definition: NvInfer.h:5259
virtual ~IIfConditional() noexcept=default
IConditionLayer * setCondition(ITensor &condition) noexcept
Set the condition tensor for this If-Conditional construct.
Definition: NvInfer.h:5208
IIfConditionalOutputLayer * addOutput(ITensor &trueSubgraphOutput, ITensor &falseSubgraphOutput) noexcept
Add an If-conditional output.
Definition: NvInfer.h:5224
void setName(char const *name) noexcept
Set the name of the conditional.
Definition: NvInfer.h:5249
Definition: NvInfer.h:5168
virtual ~IIfConditionalInputLayer() noexcept=default
Definition: NvInfer.h:5157
virtual ~IIfConditionalOutputLayer() noexcept=default
Application-implemented interface for calibration.
Definition: NvInfer.h:7426
virtual int32_t getBatchSize() const noexcept=0
Get the batch size used for calibration batches.
Definition: NvInfer.h:7509
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7514
virtual ~IInt8EntropyCalibrator2() noexcept=default
Definition: NvInfer.h:7491
virtual ~IInt8EntropyCalibrator() noexcept=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7496
Definition: NvInfer.h:7544
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7549
virtual double getQuantile() const noexcept=0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
Definition: NvInfer.h:7526
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7531
virtual ~IInt8MinMaxCalibrator() noexcept=default
Definition: NvInfer.h:5387
virtual ~IIteratorLayer() noexcept=default
void setReverse(bool reverse) noexcept
Definition: NvInfer.h:5406
bool getReverse() const noexcept
True if and only if reversing input.
Definition: NvInfer.h:5412
int32_t getAxis() const noexcept
Get axis being iterated over.
Definition: NvInfer.h:5396
void setAxis(int32_t axis) noexcept
Set axis to iterate over.
Definition: NvInfer.h:5390
A LRN layer in a network definition.
Definition: NvInfer.h:2004
int32_t getWindowSize() const noexcept
Get the LRN window size.
Definition: NvInfer.h:2025
float getAlpha() const noexcept
Get the LRN alpha value.
Definition: NvInfer.h:2046
void setWindowSize(int32_t windowSize) noexcept
Set the LRN window size.
Definition: NvInfer.h:2015
void setK(float k) noexcept
Set the LRN K value.
Definition: NvInfer.h:2078
void setAlpha(float alpha) noexcept
Set the LRN alpha value.
Definition: NvInfer.h:2036
void setBeta(float beta) noexcept
Set the LRN beta value.
Definition: NvInfer.h:2057
virtual ~ILRNLayer() noexcept=default
float getBeta() const noexcept
Get the LRN beta value.
Definition: NvInfer.h:2067
float getK() const noexcept
Get the LRN K value.
Definition: NvInfer.h:2088
Base class for all layer classes in a network definition.
Definition: NvInfer.h:492
bool precisionIsSet() const noexcept
whether the computational precision has been set for this layer
Definition: NvInfer.h:631
void setPrecision(DataType dataType) noexcept
Set the computational precision of this layer.
Definition: NvInfer.h:607
void setName(char const *name) noexcept
Set the name of a layer.
Definition: NvInfer.h:511
void resetPrecision() noexcept
reset the computational precision for this layer
Definition: NvInfer.h:641
int32_t getNbInputs() const noexcept
Get the number of inputs of a layer.
Definition: NvInfer.h:530
DataType getOutputType(int32_t index) const noexcept
get the output type of this layer
Definition: NvInfer.h:693
DataType getPrecision() const noexcept
get the computational precision of this layer
Definition: NvInfer.h:619
char const * getName() const noexcept
Return the name of a layer.
Definition: NvInfer.h:522
int32_t getNbOutputs() const noexcept
Get the number of outputs of a layer.
Definition: NvInfer.h:551
bool outputTypeIsSet(int32_t index) const noexcept
whether the output type has been set for this layer
Definition: NvInfer.h:706
ITensor * getOutput(int32_t index) const noexcept
Get the layer output corresponding to the given index.
Definition: NvInfer.h:562
void setInput(int32_t index, ITensor &tensor) noexcept
Replace an input of this layer with a specific tensor.
Definition: NvInfer.h:579
void resetOutputType(int32_t index) noexcept
reset the output type for this layer
Definition: NvInfer.h:718
ITensor * getInput(int32_t index) const noexcept
Get the layer input corresponding to the given index.
Definition: NvInfer.h:543
void setOutputType(int32_t index, DataType dataType) noexcept
Set the output type of this layer.
Definition: NvInfer.h:679
LayerType getType() const noexcept
Return the type of a layer.
Definition: NvInfer.h:499
virtual ~ILayer() noexcept=default
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeCommon.h:1476
Definition: NvInfer.h:5108
virtual ~ILoopBoundaryLayer() noexcept=default
ILoop * getLoop() const noexcept
Return pointer to ILoop associated with this boundary layer.
Definition: NvInfer.h:5111
Definition: NvInfer.h:5428
void setName(char const *name) noexcept
Set the name of the loop.
Definition: NvInfer.h:5495
ITripLimitLayer * addTripLimit(ITensor &tensor, TripLimit limit) noexcept
Add a trip-count limiter, based on the given tensor.
Definition: NvInfer.h:5457
IIteratorLayer * addIterator(ITensor &tensor, int32_t axis=0, bool reverse=false) noexcept
Return layer that subscripts tensor by loop iteration.
Definition: NvInfer.h:5470
ILoopOutputLayer * addLoopOutput(ITensor &tensor, LoopOutput outputKind, int32_t axis=0) noexcept
Make an output for this loop, based on the given tensor.
Definition: NvInfer.h:5482
virtual ~ILoop() noexcept=default
char const * getName() const noexcept
Return the name of the loop.
Definition: NvInfer.h:5505
IRecurrenceLayer * addRecurrence(ITensor &initialValue) noexcept
Create a recurrence layer for this loop with initialValue as its first input.
Definition: NvInfer.h:5436
Definition: NvInfer.h:5316
virtual ~ILoopOutputLayer() noexcept=default
int32_t getAxis() const noexcept
Get axis being concatenated over.
Definition: NvInfer.h:5341
LoopOutput getLoopOutput() const noexcept
Definition: NvInfer.h:5318
void setAxis(int32_t axis) noexcept
Set where to insert the contenation axis. Ignored if getLoopOutput() is kLAST_VALUE.
Definition: NvInfer.h:5335
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:4486
apiv::VMatrixMultiplyLayer * mImpl
Definition: NvInfer.h:4512
virtual ~IMatrixMultiplyLayer() noexcept=default
MatrixOperation getOperation(int32_t index) const noexcept
Get the operation for an input tensor.
Definition: NvInfer.h:4506
void setOperation(int32_t index, MatrixOperation op) noexcept
Set the operation for an input tensor.
Definition: NvInfer.h:4494
A network definition for input to the builder.
Definition: NvInfer.h:6158
IPluginV2Layer * addPluginV2(ITensor *const *inputs, int32_t nbInputs, IPluginV2 &plugin) noexcept
Add a plugin layer to the network using the IPluginV2 interface.
Definition: NvInfer.h:6860
IConvolutionLayer * addConvolutionNd(ITensor &input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension convolution layer to the network.
Definition: NvInfer.h:7021
IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7313
IConcatenationLayer * addConcatenation(ITensor *const *inputs, int32_t nbInputs) noexcept
Add a concatenation layer to the network.
Definition: NvInfer.h:6372
TRT_DEPRECATED IDeconvolutionLayer * addDeconvolution(ITensor &input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a deconvolution layer to the network.
Definition: NvInfer.h:6395
IGatherLayer * addGatherV2(ITensor &data, ITensor &indices, GatherMode mode)
Add gather with specified mode, axis=0 and nbElementWiseDims=0.
Definition: NvInfer.h:6665
IShuffleLayer * addShuffle(ITensor &input) noexcept
Add a shuffle layer to the network.
Definition: NvInfer.h:6476
void setName(char const *name) noexcept
Sets the name of the network.
Definition: NvInfer.h:6901
ILRNLayer * addLRN(ITensor &input, int32_t window, float alpha, float beta, float k) noexcept
Add a LRN layer to the network.
Definition: NvInfer.h:6315
ITopKLayer * addTopK(ITensor &input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
Add a TopK layer to the network.
Definition: NvInfer.h:6633
IAssertionLayer * addAssertion(ITensor &condition, char const *message) noexcept
Add an assertion layer to the network.
Definition: NvInfer.h:7205
IScaleLayer * addScaleNd(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
Add a multi-dimension scale layer to the network.
Definition: NvInfer.h:7099
char const * getName() const noexcept
Returns the name associated with the network.
Definition: NvInfer.h:6915
TRT_DEPRECATED bool hasExplicitPrecision() const noexcept
True if network is an explicit precision network.
Definition: NvInfer.h:7132
IParametricReLULayer * addParametricReLU(ITensor &input, ITensor &slope) noexcept
Add a parametric ReLU layer to the network.
Definition: NvInfer.h:6999
ITensor * getOutput(int32_t index) const noexcept
Get the output tensor specified by the given index.
Definition: NvInfer.h:6560
ITensor * getInput(int32_t index) const noexcept
Get the input tensor specified by the given index.
Definition: NvInfer.h:6530
bool unmarkOutputForShapes(ITensor &tensor) noexcept
Undo markOutputForShapes.
Definition: NvInfer.h:6981
IFillLayer * addFill(Dims dimensions, FillOperation op) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7223
ILoop * addLoop() noexcept
Add a loop to the network.
Definition: NvInfer.h:7148
IDeconvolutionLayer * addDeconvolutionNd(ITensor &input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension deconvolution layer to the network.
Definition: NvInfer.h:7063
IActivationLayer * addActivation(ITensor &input, ActivationType type) noexcept
Add an activation layer to the network.
Definition: NvInfer.h:6277
virtual ~INetworkDefinition() noexcept=default
ILayer * getLayer(int32_t index) const noexcept
Get the layer specified by the given index.
Definition: NvInfer.h:6502
TRT_DEPRECATED IRNNv2Layer * addRNNv2(ITensor &input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
Add an layerCount deep RNN layer to the network with hiddenSize internal states that can take a batch...
Definition: NvInfer.h:6799
TRT_DEPRECATED IFullyConnectedLayer * addFullyConnected(ITensor &input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
Add a fully connected layer to the network.
Definition: NvInfer.h:6257
IIfConditional * addIfConditional() noexcept
Add an If-conditional layer to the network.
Definition: NvInfer.h:7367
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:7294
int32_t getNbInputs() const noexcept
Get the number of inputs in the network.
Definition: NvInfer.h:6514
bool hasImplicitBatchDimension() const noexcept
Query whether the network was created with an implicit batch dimension.
Definition: NvInfer.h:6951
IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
Add a reduce layer to the network.
Definition: NvInfer.h:6599
IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation) noexcept
Add a unary layer to the network.
Definition: NvInfer.h:6445
void removeTensor(ITensor &tensor) noexcept
remove a tensor from the network definition.
Definition: NvInfer.h:6829
ISelectLayer * addSelect(ITensor &condition, ITensor &thenInput, ITensor &elseInput) noexcept
Add a select layer to the network.
Definition: NvInfer.h:7188
IScatterLayer * addScatter(ITensor &data, ITensor &indices, ITensor &updates, ScatterMode mode) noexcept
Add a Scatter layer to the network with specified mode and axis=0.
Definition: NvInfer.h:7333
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInfer.h:6488
apiv::VNetworkDefinition * mImpl
Definition: NvInfer.h:7387
IPoolingLayer * addPoolingNd(ITensor &input, PoolingType type, Dims windowSize) noexcept
Add a multi-dimension pooling layer to the network.
Definition: NvInfer.h:7041
bool markOutputForShapes(ITensor &tensor) noexcept
Enable tensor's value to be computed by IExecutionContext::getShapeBinding.
Definition: NvInfer.h:6969
TRT_DEPRECATED IPaddingLayer * addPadding(ITensor &input, DimsHW prePadding, DimsHW postPadding) noexcept
Add a padding layer to the network.
Definition: NvInfer.h:6462
IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
Add a Scale layer to the network.
Definition: NvInfer.h:6342
void unmarkOutput(ITensor &tensor) noexcept
unmark a tensor as a network output.
Definition: NvInfer.h:6841
IIdentityLayer * addIdentity(ITensor &input) noexcept
Add an identity layer.
Definition: NvInfer.h:6814
TRT_DEPRECATED IConvolutionLayer * addConvolution(ITensor &input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a convolution layer to the network.
Definition: NvInfer.h:6234
IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7352
IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op) noexcept
Add an elementwise layer to the network.
Definition: NvInfer.h:6423
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:7279
ISliceLayer * addSlice(ITensor &input, Dims start, Dims size, Dims stride) noexcept
Add a slice layer to the network.
Definition: NvInfer.h:6879
IConstantLayer * addConstant(Dims dimensions, Weights weights) noexcept
Add a constant layer to the network.
Definition: NvInfer.h:6730
IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds) noexcept
Add a RaggedSoftMax layer to the network.
Definition: NvInfer.h:6683
IShapeLayer * addShape(ITensor &input) noexcept
Add a shape layer to the network.
Definition: NvInfer.h:6933
IGatherLayer * addGather(ITensor &data, ITensor &indices, int32_t axis) noexcept
Add gather with mode GatherMode::kDEFAULT and specified axis and nbElementWiseDims=0.
Definition: NvInfer.h:6649
TRT_DEPRECATED IPoolingLayer * addPooling(ITensor &input, PoolingType type, DimsHW windowSize) noexcept
Add a pooling layer to the network.
Definition: NvInfer.h:6296
TRT_DEPRECATED void destroy() noexcept
Destroy this INetworkDefinition object.
Definition: NvInfer.h:6572
IResizeLayer * addResize(ITensor &input) noexcept
Add a resize layer to the network.
Definition: NvInfer.h:7115
IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1) noexcept
Add a MatrixMultiply layer to the network.
Definition: NvInfer.h:6704
ISoftMaxLayer * addSoftMax(ITensor &input) noexcept
Add a SoftMax layer to the network.
Definition: NvInfer.h:6355
IEinsumLayer * addEinsum(ITensor *const *inputs, int32_t nbInputs, char const *equation) noexcept
Add an Einsum layer to the network.
Definition: NvInfer.h:7381
void markOutput(ITensor &tensor) noexcept
Mark a tensor as a network output.
Definition: NvInfer.h:6211
int32_t getNbOutputs() const noexcept
Get the number of outputs in the network.
Definition: NvInfer.h:6544
TRT_DEPRECATED IPaddingLayer * addPaddingNd(ITensor &input, Dims prePadding, Dims postPadding) noexcept
Add a padding layer to the network. Only 2D padding is currently supported.
Definition: NvInfer.h:7240
bool setWeightsName(Weights weights, char const *name) noexcept
Associate a name with all current uses of the given weights.
Definition: NvInfer.h:7260
Forward declaration of IEngineInspector for use by other interfaces.
Definition: NvInferRuntime.h:43
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:1120
Layer that represents a padding operation.
Definition: NvInfer.h:3801
Dims getPostPaddingNd() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3902
TRT_DEPRECATED DimsHW getPrePadding() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3824
TRT_DEPRECATED void setPostPadding(DimsHW padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3838
virtual ~IPaddingLayer() noexcept=default
TRT_DEPRECATED void setPrePadding(DimsHW padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3812
Dims getPrePaddingNd() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3876
TRT_DEPRECATED DimsHW getPostPadding() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3850
void setPostPaddingNd(Dims padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3890
void setPrePaddingNd(Dims padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3864
apiv::VPaddingLayer * mImpl
Definition: NvInfer.h:3908
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:4641
apiv::VParametricReLULayer * mImpl
Definition: NvInfer.h:4643
virtual ~IParametricReLULayer() noexcept=default
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:1210
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:373
Layer type for pluginV2.
Definition: NvInfer.h:3572
virtual ~IPluginV2Layer() noexcept=default
apiv::VPluginV2Layer * mImpl
Definition: NvInfer.h:3585
IPluginV2 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:3579
A Pooling layer in a network definition.
Definition: NvInfer.h:1666
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride for pooling.
Definition: NvInfer.h:1739
PoolingType getPoolingType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1685
void setWindowSizeNd(Dims windowSize) noexcept
Set the multi-dimension window size for pooling.
Definition: NvInfer.h:1918
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1905
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:1881
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Set the stride for pooling.
Definition: NvInfer.h:1727
bool getAverageCountExcludesPadding() const noexcept
Get whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1825
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1853
void setPoolingType(PoolingType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1675
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1894
TRT_DEPRECATED void setWindowSize(DimsHW windowSize) noexcept
Set the window size for pooling.
Definition: NvInfer.h:1699
Dims getWindowSizeNd() const noexcept
Get the multi-dimension window size for pooling.
Definition: NvInfer.h:1928
void setAverageCountExcludesPadding(bool exclusive) noexcept
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1814
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding for pooling.
Definition: NvInfer.h:1972
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding for pooling.
Definition: NvInfer.h:1843
float getBlendFactor() const noexcept
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1797
Dims getStrideNd() const noexcept
Get the multi-dimension stride for pooling.
Definition: NvInfer.h:1953
virtual ~IPoolingLayer() noexcept=default
Dims getPaddingNd() const noexcept
Get the multi-dimension padding for pooling.
Definition: NvInfer.h:1984
TRT_DEPRECATED DimsHW getWindowSize() const noexcept
Get the window size for pooling.
Definition: NvInfer.h:1711
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding for pooling.
Definition: NvInfer.h:1769
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride for pooling.
Definition: NvInfer.h:1943
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding for pooling.
Definition: NvInfer.h:1871
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding for pooling.
Definition: NvInfer.h:1755
void setBlendFactor(float blendFactor) noexcept
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1784
A Quantize layer in a network definition.
Definition: NvInfer.h:5829
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5850
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5839
virtual ~IQuantizeLayer() noexcept=default
An RNN layer in a network definition, version 2.
Definition: NvInfer.h:3282
void setDirection(RNNDirection op) noexcept
Set the direction of the RNN layer.
Definition: NvInfer.h:3382
void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept
Set the bias parameters for an individual gate in the RNN.
Definition: NvInfer.h:3486
void setCellState(ITensor &cell) noexcept
Set the initial cell state of the LSTM with the provided cell ITensor.
Definition: NvInfer.h:3542
int32_t getDataLength() const noexcept
Get the maximum data length of the RNN.
Definition: NvInfer.h:3296
void setInputMode(RNNInputMode op) noexcept
Set the input mode of the RNN layer.
Definition: NvInfer.h:3357
Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
Get the bias parameters for an individual gate in the RNN.
Definition: NvInfer.h:3496
void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept
Set the weight parameters for an individual gate in the RNN.
Definition: NvInfer.h:3451
void setOperation(RNNOperation op) noexcept
Set the operation of the RNN layer.
Definition: NvInfer.h:3337
Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
Get the weight parameters for an individual gate in the RNN.
Definition: NvInfer.h:3461
RNNDirection getDirection() const noexcept
Get the direction of the RNN layer.
Definition: NvInfer.h:3392
void setSequenceLengths(ITensor &seqLengths) noexcept
Specify individual sequence lengths in the batch with the ITensor pointed to by seqLengths.
Definition: NvInfer.h:3315
RNNInputMode getInputMode() const noexcept
Get the input mode of the RNN layer.
Definition: NvInfer.h:3367
ITensor * getHiddenState() const noexcept
Get the initial hidden state of the RNN.
Definition: NvInfer.h:3523
ITensor * getCellState() const noexcept
Get the initial cell state of the RNN.
Definition: NvInfer.h:3552
int32_t getMaxSeqLength() const noexcept
Get the maximum sequence length of the RNN.
Definition: NvInfer.h:3292
int32_t getLayerCount() const noexcept
Get the layer count of the RNN.
Definition: NvInfer.h:3284
apiv::VRNNv2Layer * mImpl
Definition: NvInfer.h:3558
ITensor * getSequenceLengths() const noexcept
Get the sequence lengths specified for the RNN.
Definition: NvInfer.h:3327
RNNOperation getOperation() const noexcept
Get the operation of the RNN layer.
Definition: NvInfer.h:3347
virtual ~IRNNv2Layer() noexcept=default
int32_t getHiddenSize() const noexcept
Get the hidden size of the RNN.
Definition: NvInfer.h:3288
void setHiddenState(ITensor &hidden) noexcept
Set the initial hidden state of the RNN with the provided hidden ITensor.
Definition: NvInfer.h:3513
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:4531
apiv::VRaggedSoftMaxLayer * mImpl
Definition: NvInfer.h:4533
virtual ~IRaggedSoftMaxLayer() noexcept=default
Definition: NvInfer.h:5271
virtual ~IRecurrenceLayer() noexcept=default
Layer that represents a reduction across a non-bool tensor.
Definition: NvInfer.h:3723
void setKeepDimensions(bool keepDimensions) noexcept
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:3770
void setOperation(ReduceOperation op) noexcept
Set the reduce operation for the layer.
Definition: NvInfer.h:3730
ReduceOperation getOperation() const noexcept
Get the reduce operation for the layer.
Definition: NvInfer.h:3740
virtual ~IReduceLayer() noexcept=default
uint32_t getReduceAxes() const noexcept
Get the axes over which to reduce for the layer.
Definition: NvInfer.h:3760
void setReduceAxes(uint32_t reduceAxes) noexcept
Set the axes over which to reduce.
Definition: NvInfer.h:3750
apiv::VReduceLayer * mImpl
Definition: NvInfer.h:3786
bool getKeepDimensions() const noexcept
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:3780
A resize layer in a network definition.
Definition: NvInfer.h:4829
void setSelectorForSinglePixel(ResizeSelector selector) noexcept
Set coordinate selector function when resized to single pixel.
Definition: NvInfer.h:5018
void setOutputDimensions(Dims dimensions) noexcept
Set the output dimensions.
Definition: NvInfer.h:4849
void setNearestRounding(ResizeRoundMode value) noexcept
Set rounding mode for nearest neighbor resize.
Definition: NvInfer.h:5042
virtual ~IResizeLayer() noexcept=default
int32_t getScales(int32_t size, float *scales) const noexcept
Copies resize scales to scales[0, ..., nbScales-1], where nbScales is the number of scales that were ...
Definition: NvInfer.h:4908
ResizeMode getResizeMode() const noexcept
Get resize mode for an input tensor.
Definition: NvInfer.h:4930
void setScales(float const *scales, int32_t nbScales) noexcept
Set the resize scales.
Definition: NvInfer.h:4889
void setResizeMode(ResizeMode resizeMode) noexcept
Set resize mode for an input tensor.
Definition: NvInfer.h:4920
ResizeSelector getSelectorForSinglePixel() const noexcept
Get the coordinate selector function when resized to single pixel.
Definition: NvInfer.h:5028
TRT_DEPRECATED bool getAlignCorners() const noexcept
True if align corners has been set.
Definition: NvInfer.h:4958
void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept
Set coordinate transformation function.
Definition: NvInfer.h:4993
Dims getOutputDimensions() const noexcept
Get the output dimensions.
Definition: NvInfer.h:4859
ResizeRoundMode getNearestRounding() const noexcept
Get rounding mode for nearest neighbor resize.
Definition: NvInfer.h:5052
TRT_DEPRECATED void setAlignCorners(bool alignCorners) noexcept
Set whether to align corners while resizing.
Definition: NvInfer.h:4946
ResizeCoordinateTransformation getCoordinateTransformation() const noexcept
Get coordinate transformation function.
Definition: NvInfer.h:5003
A Scale layer in a network definition.
Definition: NvInfer.h:2148
Weights getScale() const noexcept
Get the scale value.
Definition: NvInfer.h:2205
Weights getPower() const noexcept
Get the power value.
Definition: NvInfer.h:2225
void setScale(Weights scale) noexcept
Set the scale value.
Definition: NvInfer.h:2195
void setPower(Weights power) noexcept
Set the power value.
Definition: NvInfer.h:2215
ScaleMode getMode() const noexcept
Get the scale mode.
Definition: NvInfer.h:2165
void setShift(Weights shift) noexcept
Set the shift value.
Definition: NvInfer.h:2175
void setChannelAxis(int32_t channelAxis) noexcept
Set the channel axis.
Definition: NvInfer.h:2261
Weights getShift() const noexcept
Get the shift value.
Definition: NvInfer.h:2185
virtual ~IScaleLayer() noexcept=default
void setMode(ScaleMode mode) noexcept
Set the scale mode.
Definition: NvInfer.h:2155
int32_t getChannelAxis() const noexcept
Get the channel axis.
Definition: NvInfer.h:2240
A scatter layer in a network definition. Supports several kinds of scattering.
Definition: NvInfer.h:6093
void setMode(ScatterMode mode) noexcept
Set the scatter mode.
Definition: NvInfer.h:6100
apiv::VScatterLayer * mImpl
Definition: NvInfer.h:6134
void setAxis(int32_t axis) noexcept
Set the axis used by ScatterMode::kELEMENTS.
Definition: NvInfer.h:6120
int32_t getAxis() const noexcept
Get the axis.
Definition: NvInfer.h:6128
ScatterMode getMode() const noexcept
Get the scatter mode.
Definition: NvInfer.h:6110
virtual ~IScatterLayer() noexcept=default
Definition: NvInfer.h:5519
virtual ~ISelectLayer() noexcept=default
Layer type for getting shape of a tensor.
Definition: NvInfer.h:4315
virtual ~IShapeLayer() noexcept=default
apiv::VShapeLayer * mImpl
Definition: NvInfer.h:4317
Layer type for shuffling data.
Definition: NvInfer.h:3936
apiv::VShuffleLayer * mImpl
Definition: NvInfer.h:4091
void setReshapeDimensions(Dims dimensions) noexcept
Set the reshaped dimensions.
Definition: NvInfer.h:3984
void setFirstTranspose(Permutation permutation) noexcept
Set the permutation applied by the first transpose operation.
Definition: NvInfer.h:3947
void setSecondTranspose(Permutation permutation) noexcept
Set the permutation applied by the second transpose operation.
Definition: NvInfer.h:4044
Dims getReshapeDimensions() const noexcept
Get the reshaped dimensions.
Definition: NvInfer.h:3997
Permutation getFirstTranspose() const noexcept
Get the permutation applied by the first transpose operation.
Definition: NvInfer.h:3959
virtual ~IShuffleLayer() noexcept=default
Permutation getSecondTranspose() const noexcept
Get the permutation applied by the second transpose operation.
Definition: NvInfer.h:4056
bool getZeroIsPlaceholder() const noexcept
Get meaning of 0 in reshape dimensions.
Definition: NvInfer.h:4085
void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
Set meaning of 0 in reshape dimensions.
Definition: NvInfer.h:4072
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:4165
void setMode(SliceMode mode) noexcept
Set the slice mode.
Definition: NvInfer.h:4259
apiv::VSliceLayer * mImpl
Definition: NvInfer.h:4298
virtual ~ISliceLayer() noexcept=default
void setStride(Dims stride) noexcept
Set the stride for computing the output slice data.
Definition: NvInfer.h:4234
void setStart(Dims start) noexcept
Set the start offset that the slice layer uses to create the output slice.
Definition: NvInfer.h:4176
Dims getStart() const noexcept
Get the start offset for the slice layer.
Definition: NvInfer.h:4191
void setSize(Dims size) noexcept
Set the dimensions of the output slice.
Definition: NvInfer.h:4205
Dims getSize() const noexcept
Get dimensions of the output slice.
Definition: NvInfer.h:4220
SliceMode getMode() const noexcept
Get the slice mode.
Definition: NvInfer.h:4269
Dims getStride() const noexcept
Get the stride for the output slice.
Definition: NvInfer.h:4249
A Softmax layer in a network definition.
Definition: NvInfer.h:2293
void setAxes(uint32_t axes) noexcept
Set the axis along which softmax is computed. Currently, only one axis can be set.
Definition: NvInfer.h:2325
uint32_t getAxes() const noexcept
Get the axis along which softmax occurs.
Definition: NvInfer.h:2335
virtual ~ISoftMaxLayer() noexcept=default
A tensor in a network definition.
Definition: NvInfer.h:167
bool setDynamicRange(float min, float max) noexcept
Set dynamic range for the tensor.
Definition: NvInfer.h:267
void setAllowedFormats(TensorFormats formats) noexcept
Set allowed formats for this tensor. By default all formats are allowed. Shape tensors (for which isS...
Definition: NvInfer.h:403
TensorLocation getLocation() const noexcept
Get the storage location of a tensor.
Definition: NvInfer.h:331
void resetDynamicRange() noexcept
Undo effect of setDynamicRange.
Definition: NvInfer.h:364
void setName(char const *name) noexcept
Set the tensor name.
Definition: NvInfer.h:181
bool isExecutionTensor() const noexcept
Whether the tensor is an execution tensor.
Definition: NvInfer.h:474
void setType(DataType type) noexcept
Set the data type of a tensor.
Definition: NvInfer.h:240
bool dynamicRangeIsSet() const noexcept
Query whether dynamic range is set.
Definition: NvInfer.h:356
void setLocation(TensorLocation location) noexcept
Set the storage location of a tensor.
Definition: NvInfer.h:346
char const * getName() const noexcept
Get the tensor name.
Definition: NvInfer.h:193
bool isShapeTensor() const noexcept
Whether the tensor is a shape tensor.
Definition: NvInfer.h:451
float getDynamicRangeMax() const noexcept
Get maximum of dynamic range.
Definition: NvInfer.h:384
bool isNetworkInput() const noexcept
Whether the tensor is a network input.
Definition: NvInfer.h:275
bool isNetworkOutput() const noexcept
Whether the tensor is a network output.
Definition: NvInfer.h:283
bool getBroadcastAcrossBatch() const noexcept
Check if tensor is broadcast across the batch.
Definition: NvInfer.h:321
void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
Set whether to enable broadcast of tensor across the batch.
Definition: NvInfer.h:305
DataType getType() const noexcept
Get the data type of a tensor.
Definition: NvInfer.h:252
apiv::VTensor * mImpl
Definition: NvInfer.h:480
float getDynamicRangeMin() const noexcept
Get minimum of dynamic range.
Definition: NvInfer.h:374
virtual ~ITensor() noexcept=default
void setDimensions(Dims dimensions) noexcept
Set the dimensions of a tensor.
Definition: NvInfer.h:212
Dims getDimensions() const noexcept
Get the dimensions of a tensor.
Definition: NvInfer.h:225
TensorFormats getAllowedFormats() const noexcept
Get a bitmask of TensorFormat values that the tensor supports. For a shape tensor,...
Definition: NvInfer.h:416
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:7963
bool combine(ITimingCache const &inputCache, bool ignoreMismatch) noexcept
Combine input timing cache into local instance.
Definition: NvInfer.h:8000
virtual ~ITimingCache() noexcept=default
apiv::VTimingCache * mImpl
Definition: NvInfer.h:8016
bool reset() noexcept
Empty the timing cache.
Definition: NvInfer.h:8010
Layer that represents a TopK reduction.
Definition: NvInfer.h:4351
void setK(int32_t k) noexcept
Set the k value for the layer.
Definition: NvInfer.h:4380
void setReduceAxes(uint32_t reduceAxes) noexcept
Set which axes to reduce for the layer.
Definition: NvInfer.h:4400
TopKOperation getOperation() const noexcept
Get the operation for the layer.
Definition: NvInfer.h:4368
apiv::VTopKLayer * mImpl
Definition: NvInfer.h:4416
void setOperation(TopKOperation op) noexcept
Set the operation for the layer.
Definition: NvInfer.h:4358
int32_t getK() const noexcept
Get the k value for the layer.
Definition: NvInfer.h:4390
uint32_t getReduceAxes() const noexcept
Get the axes to reduce for the layer.
Definition: NvInfer.h:4410
virtual ~ITopKLayer() noexcept=default
Definition: NvInfer.h:5374
TripLimit getTripLimit() const noexcept
Definition: NvInfer.h:5376
virtual ~ITripLimitLayer() noexcept=default
Layer that represents an unary operation.
Definition: NvInfer.h:3648
void setOperation(UnaryOperation op) noexcept
Set the unary operation for the layer.
Definition: NvInfer.h:3657
apiv::VUnaryLayer * mImpl
Definition: NvInfer.h:3673
UnaryOperation getOperation() const noexcept
Get the unary operation for the layer.
Definition: NvInfer.h:3667
virtual ~IUnaryLayer() noexcept=default
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:126
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:1305
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4734
@ kFORMULA
Use formula to map the original index.
@ kUPPER
Select the upper left pixel.
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:69
nvinfer1::IPluginRegistry * getBuilderPluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for the given capability or nullptr if no registry exists.
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8027
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2104
@ kUNIFORM
Identical coefficients across all elements of the tensor.
@ kCHANNEL
Per-channel coefficients.
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:7844
constexpr int32_t EnumMax< RNNDirection >() noexcept
Definition: NvInfer.h:3204
constexpr int32_t EnumMax< BuilderFlag >() noexcept
Definition: NvInfer.h:7947
constexpr int32_t EnumMax< LayerType >() noexcept
Definition: NvInfer.h:103
constexpr int32_t EnumMax< RNNGateType >() noexcept
Definition: NvInfer.h:3265
constexpr int32_t EnumMax< CalibrationAlgoType >() noexcept
Definition: NvInfer.h:7409
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3603
@ kCOSH
Hyperbolic cosine.
@ kACOSH
Inverse hyperbolic cosine.
@ kERF
Gauss error function.
@ kACOS
Inverse cosine.
@ kABS
Absolute value.
@ kSINH
Hyperbolic sine.
@ kROUND
Round to nearest even for float datatype.
@ kATANH
Inverse hyperbolic tangent.
@ kASINH
Inverse hyperbolic sine.
@ kSIGN
Sign, If input > 0, output 1; if input < 0, output -1; if input == 0, output 0.
@ kEXP
Exponentiation.
@ kATAN
Inverse tangent.
constexpr int32_t EnumMax< ReduceOperation >() noexcept
Definition: NvInfer.h:3710
constexpr int32_t EnumMax< TripLimit >() noexcept
Definition: NvInfer.h:5100
constexpr int32_t EnumMax< RNNInputMode >() noexcept
Definition: NvInfer.h:3236
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3225
@ kSKIP
No operation is performed on the first recurrent layer.
@ kLINEAR
Perform the normal matrix multiplication in the first recurrent layer.
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:122
@ kSELU
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
@ kTANH
TanH activation.
@ kSCALED_TANH
Scaled tanh activation: alpha*tanh(beta*x)
@ kRELU
Rectified linear activation.
@ kELU
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
@ kLEAKY_RELU
LeakyRelu activation: x>=0 ? x : alpha * x.
@ kSOFTSIGN
Softsign activation: x / (1+|x|)
@ kHARD_SIGMOID
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
@ kTHRESHOLDED_RELU
Thresholded ReLU activation: x>alpha ? x : 0.
@ kSIGMOID
Sigmoid activation.
@ kCLIP
Clip activation: max(alpha, min(beta, x))
@ kSOFTPLUS
Parametric softplus activation: alpha*log(exp(beta*x)+1)
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5579
@ kLINSPACE
Generate evenly spaced numbers over a specified interval.
@ kRANDOM_UNIFORM
Generate a tensor with random values drawn from a uniform distribution.
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4764
@ kHALF_UP
Round half up.
@ kHALF_DOWN
Round half down.
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3249
@ kUPDATE
Update gate (z).
@ kHIDDEN
Hidden gate (h).
@ kFORGET
Forget gate (f).
@ kOUTPUT
Output gate (o).
@ kRESET
Reset gate (r).
@ kINPUT
Input gate (i).
@ kCELL
Cell gate (c).
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:951
@ kSAME_LOWER
Use SAME padding, with prePadding >= postPadding.
@ kEXPLICIT_ROUND_DOWN
Use explicit padding, rounding output size down.
@ kCAFFE_ROUND_DOWN
Use CAFFE padding, rounding output size down, uses prePadding value.
@ kEXPLICIT_ROUND_UP
Use explicit padding, rounding output size up.
@ kCAFFE_ROUND_UP
Use CAFFE padding, rounding output size up, uses prePadding value.
@ kSAME_UPPER
Use SAME padding, with prePadding <= postPadding.
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:5088
@ kWHILE
Tensor is a scalar of type kBOOL. Loop terminates when value is false.
@ kCOUNT
Tensor is scalar of type kINT32 that contains the trip count.
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:8738
constexpr int32_t EnumMax< GatherMode >() noexcept
Definition: NvInfer.h:2920
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:114
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:7878
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:598
constexpr int32_t EnumMax< ScaleMode >() noexcept
Definition: NvInfer.h:2116
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7396
LayerType
The type values of layer classes.
Definition: NvInfer.h:53
@ kRAGGED_SOFTMAX
Ragged softmax layer.
@ kDECONVOLUTION
Deconvolution layer.
@ kREDUCE
Reduce layer.
@ kASSERTION
Assertion layer.
@ kTOPK
TopK layer.
@ kRESIZE
Resize Layer.
@ kPADDING
Padding layer.
@ kMATRIX_MULTIPLY
Matrix multiply layer.
@ kCONDITION
Condition layer.
@ kCONDITIONAL_INPUT
Conditional Input layer.
@ kIDENTITY
Identity layer.
@ kQUANTIZE
Quantize layer.
@ kSCATTER
Scatter layer.
@ kCONVOLUTION
Convolution layer.
@ kPARAMETRIC_RELU
Parametric ReLU layer.
@ kCONCATENATION
Concatenation layer.
@ kSLICE
Slice layer.
@ kEINSUM
Einsum layer.
@ kSOFTMAX
SoftMax layer.
@ kSHAPE
Shape layer.
@ kFULLY_CONNECTED
Fully connected layer.
@ kRECURRENCE
Loop Recurrence layer.
@ kDEQUANTIZE
Dequantize layer.
@ kSHUFFLE
Shuffle layer.
@ kITERATOR
Loop Iterator layer.
@ kPOOLING
Pooling layer.
@ kTRIP_LIMIT
Loop Trip limit layer.
@ kSCALE
Scale layer.
@ kGATHER
Gather layer.
@ kUNARY
UnaryOp operation Layer.
@ kACTIVATION
Activation layer.
@ kELEMENTWISE
Elementwise layer.
@ kSELECT
Select layer.
@ kPLUGIN_V2
PluginV2 layer.
@ kLOOP_OUTPUT
Loop output layer.
@ kCONDITIONAL_OUTPUT
Conditional Output layer.
@ kCONSTANT
Constant layer.
@ kRNN_V2
RNNv2 layer.
@ kFILL
Fill layer.
@ kPLUGIN
Plugin layer.
SliceMode
Controls how ISliceLayer handles out of bounds coordinates.
Definition: NvInfer.h:4101
@ kCLAMP
Out of bounds indices are clamped to bounds.
@ kWRAP
Coordinates wrap around periodically.
constexpr int32_t EnumMax< QuantizationFlag >() noexcept
Definition: NvInfer.h:7867
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2908
@ kDEFAULT
Similar to ONNX Gather.
@ kELEMENT
Similar to ONNX GatherElements.
@ kND
Similar to ONNX GatherND.
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:114
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:1317
ResizeMode
Enumerates various modes of resize in the resize layer. Resize mode set using setResizeMode().
Definition: NvInfer.h:4653
@ kNEAREST
ND (0 < N <= 8) nearest neighbor resizing.
constexpr int32_t EnumMax< SliceMode >() noexcept
Definition: NvInfer.h:4117
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:8749
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2818
@ kSUB
Subtract the second element from the first.
@ kSUM
Sum of the two elements.
@ kPROD
Product of the two elements.
@ kFLOOR_DIV
Floor division of the first element by the second.
@ kEQUAL
Check if two elements are equal.
@ kAND
Logical AND of two elements.
@ kOR
Logical OR of two elements.
@ kMIN
Minimum of the two elements.
@ kPOW
The first element to the power of the second element.
@ kLESS
Check if element in first tensor is less than corresponding element in second tensor.
@ kGREATER
Check if element in first tensor is greater than corresponding element in second tensor.
@ kXOR
Logical XOR of two elements.
@ kDIV
Divide the first element by the second.
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:7854
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3193
@ kBIDIRECTION
Network iterates from first to last and vice versa and outputs concatenated.
@ kUNIDIRECTION
Network iterations from first input to last input.
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7888
@ kDEBUG
Enable debugging of layers via synchronizing after every layer.
@ kGPU_FALLBACK
Enable layers marked to execute on GPU if layer cannot execute on DLA.
@ kSPARSE_WEIGHTS
Allow the builder to examine weights and use optimized functions when weights have suitable sparsity.
@ kFP16
Enable FP16 layer selection, with FP32 fallback.
@ kINT8
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
@ kDISABLE_TIMING_CACHE
Disable reuse of timing information across identical layers.
@ kREFIT
Enable building a refittable engine.
@ kOBEY_PRECISION_CONSTRAINTS
Require that layers execute in specified precisions. Build fails otherwise.
@ kREJECT_EMPTY_ALGORITHMS
Fail if IAlgorithmSelector::selectAlgorithms returns an empty set of algorithms.
constexpr int32_t EnumMax< TopKOperation >() noexcept
Definition: NvInfer.h:4338
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:183
constexpr int32_t EnumMax< MemoryPoolType >() noexcept
Definition: NvInfer.h:8066
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4327
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3696
constexpr int32_t EnumMax< LoopOutput >() noexcept
Definition: NvInfer.h:5081
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:3167
@ kGRU
Three-gate network consisting of Gated Recurrent Units.
@ kLSTM
Four-gate LSTM network w/o peephole connections.
constexpr int32_t EnumMax< NetworkDefinitionCreationFlag >() noexcept
Definition: NvInfer.h:8768
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:6020
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4427
@ kTRANSPOSE
Like kNONE, but transpose the matrix dimensions.
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4680
constexpr int32_t EnumMax< UnaryOperation >() noexcept
Definition: NvInfer.h:3635
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:5064
@ kLAST_VALUE
Output value is value of tensor for last iteration.
@ kCONCATENATE
Output value is concatenation of values of tensor for each iteration, in forward order.
@ kREVERSE
Output value is concatenation of values of tensor for each iteration, in reverse order.
constexpr int32_t EnumMax< MatrixOperation >() noexcept
Definition: NvInfer.h:4455
constexpr int32_t EnumMax< RNNOperation >() noexcept
Definition: NvInfer.h:3180
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1634
constexpr int32_t EnumMax< FillOperation >() noexcept
Definition: NvInfer.h:5590
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:216
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1080
constexpr int32_t EnumMax< ScatterMode >() noexcept
Definition: NvInfer.h:6031
Definition: NvInfer.h:3913
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:99