TensorRT 10.8.0
NvInfer.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_H
19#define NV_INFER_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntime.h"
23
32//
35
41
47namespace nvinfer1
48{
49
57enum class LayerType : int32_t
58{
59 kCONVOLUTION = 0,
60 kCAST = 1,
61 kACTIVATION = 2,
62 kPOOLING = 3,
63 kLRN = 4,
64 kSCALE = 5,
65 kSOFTMAX = 6,
66 kDECONVOLUTION = 7,
67 kCONCATENATION = 8,
68 kELEMENTWISE = 9,
69 kPLUGIN = 10,
70 kUNARY = 11,
71 kPADDING = 12,
72 kSHUFFLE = 13,
73 kREDUCE = 14,
74 kTOPK = 15,
75 kGATHER = 16,
76 kMATRIX_MULTIPLY = 17,
77 kRAGGED_SOFTMAX = 18,
78 kCONSTANT = 19,
79 kIDENTITY = 20,
80 kPLUGIN_V2 = 21,
81 kSLICE = 22,
82 kSHAPE = 23,
83 kPARAMETRIC_RELU = 24,
84 kRESIZE = 25,
85 kTRIP_LIMIT = 26,
86 kRECURRENCE = 27,
87 kITERATOR = 28,
88 kLOOP_OUTPUT = 29,
89 kSELECT = 30,
90 kFILL = 31,
91 kQUANTIZE = 32,
92 kDEQUANTIZE = 33,
93 kCONDITION = 34,
96 kSCATTER = 37,
97 kEINSUM = 38,
98 kASSERTION = 39,
99 kONE_HOT = 40,
100 kNON_ZERO = 41,
101 kGRID_SAMPLE = 42,
102 kNMS = 43,
103 kREVERSE_SEQUENCE = 44,
104 kNORMALIZATION = 45,
105 kPLUGIN_V3 = 46,
106 kSQUEEZE = 47,
107 kUNSQUEEZE = 48,
108 kCUMULATIVE = 49,
110};
111
117template <>
118constexpr inline int32_t EnumMax<LayerType>() noexcept
119{
120 return 51;
121}
122
129using TensorFormats = uint32_t;
130
136enum class ActivationType : int32_t
137{
138 kRELU = 0,
139 kSIGMOID = 1,
140 kTANH = 2,
141 kLEAKY_RELU = 3,
142 kELU = 4,
143 kSELU = 5,
144 kSOFTSIGN = 6,
145 kSOFTPLUS = 7,
146 kCLIP = 8,
147 kHARD_SIGMOID = 9,
148 kSCALED_TANH = 10,
149 kTHRESHOLDED_RELU = 11,
150 kGELU_ERF = 12,
151 kGELU_TANH = 13
152};
153
154namespace impl
155{
161template <>
163{
164 static constexpr int32_t kVALUE = 14;
165};
166} // namespace impl
167
184class ITensor : public INoCopy
185{
186public:
202 void setName(char const* name) noexcept
203 {
204 mImpl->setName(name);
205 }
206
214 char const* getName() const noexcept
215 {
216 return mImpl->getName();
217 }
218
233 void setDimensions(Dims const& dimensions) noexcept
234 {
235 mImpl->setDimensions(dimensions);
236 }
237
247 Dims getDimensions() const noexcept
248 {
249 return mImpl->getDimensions();
250 }
251
262 void setType(DataType type) noexcept
263 {
264 mImpl->setType(type);
265 }
266
274 DataType getType() const noexcept
275 {
276 return mImpl->getType();
277 }
278
291 TRT_DEPRECATED bool setDynamicRange(float min, float max) noexcept
292 {
293 return mImpl->setDynamicRange(min, max);
294 }
295
299 bool isNetworkInput() const noexcept
300 {
301 return mImpl->isNetworkInput();
302 }
303
307 bool isNetworkOutput() const noexcept
308 {
309 return mImpl->isNetworkOutput();
310 }
311
324 TRT_DEPRECATED void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
325 {
326 mImpl->setBroadcastAcrossBatch(broadcastAcrossBatch);
327 }
328
339 {
340 return mImpl->getBroadcastAcrossBatch();
341 }
342
351 {
352 return mImpl->getLocation();
353 }
354
370 {
371 mImpl->setLocation(location);
372 }
373
381 TRT_DEPRECATED bool dynamicRangeIsSet() const noexcept
382 {
383 return mImpl->dynamicRangeIsSet();
384 }
385
389 void resetDynamicRange() noexcept
390 {
391 mImpl->resetDynamicRange();
392 }
393
399 float getDynamicRangeMin() const noexcept
400 {
401 return mImpl->getDynamicRangeMin();
402 }
403
409 float getDynamicRangeMax() const noexcept
410 {
411 return mImpl->getDynamicRangeMax();
412 }
413
431 void setAllowedFormats(TensorFormats formats) noexcept
432 {
433 mImpl->setAllowedFormats(formats);
434 }
435
445 {
446 return mImpl->getAllowedFormats();
447 }
448
475 bool isShapeTensor() const noexcept
476 {
477 return mImpl->isShapeTensor();
478 }
479
496 bool isExecutionTensor() const noexcept
497 {
498 return mImpl->isExecutionTensor();
499 }
500
522 void setDimensionName(int32_t index, char const* name) noexcept
523 {
524 mImpl->setDimensionName(index, name);
525 }
526
537 char const* getDimensionName(int32_t index) const noexcept
538 {
539 return mImpl->getDimensionName(index);
540 }
541
542protected:
543 apiv::VTensor* mImpl;
544 virtual ~ITensor() noexcept = default;
545};
546
554class ILayer : public INoCopy
555{
556public:
562 LayerType getType() const noexcept
563 {
564 return mLayer->getType();
565 }
566
576 void setName(char const* name) noexcept
577 {
578 mLayer->setName(name);
579 }
580
586 char const* getName() const noexcept
587 {
588 return mLayer->getName();
589 }
590
594 int32_t getNbInputs() const noexcept
595 {
596 return mLayer->getNbInputs();
597 }
598
607 ITensor* getInput(int32_t index) const noexcept
608 {
609 return mLayer->getInput(index);
610 }
611
615 int32_t getNbOutputs() const noexcept
616 {
617 return mLayer->getNbOutputs();
618 }
619
625 ITensor* getOutput(int32_t index) const noexcept
626 {
627 return mLayer->getOutput(index);
628 }
629
642 void setInput(int32_t index, ITensor& tensor) noexcept
643 {
644 return mLayer->setInput(index, tensor);
645 }
646
673 void setPrecision(DataType dataType) noexcept
674 {
675 mLayer->setPrecision(dataType);
676 }
677
685 DataType getPrecision() const noexcept
686 {
687 return mLayer->getPrecision();
688 }
689
697 bool precisionIsSet() const noexcept
698 {
699 return mLayer->precisionIsSet();
700 }
701
707 void resetPrecision() noexcept
708 {
709 mLayer->resetPrecision();
710 }
711
754 void setOutputType(int32_t index, DataType dataType) noexcept
755 {
756 mLayer->setOutputType(index, dataType);
757 }
758
769 DataType getOutputType(int32_t index) const noexcept
770 {
771 return mLayer->getOutputType(index);
772 }
773
783 bool outputTypeIsSet(int32_t index) const noexcept
784 {
785 return mLayer->outputTypeIsSet(index);
786 }
787
795 void resetOutputType(int32_t index) noexcept
796 {
797 return mLayer->resetOutputType(index);
798 }
799
813 void setMetadata(char const* metadata) noexcept
814 {
815 mLayer->setMetadata(metadata);
816 }
817
826 char const* getMetadata() const noexcept
827 {
828 return mLayer->getMetadata();
829 }
830
831protected:
832 virtual ~ILayer() noexcept = default;
833 apiv::VLayer* mLayer;
834};
835
992enum class PaddingMode : int32_t
993{
996 kSAME_UPPER = 2,
997 kSAME_LOWER = 3,
998};
999
1000namespace impl
1001{
1007template <>
1009{
1010 static constexpr int32_t kVALUE = 4;
1011};
1012} // namespace impl
1013
1027{
1028public:
1036 void setNbOutputMaps(int64_t nbOutputMaps) noexcept
1037 {
1038 mImpl->setNbOutputMaps(nbOutputMaps);
1039 }
1040
1046 int64_t getNbOutputMaps() const noexcept
1047 {
1048 return mImpl->getNbOutputMaps();
1049 }
1050
1066 void setNbGroups(int64_t nbGroups) noexcept
1067 {
1068 mImpl->setNbGroups(nbGroups);
1069 }
1070
1076 int64_t getNbGroups() const noexcept
1077 {
1078 return mImpl->getNbGroups();
1079 }
1080
1090 void setKernelWeights(Weights weights) noexcept
1091 {
1092 mImpl->setKernelWeights(weights);
1093 }
1094
1100 Weights getKernelWeights() const noexcept
1101 {
1102 return mImpl->getKernelWeights();
1103 }
1104
1115 void setBiasWeights(Weights weights) noexcept
1116 {
1117 mImpl->setBiasWeights(weights);
1118 }
1119
1125 Weights getBiasWeights() const noexcept
1126 {
1127 return mImpl->getBiasWeights();
1128 }
1129
1142 void setPrePadding(Dims const& padding) noexcept
1143 {
1144 mImpl->setPrePadding(padding);
1145 }
1146
1152 Dims getPrePadding() const noexcept
1153 {
1154 return mImpl->getPrePadding();
1155 }
1156
1169 void setPostPadding(Dims const& padding) noexcept
1170 {
1171 mImpl->setPostPadding(padding);
1172 }
1173
1179 Dims getPostPadding() const noexcept
1180 {
1181 return mImpl->getPostPadding();
1182 }
1183
1193 void setPaddingMode(PaddingMode paddingMode) noexcept
1194 {
1195 mImpl->setPaddingMode(paddingMode);
1196 }
1197
1206 {
1207 return mImpl->getPaddingMode();
1208 }
1209
1218 void setKernelSizeNd(Dims const& kernelSize) noexcept
1219 {
1220 mImpl->setKernelSizeNd(kernelSize);
1221 }
1222
1228 Dims getKernelSizeNd() const noexcept
1229 {
1230 return mImpl->getKernelSizeNd();
1231 }
1232
1243 void setStrideNd(Dims const& stride) noexcept
1244 {
1245 mImpl->setStrideNd(stride);
1246 }
1247
1253 Dims getStrideNd() const noexcept
1254 {
1255 return mImpl->getStrideNd();
1256 }
1257
1271 void setPaddingNd(Dims const& padding) noexcept
1272 {
1273 mImpl->setPaddingNd(padding);
1274 }
1275
1283 Dims getPaddingNd() const noexcept
1284 {
1285 return mImpl->getPaddingNd();
1286 }
1287
1297 void setDilationNd(Dims const& dilation) noexcept
1298 {
1299 mImpl->setDilationNd(dilation);
1300 }
1301
1307 Dims getDilationNd() const noexcept
1308 {
1309 return mImpl->getDilationNd();
1310 }
1311
1326 using ILayer::setInput;
1327
1328protected:
1329 virtual ~IConvolutionLayer() noexcept = default;
1330 apiv::VConvolutionLayer* mImpl;
1331};
1332
1347{
1348public:
1357 {
1358 mImpl->setActivationType(type);
1359 }
1360
1367 {
1368 return mImpl->getActivationType();
1369 }
1370
1381 void setAlpha(float alpha) noexcept
1382 {
1383 mImpl->setAlpha(alpha);
1384 }
1385
1395 void setBeta(float beta) noexcept
1396 {
1397 mImpl->setBeta(beta);
1398 }
1399
1404 float getAlpha() const noexcept
1405 {
1406 return mImpl->getAlpha();
1407 }
1408
1413 float getBeta() const noexcept
1414 {
1415 return mImpl->getBeta();
1416 }
1417
1418protected:
1419 virtual ~IActivationLayer() noexcept = default;
1420 apiv::VActivationLayer* mImpl;
1421};
1422
1428enum class PoolingType : int32_t
1429{
1430 kMAX = 0,
1431 kAVERAGE = 1,
1433};
1434
1435namespace impl
1436{
1442template <>
1444{
1445 static constexpr int32_t kVALUE = 3;
1446};
1447} // namespace impl
1448
1460class IPoolingLayer : public ILayer
1461{
1462public:
1470 void setPoolingType(PoolingType type) noexcept
1471 {
1472 mImpl->setPoolingType(type);
1473 }
1474
1481 {
1482 return mImpl->getPoolingType();
1483 }
1484
1495 void setBlendFactor(float blendFactor) noexcept
1496 {
1497 mImpl->setBlendFactor(blendFactor);
1498 }
1499
1508 float getBlendFactor() const noexcept
1509 {
1510 return mImpl->getBlendFactor();
1511 }
1512
1522 void setAverageCountExcludesPadding(bool exclusive) noexcept
1523 {
1524 mImpl->setAverageCountExcludesPadding(exclusive);
1525 }
1526
1534 {
1535 return mImpl->getAverageCountExcludesPadding();
1536 }
1537
1551 void setPrePadding(Dims const& padding) noexcept
1552 {
1553 mImpl->setPrePadding(padding);
1554 }
1555
1561 Dims getPrePadding() const noexcept
1562 {
1563 return mImpl->getPrePadding();
1564 }
1565
1579 void setPostPadding(Dims const& padding) noexcept
1580 {
1581 mImpl->setPostPadding(padding);
1582 }
1583
1589 Dims getPostPadding() const noexcept
1590 {
1591 return mImpl->getPostPadding();
1592 }
1593
1602 void setPaddingMode(PaddingMode paddingMode) noexcept
1603 {
1604 mImpl->setPaddingMode(paddingMode);
1605 }
1606
1614 {
1615 return mImpl->getPaddingMode();
1616 }
1617
1626 void setWindowSizeNd(Dims const& windowSize) noexcept
1627 {
1628 mImpl->setWindowSizeNd(windowSize);
1629 }
1630
1636 Dims getWindowSizeNd() const noexcept
1637 {
1638 return mImpl->getWindowSizeNd();
1639 }
1640
1651 void setStrideNd(Dims const& stride) noexcept
1652 {
1653 mImpl->setStrideNd(stride);
1654 }
1655
1661 Dims getStrideNd() const noexcept
1662 {
1663 return mImpl->getStrideNd();
1664 }
1665
1680 void setPaddingNd(Dims const& padding) noexcept
1681 {
1682 mImpl->setPaddingNd(padding);
1683 }
1684
1692 Dims getPaddingNd() const noexcept
1693 {
1694 return mImpl->getPaddingNd();
1695 }
1696
1697protected:
1698 virtual ~IPoolingLayer() noexcept = default;
1699 apiv::VPoolingLayer* mImpl;
1700};
1701
1711class ILRNLayer : public ILayer
1712{
1713public:
1723 void setWindowSize(int64_t windowSize) noexcept
1724 {
1725 mImpl->setWindowSize(windowSize);
1726 }
1727
1733 int64_t getWindowSize() const noexcept
1734 {
1735 return mImpl->getWindowSize();
1736 }
1737
1745 void setAlpha(float alpha) noexcept
1746 {
1747 mImpl->setAlpha(alpha);
1748 }
1749
1755 float getAlpha() const noexcept
1756 {
1757 return mImpl->getAlpha();
1758 }
1759
1767 void setBeta(float beta) noexcept
1768 {
1769 mImpl->setBeta(beta);
1770 }
1771
1777 float getBeta() const noexcept
1778 {
1779 return mImpl->getBeta();
1780 }
1781
1789 void setK(float k) noexcept
1790 {
1791 mImpl->setK(k);
1792 }
1793
1799 float getK() const noexcept
1800 {
1801 return mImpl->getK();
1802 }
1803
1804protected:
1805 virtual ~ILRNLayer() noexcept = default;
1806 apiv::VLRNLayer* mImpl;
1807};
1808
1814enum class ScaleMode : int32_t
1815{
1816 kUNIFORM = 0,
1817 kCHANNEL = 1,
1818 kELEMENTWISE = 2
1819};
1820
1826template <>
1827constexpr inline int32_t EnumMax<ScaleMode>() noexcept
1828{
1829 return 3;
1830}
1831
1857class IScaleLayer : public ILayer
1858{
1859public:
1865 void setMode(ScaleMode mode) noexcept
1866 {
1867 mImpl->setMode(mode);
1868 }
1869
1875 ScaleMode getMode() const noexcept
1876 {
1877 return mImpl->getMode();
1878 }
1879
1885 void setShift(Weights shift) noexcept
1886 {
1887 mImpl->setShift(shift);
1888 }
1889
1895 Weights getShift() const noexcept
1896 {
1897 return mImpl->getShift();
1898 }
1899
1905 void setScale(Weights scale) noexcept
1906 {
1907 mImpl->setScale(scale);
1908 }
1909
1915 Weights getScale() const noexcept
1916 {
1917 return mImpl->getScale();
1918 }
1919
1925 void setPower(Weights power) noexcept
1926 {
1927 mImpl->setPower(power);
1928 }
1929
1935 Weights getPower() const noexcept
1936 {
1937 return mImpl->getPower();
1938 }
1939
1950 int32_t getChannelAxis() const noexcept
1951 {
1952 return mImpl->getChannelAxis();
1953 }
1954
1971 void setChannelAxis(int32_t channelAxis) noexcept
1972 {
1973 mImpl->setChannelAxis(channelAxis);
1974 }
1975
1976protected:
1977 virtual ~IScaleLayer() noexcept = default;
1978 apiv::VScaleLayer* mImpl;
1979};
1980
2001class ISoftMaxLayer : public ILayer
2002{
2003public:
2024 void setAxes(uint32_t axes) noexcept
2025 {
2026 mImpl->setAxes(axes);
2027 }
2028
2034 uint32_t getAxes() const noexcept
2035 {
2036 return mImpl->getAxes();
2037 }
2038
2039protected:
2040 virtual ~ISoftMaxLayer() noexcept = default;
2041 apiv::VSoftMaxLayer* mImpl;
2042};
2043
2057{
2058public:
2070 void setAxis(int32_t axis) noexcept
2071 {
2072 mImpl->setAxis(axis);
2073 }
2074
2080 int32_t getAxis() const noexcept
2081 {
2082 return mImpl->getAxis();
2083 }
2084
2085protected:
2086 virtual ~IConcatenationLayer() noexcept = default;
2087 apiv::VConcatenationLayer* mImpl;
2088};
2089
2098{
2099public:
2107 void setNbOutputMaps(int64_t nbOutputMaps) noexcept
2108 {
2109 mImpl->setNbOutputMaps(nbOutputMaps);
2110 }
2111
2117 int64_t getNbOutputMaps() const noexcept
2118 {
2119 return mImpl->getNbOutputMaps();
2120 }
2121
2137 void setNbGroups(int64_t nbGroups) noexcept
2138 {
2139 mImpl->setNbGroups(nbGroups);
2140 }
2141
2147 int64_t getNbGroups() const noexcept
2148 {
2149 return mImpl->getNbGroups();
2150 }
2151
2161 void setKernelWeights(Weights weights) noexcept
2162 {
2163 mImpl->setKernelWeights(weights);
2164 }
2165
2171 Weights getKernelWeights() const noexcept
2172 {
2173 return mImpl->getKernelWeights();
2174 }
2175
2186 void setBiasWeights(Weights weights) noexcept
2187 {
2188 mImpl->setBiasWeights(weights);
2189 }
2190
2196 Weights getBiasWeights() const noexcept
2197 {
2198 return mImpl->getBiasWeights();
2199 }
2200
2213 void setPrePadding(Dims const& padding) noexcept
2214 {
2215 mImpl->setPrePadding(padding);
2216 }
2217
2223 Dims getPrePadding() const noexcept
2224 {
2225 return mImpl->getPrePadding();
2226 }
2227
2240 void setPostPadding(Dims const& padding) noexcept
2241 {
2242 mImpl->setPostPadding(padding);
2243 }
2244
2250 Dims getPostPadding() const noexcept
2251 {
2252 return mImpl->getPostPadding();
2253 }
2254
2264 void setPaddingMode(PaddingMode paddingMode) noexcept
2265 {
2266 mImpl->setPaddingMode(paddingMode);
2267 }
2268
2277 {
2278 return mImpl->getPaddingMode();
2279 }
2280
2291 void setKernelSizeNd(Dims const& kernelSize) noexcept
2292 {
2293 mImpl->setKernelSizeNd(kernelSize);
2294 }
2295
2301 Dims getKernelSizeNd() const noexcept
2302 {
2303 return mImpl->getKernelSizeNd();
2304 }
2305
2318 void setStrideNd(Dims const& stride) noexcept
2319 {
2320 mImpl->setStrideNd(stride);
2321 }
2322
2328 Dims getStrideNd() const noexcept
2329 {
2330 return mImpl->getStrideNd();
2331 }
2332
2346 void setPaddingNd(Dims const& padding) noexcept
2347 {
2348 mImpl->setPaddingNd(padding);
2349 }
2350
2358 Dims getPaddingNd() const noexcept
2359 {
2360 return mImpl->getPaddingNd();
2361 }
2362
2375 using ILayer::setInput;
2376
2384 void setDilationNd(Dims const& dilation) noexcept
2385 {
2386 mImpl->setDilationNd(dilation);
2387 }
2388
2394 Dims getDilationNd() const noexcept
2395 {
2396 return mImpl->getDilationNd();
2397 }
2398
2399protected:
2400 virtual ~IDeconvolutionLayer() noexcept = default;
2401 apiv::VDeconvolutionLayer* mImpl;
2402};
2403
2418enum class ElementWiseOperation : int32_t
2419{
2420 kSUM = 0,
2421 kPROD = 1,
2422 kMAX = 2,
2423 kMIN = 3,
2424 kSUB = 4,
2425 kDIV = 5,
2426 kPOW = 6,
2427 kFLOOR_DIV = 7,
2428 kAND = 8,
2429 kOR = 9,
2430 kXOR = 10,
2431 kEQUAL = 11,
2432 kGREATER = 12,
2433 kLESS = 13
2434};
2435
2436namespace impl
2437{
2443template <>
2445{
2446 static constexpr int32_t kVALUE = 14;
2447};
2448} // namespace impl
2449
2470{
2471public:
2482 {
2483 return mImpl->setOperation(op);
2484 }
2485
2494 {
2495 return mImpl->getOperation();
2496 }
2497
2498protected:
2499 apiv::VElementWiseLayer* mImpl;
2500 virtual ~IElementWiseLayer() noexcept = default;
2501};
2502
2508enum class GatherMode : int32_t
2509{
2510 kDEFAULT = 0,
2511 kELEMENT = 1,
2512 kND = 2
2513};
2514
2520template <>
2521constexpr inline int32_t EnumMax<GatherMode>() noexcept
2522{
2523 return 3;
2524}
2525
2602class IGatherLayer : public ILayer
2603{
2604public:
2614 void setGatherAxis(int32_t axis) noexcept
2615 {
2616 mImpl->setGatherAxis(axis);
2617 }
2618
2626 int32_t getGatherAxis() const noexcept
2627 {
2628 return mImpl->getGatherAxis();
2629 }
2630
2649 void setNbElementWiseDims(int32_t elementWiseDims) noexcept
2650 {
2651 mImpl->setNbElementWiseDims(elementWiseDims);
2652 }
2653
2659 int32_t getNbElementWiseDims() const noexcept
2660 {
2661 return mImpl->getNbElementWiseDims();
2662 }
2663
2669 void setMode(GatherMode mode) noexcept
2670 {
2671 mImpl->setMode(mode);
2672 }
2673
2679 GatherMode getMode() const noexcept
2680 {
2681 return mImpl->getMode();
2682 }
2683
2684protected:
2685 apiv::VGatherLayer* mImpl;
2686 virtual ~IGatherLayer() noexcept = default;
2687};
2688
2701{
2702public:
2709 {
2710 return mImpl->getPlugin();
2711 }
2712
2713protected:
2714 apiv::VPluginV2Layer* mImpl;
2715 virtual ~IPluginV2Layer() noexcept = default;
2716};
2717
2728{
2729public:
2736 {
2737 return mImpl->getPlugin();
2738 }
2739
2740protected:
2741 apiv::VPluginV3Layer* mImpl;
2742 virtual ~IPluginV3Layer() noexcept = default;
2743};
2744
2761enum class UnaryOperation : int32_t
2762{
2763 kEXP = 0,
2764 kLOG = 1,
2765 kSQRT = 2,
2766 kRECIP = 3,
2767 kABS = 4,
2768 kNEG = 5,
2769 kSIN = 6,
2770 kCOS = 7,
2771 kTAN = 8,
2772 kSINH = 9,
2773 kCOSH = 10,
2774 kASIN = 11,
2775 kACOS = 12,
2776 kATAN = 13,
2777 kASINH = 14,
2778 kACOSH = 15,
2779 kATANH = 16,
2780 kCEIL = 17,
2781 kFLOOR = 18,
2782 kERF = 19,
2783 kNOT = 20,
2784 kSIGN = 21,
2785 kROUND = 22,
2786 kISINF = 23,
2787 kISNAN = 24,
2788};
2789
2795template <>
2796constexpr inline int32_t EnumMax<UnaryOperation>() noexcept
2797{
2798 return 25;
2799}
2800
2808class IUnaryLayer : public ILayer
2809{
2810public:
2819 {
2820 mImpl->setOperation(op);
2821 }
2822
2829 {
2830 return mImpl->getOperation();
2831 }
2832
2833protected:
2834 apiv::VUnaryLayer* mImpl;
2835 virtual ~IUnaryLayer() noexcept = default;
2836};
2837
2856enum class ReduceOperation : int32_t
2857{
2858 kSUM = 0,
2859 kPROD = 1,
2860 kMAX = 2,
2861 kMIN = 3,
2862 kAVG = 4
2863};
2864
2870template <>
2871constexpr inline int32_t EnumMax<ReduceOperation>() noexcept
2872{
2873 return 5;
2874}
2875
2883class IReduceLayer : public ILayer
2884{
2885public:
2892 {
2893 mImpl->setOperation(op);
2894 }
2895
2902 {
2903 return mImpl->getOperation();
2904 }
2905
2911 void setReduceAxes(uint32_t reduceAxes) noexcept
2912 {
2913 mImpl->setReduceAxes(reduceAxes);
2914 }
2915
2921 uint32_t getReduceAxes() const noexcept
2922 {
2923 return mImpl->getReduceAxes();
2924 }
2925
2931 void setKeepDimensions(bool keepDimensions) noexcept
2932 {
2933 mImpl->setKeepDimensions(keepDimensions);
2934 }
2935
2941 bool getKeepDimensions() const noexcept
2942 {
2943 return mImpl->getKeepDimensions();
2944 }
2945
2946protected:
2947 apiv::VReduceLayer* mImpl;
2948 virtual ~IReduceLayer() noexcept = default;
2949};
2950
2963class IPaddingLayer : public ILayer
2964{
2965public:
2975 void setPrePaddingNd(Dims const& padding) noexcept
2976 {
2977 mImpl->setPrePaddingNd(padding);
2978 }
2979
2987 Dims getPrePaddingNd() const noexcept
2988 {
2989 return mImpl->getPrePaddingNd();
2990 }
2991
3001 void setPostPaddingNd(Dims const& padding) noexcept
3002 {
3003 mImpl->setPostPaddingNd(padding);
3004 }
3005
3013 Dims getPostPaddingNd() const noexcept
3014 {
3015 return mImpl->getPostPaddingNd();
3016 }
3017
3018protected:
3019 apiv::VPaddingLayer* mImpl;
3020 virtual ~IPaddingLayer() noexcept = default;
3021};
3022
3029{
3036 int32_t order[Dims::MAX_DIMS];
3037};
3038
3051class IShuffleLayer : public ILayer
3052{
3053public:
3063 void setFirstTranspose(Permutation permutation) noexcept
3064 {
3065 mImpl->setFirstTranspose(permutation);
3066 }
3067
3076 {
3077 return mImpl->getFirstTranspose();
3078 }
3079
3103 void setReshapeDimensions(Dims const& dimensions) noexcept
3104 {
3105 mImpl->setReshapeDimensions(dimensions);
3106 }
3107
3117 {
3118 return mImpl->getReshapeDimensions();
3119 }
3120
3126 //
3149 using ILayer::setInput;
3150
3163 void setSecondTranspose(Permutation permutation) noexcept
3164 {
3165 mImpl->setSecondTranspose(permutation);
3166 }
3167
3176 {
3177 return mImpl->getSecondTranspose();
3178 }
3179
3191 void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
3192 {
3193 return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);
3194 }
3195
3204 bool getZeroIsPlaceholder() const noexcept
3205 {
3206 return mImpl->getZeroIsPlaceholder();
3207 }
3208
3209protected:
3210 apiv::VShuffleLayer* mImpl;
3211 virtual ~IShuffleLayer() noexcept = default;
3212};
3213
3219enum class SampleMode : int32_t
3220{
3221 kSTRICT_BOUNDS = 0,
3222 kWRAP = 1,
3223 kCLAMP = 2,
3224 kFILL = 3,
3225 kREFLECT = 4,
3228};
3229
3235template <>
3236constexpr inline int32_t EnumMax<SampleMode>() noexcept
3237{
3238 return 5;
3239}
3240
3303class ISliceLayer : public ILayer
3304{
3305public:
3315 void setStart(Dims const& start) noexcept
3316 {
3317 mImpl->setStart(start);
3318 }
3319
3330 Dims getStart() const noexcept
3331 {
3332 return mImpl->getStart();
3333 }
3334
3344 void setSize(Dims const& size) noexcept
3345 {
3346 return mImpl->setSize(size);
3347 }
3348
3359 Dims getSize() const noexcept
3360 {
3361 return mImpl->getSize();
3362 }
3363
3373 void setStride(Dims const& stride) noexcept
3374 {
3375 mImpl->setStride(stride);
3376 }
3377
3388 Dims getStride() const noexcept
3389 {
3390 return mImpl->getStride();
3391 }
3392
3398 void setMode(SampleMode mode) noexcept
3399 {
3400 mImpl->setMode(mode);
3401 }
3402
3408 SampleMode getMode() const noexcept
3409 {
3410 return mImpl->getMode();
3411 }
3412
3440 using ILayer::setInput;
3441
3451 void setAxes(Dims const& axes) noexcept
3452 {
3453 mImpl->setAxes(axes);
3454 }
3455
3466 Dims getAxes() const noexcept
3467 {
3468 return mImpl->getAxes();
3469 }
3470
3471protected:
3472 apiv::VSliceLayer* mImpl;
3473 virtual ~ISliceLayer() noexcept = default;
3474};
3475
3488class IShapeLayer : public ILayer
3489{
3490protected:
3491 apiv::VShapeLayer* mImpl;
3492 virtual ~IShapeLayer() noexcept = default;
3493};
3494
3500enum class TopKOperation : int32_t
3501{
3502 kMAX = 0,
3503 kMIN = 1,
3504};
3505
3511template <>
3512constexpr inline int32_t EnumMax<TopKOperation>() noexcept
3513{
3514 return 2;
3515}
3516
3528class ITopKLayer : public ILayer
3529{
3530public:
3536 void setOperation(TopKOperation op) noexcept
3537 {
3538 mImpl->setOperation(op);
3539 }
3540
3547 {
3548 return mImpl->getOperation();
3549 }
3550
3560 void setK(int32_t k) noexcept
3561 {
3562 mImpl->setK(k);
3563 }
3564
3574 int32_t getK() const noexcept
3575 {
3576 return mImpl->getK();
3577 }
3578
3584 void setReduceAxes(uint32_t reduceAxes) noexcept
3585 {
3586 mImpl->setReduceAxes(reduceAxes);
3587 }
3588
3594 uint32_t getReduceAxes() const noexcept
3595 {
3596 return mImpl->getReduceAxes();
3597 }
3598
3613 using ILayer::setInput;
3614
3615protected:
3616 apiv::VTopKLayer* mImpl;
3617 virtual ~ITopKLayer() noexcept = default;
3618};
3619
3626enum class MatrixOperation : int32_t
3627{
3631 kNONE = 0,
3632
3634 kTRANSPOSE = 1,
3635
3646 kVECTOR = 2,
3647};
3648
3654template <>
3655constexpr inline int32_t EnumMax<MatrixOperation>() noexcept
3656{
3657 return 3;
3658}
3659
3686{
3687public:
3696 void setOperation(int32_t index, MatrixOperation op) noexcept
3697 {
3698 mImpl->setOperation(index, op);
3699 }
3700
3708 MatrixOperation getOperation(int32_t index) const noexcept
3709 {
3710 return mImpl->getOperation(index);
3711 }
3712
3713protected:
3714 apiv::VMatrixMultiplyLayer* mImpl;
3715 virtual ~IMatrixMultiplyLayer() noexcept = default;
3716};
3717
3739class INonZeroLayer : public ILayer
3740{
3741protected:
3742 virtual ~INonZeroLayer() noexcept = default;
3743 apiv::VNonZeroLayer* mImpl;
3744};
3745
3761{
3762protected:
3763 apiv::VRaggedSoftMaxLayer* mImpl;
3764 virtual ~IRaggedSoftMaxLayer() noexcept = default;
3765};
3766
3794{
3795protected:
3796 apiv::VIdentityLayer* mImpl;
3797 virtual ~IIdentityLayer() noexcept = default;
3798};
3799
3806class ICastLayer : public ILayer
3807{
3808public:
3816 void setToType(DataType toType) noexcept
3817 {
3818 mImpl->setToType(toType);
3819 }
3820
3827 DataType getToType() const noexcept
3828 {
3829 return mImpl->getToType();
3830 }
3831
3832protected:
3833 apiv::VCastLayer* mImpl;
3834 virtual ~ICastLayer() noexcept = default;
3835};
3836
3846{
3847public:
3856 void setWeights(Weights weights) noexcept
3857 {
3858 mImpl->setWeights(weights);
3859 }
3860
3866 Weights getWeights() const noexcept
3867 {
3868 return mImpl->getWeights();
3869 }
3870
3878 void setDimensions(Dims const& dimensions) noexcept
3879 {
3880 mImpl->setDimensions(dimensions);
3881 }
3882
3890 Dims getDimensions() const noexcept
3891 {
3892 return mImpl->getDimensions();
3893 }
3894
3895protected:
3896 apiv::VConstantLayer* mImpl;
3897 virtual ~IConstantLayer() noexcept = default;
3898};
3899
3910{
3911protected:
3912 apiv::VParametricReLULayer* mImpl;
3913 virtual ~IParametricReLULayer() noexcept = default;
3914};
3915
3921enum class InterpolationMode : int32_t
3922{
3923 kNEAREST = 0,
3924 kLINEAR = 1,
3925 kCUBIC = 2
3926};
3927
3928namespace impl
3929{
3935template <>
3937{
3938 static constexpr int32_t kVALUE = 3;
3939};
3940} // namespace impl
3941
3950{
3963 kALIGN_CORNERS = 0,
3964
3971 kASYMMETRIC = 1,
3972
3979 kHALF_PIXEL = 2,
3980};
3981
3982namespace impl
3983{
3989template <>
3991{
3992 static constexpr int32_t kVALUE = 3;
3993};
3994} // namespace impl
3995
4003enum class ResizeSelector : int32_t
4004{
4006 kFORMULA = 0,
4007
4009 kUPPER = 1,
4010};
4011
4012namespace impl
4013{
4019template <>
4021{
4022 static constexpr int32_t kVALUE = 2;
4023};
4024} // namespace impl
4025
4033enum class ResizeRoundMode : int32_t
4034{
4036 kHALF_UP = 0,
4037
4039 kHALF_DOWN = 1,
4040
4042 kFLOOR = 2,
4043
4045 kCEIL = 3,
4046};
4047
4048namespace impl
4049{
4055template <>
4057{
4058 static constexpr int32_t kVALUE = 4;
4059};
4060} // namespace impl
4061
4098class IResizeLayer : public ILayer
4099{
4100public:
4119 void setOutputDimensions(Dims const& dimensions) noexcept
4120 {
4121 return mImpl->setOutputDimensions(dimensions);
4122 }
4123
4129 Dims getOutputDimensions() const noexcept
4130 {
4131 return mImpl->getOutputDimensions();
4132 }
4133
4159 void setScales(float const* scales, int32_t nbScales) noexcept
4160 {
4161 mImpl->setScales(scales, nbScales);
4162 }
4163
4178 int32_t getScales(int32_t size, float* scales) const noexcept
4179 {
4180 return mImpl->getScales(size, scales);
4181 }
4182
4190 void setResizeMode(InterpolationMode interpolationMode) noexcept
4191 {
4192 mImpl->setResizeMode(interpolationMode);
4193 }
4194
4201 {
4202 return mImpl->getResizeMode();
4203 }
4204
4224 using ILayer::setInput;
4225
4236 {
4237 mImpl->setCoordinateTransformation(coordTransform);
4238 }
4239
4246 {
4247 return mImpl->getCoordinateTransformation();
4248 }
4249
4261 {
4262 mImpl->setSelectorForSinglePixel(selector);
4263 }
4264
4271 {
4272 return mImpl->getSelectorForSinglePixel();
4273 }
4274
4285 {
4286 mImpl->setNearestRounding(value);
4287 }
4288
4295 {
4296 return mImpl->getNearestRounding();
4297 }
4298
4316 void setCubicCoeff(float A) noexcept
4317 {
4318 mImpl->setCubicCoeff(A);
4319 }
4320
4326 float getCubicCoeff() const noexcept
4327 {
4328 return mImpl->getCubicCoeff();
4329 }
4330
4339 void setExcludeOutside(bool excludeFlag) noexcept
4340 {
4341 mImpl->setExcludeOutside(excludeFlag);
4342 }
4343
4349 bool getExcludeOutside() const noexcept
4350 {
4351 return mImpl->getExcludeOutside();
4352 }
4353
4354protected:
4355 virtual ~IResizeLayer() noexcept = default;
4356 apiv::VResizeLayer* mImpl;
4357};
4358
4362enum class LoopOutput : int32_t
4363{
4365 kLAST_VALUE = 0,
4366
4368 kCONCATENATE = 1,
4369
4371 kREVERSE = 2
4372};
4373
4379template <>
4380constexpr inline int32_t EnumMax<LoopOutput>() noexcept
4381{
4382 return 3;
4383}
4384
4388enum class TripLimit : int32_t
4389{
4390
4391 kCOUNT = 0,
4392 kWHILE = 1
4393};
4394
4400template <>
4401constexpr inline int32_t EnumMax<TripLimit>() noexcept
4402{
4403 return 2;
4404}
4405
4406class ILoop;
4407
4422{
4423public:
4427 ILoop* getLoop() const noexcept
4428 {
4429 return mBoundary->getLoop();
4430 }
4431
4432protected:
4433 virtual ~ILoopBoundaryLayer() noexcept = default;
4434 apiv::VLoopBoundaryLayer* mBoundary;
4435};
4436
4445{
4446public:
4451 {
4452 return mBoundary->getConditional();
4453 }
4454
4455protected:
4456 virtual ~IIfConditionalBoundaryLayer() noexcept = default;
4457 apiv::VConditionalBoundaryLayer* mBoundary;
4458};
4459
4466{
4467public:
4468protected:
4469 virtual ~IConditionLayer() noexcept = default;
4470 apiv::VConditionLayer* mImpl;
4471};
4472
4483{
4484public:
4485protected:
4486 virtual ~IIfConditionalOutputLayer() noexcept = default;
4487 apiv::VConditionalOutputLayer* mImpl;
4488};
4489
4496{
4497public:
4498protected:
4499 virtual ~IIfConditionalInputLayer() noexcept = default;
4500 apiv::VConditionalInputLayer* mImpl;
4501};
4502
4527{
4528public:
4539 {
4540 return mImpl->setCondition(condition);
4541 }
4542
4556 IIfConditionalOutputLayer* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput) noexcept
4557 {
4558 return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);
4559 }
4560
4569 {
4570 return mImpl->addInput(input);
4571 }
4572
4583 void setName(char const* name) noexcept
4584 {
4585 mImpl->setName(name);
4586 }
4587
4593 char const* getName() const noexcept
4594 {
4595 return mImpl->getName();
4596 }
4597
4598protected:
4599 virtual ~IIfConditional() noexcept = default;
4600 apiv::VIfConditional* mImpl;
4601};
4602
4611{
4612public:
4618 //
4631 using ILayer::setInput;
4632
4633protected:
4634 virtual ~IRecurrenceLayer() noexcept = default;
4635 apiv::VRecurrenceLayer* mImpl;
4636};
4637
4658{
4659public:
4663 LoopOutput getLoopOutput() const noexcept
4664 {
4665 return mImpl->getLoopOutput();
4666 }
4667
4680 void setAxis(int32_t axis) noexcept
4681 {
4682 mImpl->setAxis(axis);
4683 }
4684
4688 int32_t getAxis() const noexcept
4689 {
4690 return mImpl->getAxis();
4691 }
4692
4698 //
4713 using ILayer::setInput;
4714
4715protected:
4716 virtual ~ILoopOutputLayer() noexcept = default;
4717 apiv::VLoopOutputLayer* mImpl;
4718};
4719
4732{
4733public:
4737 TripLimit getTripLimit() const noexcept
4738 {
4739 return mImpl->getTripLimit();
4740 }
4741
4742protected:
4743 virtual ~ITripLimitLayer() noexcept = default;
4744 apiv::VTripLimitLayer* mImpl;
4745};
4746
4758{
4759public:
4763 void setAxis(int32_t axis) noexcept
4764 {
4765 mImpl->setAxis(axis);
4766 }
4767
4771 int32_t getAxis() const noexcept
4772 {
4773 return mImpl->getAxis();
4774 }
4775
4785 void setReverse(bool reverse) noexcept
4786 {
4787 mImpl->setReverse(reverse);
4788 }
4789
4795 bool getReverse() const noexcept
4796 {
4797 return mImpl->getReverse();
4798 }
4799
4800protected:
4801 virtual ~IIteratorLayer() noexcept = default;
4802 apiv::VIteratorLayer* mImpl;
4803};
4804
4814class ILoop : public INoCopy
4815{
4816public:
4823 IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept
4824 {
4825 return mImpl->addRecurrence(initialValue);
4826 }
4827
4845 {
4846 return mImpl->addTripLimit(tensor, limit);
4847 }
4848
4857 IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept
4858 {
4859 return mImpl->addIterator(tensor, axis, reverse);
4860 }
4861
4870 ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept
4871 {
4872 return mImpl->addLoopOutput(tensor, outputKind, axis);
4873 }
4874
4885 void setName(char const* name) noexcept
4886 {
4887 mImpl->setName(name);
4888 }
4889
4895 char const* getName() const noexcept
4896 {
4897 return mImpl->getName();
4898 }
4899
4900protected:
4901 virtual ~ILoop() noexcept = default;
4902 apiv::VLoop* mImpl;
4903};
4904
4917class ISelectLayer : public ILayer
4918{
4919protected:
4920 virtual ~ISelectLayer() noexcept = default;
4921 apiv::VSelectLayer* mImpl;
4922};
4923
4940{
4941public:
4950 void setMessage(char const* message) noexcept
4951 {
4952 mImpl->setMessage(message);
4953 }
4954
4960 char const* getMessage() const noexcept
4961 {
4962 return mImpl->getMessage();
4963 }
4964
4965protected:
4966 virtual ~IAssertionLayer() noexcept = default;
4967
4968 apiv::VAssertionLayer* mImpl;
4969};
4970
4978enum class FillOperation : int32_t
4979{
4995 kLINSPACE = 0,
4996
4998 kRANDOM_UNIFORM = 1,
4999
5001 kRANDOM_NORMAL = 2
5002};
5003
5009template <>
5010constexpr inline int32_t EnumMax<FillOperation>() noexcept
5011{
5012 return 3;
5013}
5014
5050class IFillLayer : public ILayer
5051{
5052public:
5061 //
5062 void setDimensions(Dims const& dimensions) noexcept
5063 {
5064 mImpl->setDimensions(dimensions);
5065 }
5066
5077 Dims getDimensions() const noexcept
5078 {
5079 return mImpl->getDimensions();
5080 }
5081
5087 void setOperation(FillOperation op) noexcept
5088 {
5089 mImpl->setOperation(op);
5090 }
5091
5098 {
5099 return mImpl->getOperation();
5100 }
5101
5115 //
5116 void setAlpha(double alpha) noexcept
5117 {
5118 mImpl->setAlpha(alpha);
5119 }
5120
5131 double getAlpha() const noexcept
5132 {
5133 return mImpl->getAlpha();
5134 }
5135
5150 void setBeta(double beta) noexcept
5151 {
5152 mImpl->setBeta(beta);
5153 }
5154
5165 double getBeta() const noexcept
5166 {
5167 return mImpl->getBeta();
5168 }
5169
5210 using ILayer::setInput;
5211
5225 //
5226 void setAlphaInt64(int64_t alpha) noexcept
5227 {
5228 mImpl->setAlphaInt64(alpha);
5229 }
5230
5241 int64_t getAlphaInt64() const noexcept
5242 {
5243 return mImpl->getAlphaInt64();
5244 }
5245
5260 void setBetaInt64(int64_t beta) noexcept
5261 {
5262 mImpl->setBetaInt64(beta);
5263 }
5264
5275 int64_t getBetaInt64() const noexcept
5276 {
5277 return mImpl->getBetaInt64();
5278 }
5279
5283 bool isAlphaBetaInt64() const noexcept
5284 {
5285 return mImpl->isAlphaBetaInt64();
5286 }
5287
5300 void setToType(DataType toType) noexcept
5301 {
5302 mImpl->setToType(toType);
5303 }
5304
5312 DataType getToType() const noexcept
5313 {
5314 return mImpl->getToType();
5315 }
5316
5317protected:
5318 virtual ~IFillLayer() noexcept = default;
5319 apiv::VFillLayer* mImpl;
5320};
5321
5397{
5398public:
5407 int32_t getAxis() const noexcept
5408 {
5409 return mImpl->getAxis();
5410 }
5418 void setAxis(int32_t axis) noexcept
5419 {
5420 mImpl->setAxis(axis);
5421 }
5422
5434 void setToType(DataType toType) noexcept
5435 {
5436 mImpl->setToType(toType);
5437 }
5438
5446 DataType getToType() const noexcept
5447 {
5448 return mImpl->getToType();
5449 }
5450
5451protected:
5452 virtual ~IQuantizeLayer() noexcept = default;
5453 apiv::VQuantizeLayer* mImpl;
5454};
5455
5528{
5529public:
5538 int32_t getAxis() const noexcept
5539 {
5540 return mImpl->getAxis();
5541 }
5549 void setAxis(int32_t axis) noexcept
5550 {
5551 mImpl->setAxis(axis);
5552 }
5553
5565 void setToType(DataType toType) noexcept
5566 {
5567 mImpl->setToType(toType);
5568 }
5569
5577 DataType getToType() const noexcept
5578 {
5579 return mImpl->getToType();
5580 }
5581
5582protected:
5583 virtual ~IDequantizeLayer() noexcept = default;
5584 apiv::VDequantizeLayer* mImpl;
5585};
5586
5605{
5606public:
5618 using ILayer::setInput;
5619
5632 void setToType(DataType toType) noexcept
5633 {
5634 mImpl->setToType(toType);
5635 }
5636
5645 DataType getToType() const noexcept
5646 {
5647 return mImpl->getToType();
5648 }
5649
5657 void setScaleType(DataType scaleType) noexcept
5658 {
5659 mImpl->setScaleType(scaleType);
5660 }
5661
5670 DataType getScaleType() const noexcept
5671 {
5672 return mImpl->getScaleType();
5673 }
5674
5683 void setAxis(int32_t axis) noexcept
5684 {
5685 mImpl->setAxis(axis);
5686 }
5687
5693 int32_t getAxis() const noexcept
5694 {
5695 return mImpl->getAxis();
5696 }
5697
5706 void setBlockSize(int32_t size) noexcept
5707 {
5708 mImpl->setBlockSize(size);
5709 }
5710
5716 int32_t getBlockSize() const noexcept
5717 {
5718 return mImpl->getBlockSize();
5719 }
5720
5721protected:
5722 virtual ~IDynamicQuantizeLayer() noexcept = default;
5723 apiv::VDynamicQuantizeLayer* mImpl;
5724};
5725
5762class IEinsumLayer : public ILayer
5763{
5764public:
5774 bool setEquation(char const* equation) noexcept
5775 {
5776 return mImpl->setEquation(equation);
5777 }
5778
5784 char const* getEquation() const noexcept
5785 {
5786 return mImpl->getEquation();
5787 }
5788
5789protected:
5790 virtual ~IEinsumLayer() noexcept = default;
5791 apiv::VEinsumLayer* mImpl;
5792};
5793
5801enum class ScatterMode : int32_t
5802{
5803 kELEMENT = 0,
5804 kND = 1,
5805};
5806
5812template <>
5813constexpr inline int32_t EnumMax<ScatterMode>() noexcept
5814{
5815 return 2;
5816}
5817
5875class IScatterLayer : public ILayer
5876{
5877public:
5883 void setMode(ScatterMode mode) noexcept
5884 {
5885 mImpl->setMode(mode);
5886 }
5887
5893 ScatterMode getMode() const noexcept
5894 {
5895 return mImpl->getMode();
5896 }
5897
5903 void setAxis(int32_t axis) noexcept
5904 {
5905 mImpl->setAxis(axis);
5906 }
5907
5911 int32_t getAxis() const noexcept
5912 {
5913 return mImpl->getAxis();
5914 }
5915
5916protected:
5917 apiv::VScatterLayer* mImpl;
5918 virtual ~IScatterLayer() noexcept = default;
5919}; // class IScatterLayer
5920
5947class IOneHotLayer : public ILayer
5948{
5949public:
5955 void setAxis(int32_t axis) noexcept
5956 {
5957 mImpl->setAxis(axis);
5958 }
5959
5963 int32_t getAxis() const noexcept
5964 {
5965 return mImpl->getAxis();
5966 }
5967
5968protected:
5969 apiv::VOneHotLayer* mImpl;
5970};
5971
5984{
5985public:
5992 {
5993 mImpl->setInterpolationMode(mode);
5994 }
5995
6004 {
6005 return mImpl->getInterpolationMode();
6006 }
6007
6013 void setAlignCorners(bool alignCorners) noexcept
6014 {
6015 mImpl->setAlignCorners(alignCorners);
6016 }
6017
6025 bool getAlignCorners() const noexcept
6026 {
6027 return mImpl->getAlignCorners();
6028 }
6029
6037 bool setSampleMode(SampleMode mode) noexcept
6038 {
6039 return mImpl->setSampleMode(mode);
6040 }
6041
6049 SampleMode getSampleMode() const noexcept
6050 {
6051 return mImpl->getSampleMode();
6052 }
6053
6054protected:
6055 apiv::VGridSampleLayer* mImpl;
6056 virtual ~IGridSampleLayer() noexcept = default;
6057}; // class IGridSampleLayer
6058
6066enum class BoundingBoxFormat : int32_t
6067{
6069 kCORNER_PAIRS = 0,
6071 kCENTER_SIZES = 1
6072};
6073
6079template <>
6080constexpr inline int32_t EnumMax<BoundingBoxFormat>() noexcept
6081{
6082 return 2;
6083}
6084
6131class INMSLayer : public ILayer
6132{
6133public:
6144 {
6145 mImpl->setBoundingBoxFormat(fmt);
6146 }
6147
6156 {
6157 return mImpl->getBoundingBoxFormat();
6158 }
6159
6169 void setTopKBoxLimit(int32_t limit) noexcept
6170 {
6171 mImpl->setTopKBoxLimit(limit);
6172 }
6173
6179 int32_t getTopKBoxLimit() const noexcept
6180 {
6181 return mImpl->getTopKBoxLimit();
6182 }
6183
6202 using ILayer::setInput;
6203
6204protected:
6205 apiv::VNMSLayer* mImpl;
6206 virtual ~INMSLayer() noexcept = default;
6207}; // class INMSLayer
6208
6222{
6223public:
6232 void setBatchAxis(int32_t batchAxis) noexcept
6233 {
6234 mImpl->setBatchAxis(batchAxis);
6235 }
6236
6242 int32_t getBatchAxis() const noexcept
6243 {
6244 return mImpl->getBatchAxis();
6245 }
6246
6255 void setSequenceAxis(int32_t sequenceAxis) noexcept
6256 {
6257 mImpl->setSequenceAxis(sequenceAxis);
6258 }
6259
6265 int32_t getSequenceAxis() const noexcept
6266 {
6267 return mImpl->getSequenceAxis();
6268 }
6269
6270protected:
6271 apiv::VReverseSequenceLayer* mImpl;
6272 virtual ~IReverseSequenceLayer() noexcept = default;
6273}; // class IReverseSequenceLayer
6274
6294{
6295public:
6303 void setEpsilon(float eps) noexcept
6304 {
6305 return mImpl->setEpsilon(eps);
6306 }
6307
6313 float getEpsilon() const noexcept
6314 {
6315 return mImpl->getEpsilon();
6316 }
6317
6323 void setAxes(uint32_t axesMask) noexcept
6324 {
6325 return mImpl->setAxes(axesMask);
6326 }
6327
6333 uint32_t getAxes() const noexcept
6334 {
6335 return mImpl->getAxes();
6336 }
6337
6354 void setNbGroups(int64_t nbGroups) noexcept
6355 {
6356 return mImpl->setNbGroups(nbGroups);
6357 }
6358
6364 int64_t getNbGroups() const noexcept
6365 {
6366 return mImpl->getNbGroups();
6367 }
6368
6390 void setComputePrecision(DataType type) noexcept
6391 {
6392 return mImpl->setComputePrecision(type);
6393 }
6394
6401 {
6402 return mImpl->getComputePrecision();
6403 }
6404
6405protected:
6406 apiv::VNormalizationLayer* mImpl;
6407 virtual ~INormalizationLayer() noexcept = default;
6408};
6409
6418class ISqueezeLayer : public ILayer
6419{
6420public:
6433 using ILayer::setInput;
6434
6435protected:
6436 apiv::VSqueezeLayer* mImpl;
6437 virtual ~ISqueezeLayer() noexcept = default;
6438};
6439
6448{
6449public:
6462 using ILayer::setInput;
6463
6464protected:
6465 apiv::VUnsqueezeLayer* mImpl;
6466 virtual ~IUnsqueezeLayer() noexcept = default;
6467};
6468
6480enum class CumulativeOperation : int32_t
6481{
6482 kSUM = 0,
6483};
6484
6485namespace impl
6486{
6487
6493template <>
6495{
6496 static constexpr int32_t kVALUE = 1;
6497};
6498
6499} // namespace impl
6500
6529{
6530public:
6541 {
6542 return mImpl->setOperation(op);
6543 }
6544
6553 {
6554 return mImpl->getOperation();
6555 }
6556
6564 void setExclusive(bool exclusive) noexcept
6565 {
6566 mImpl->setExclusive(exclusive);
6567 }
6568
6576 bool getExclusive() const noexcept
6577 {
6578 return mImpl->getExclusive();
6579 }
6580
6588 void setReverse(bool reverse) noexcept
6589 {
6590 mImpl->setReverse(reverse);
6591 }
6592
6600 bool getReverse() const noexcept
6601 {
6602 return mImpl->getReverse();
6603 }
6604
6605protected:
6606 apiv::VCumulativeLayer* mImpl;
6607 virtual ~ICumulativeLayer() noexcept = default;
6608};
6609
6628{
6629public:
6630 virtual ~INetworkDefinition() noexcept = default;
6631
6668 ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept
6669 {
6670 return mImpl->addInput(name, type, dimensions);
6671 }
6672
6682 void markOutput(ITensor& tensor) noexcept
6683 {
6684 mImpl->markOutput(tensor);
6685 }
6686
6700 bool markDebug(ITensor& tensor) noexcept
6701 {
6702 return mImpl->markDebug(tensor);
6703 }
6704
6716 bool unmarkDebug(ITensor& tensor) noexcept
6717 {
6718 return mImpl->unmarkDebug(tensor);
6719 }
6720
6726 bool isDebugTensor(nvinfer1::ITensor const& tensor) const noexcept
6727 {
6728 return mImpl->isDebugTensor(tensor);
6729 }
6730
6747 {
6748 return mImpl->addActivation(input, type);
6749 }
6750
6765 ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept
6766 {
6767 return mImpl->addLRN(input, window, alpha, beta, k);
6768 }
6769
6791 IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
6792 {
6793 return mImpl->addScale(input, mode, shift, scale, power);
6794 }
6795
6805 {
6806 return mImpl->addSoftMax(input);
6807 }
6808
6821 IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept
6822 {
6823 return mImpl->addConcatenation(inputs, nbInputs);
6824 }
6825
6849 {
6850 return mImpl->addElementWise(input1, input2, op);
6851 }
6852
6870 IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept
6871 {
6872 return mImpl->addUnary(input, operation);
6873 }
6874
6885 {
6886 return mImpl->addShuffle(input);
6887 }
6888
6901 IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept
6902 {
6903 return mImpl->addOneHot(indices, values, depth, axis);
6904 }
6905
6913 int32_t getNbLayers() const noexcept
6914 {
6915 return mImpl->getNbLayers();
6916 }
6917
6927 ILayer* getLayer(int32_t index) const noexcept
6928 {
6929 return mImpl->getLayer(index);
6930 }
6931
6939 int32_t getNbInputs() const noexcept
6940 {
6941 return mImpl->getNbInputs();
6942 }
6943
6955 ITensor* getInput(int32_t index) const noexcept
6956 {
6957 return mImpl->getInput(index);
6958 }
6959
6969 int32_t getNbOutputs() const noexcept
6970 {
6971 return mImpl->getNbOutputs();
6972 }
6973
6985 ITensor* getOutput(int32_t index) const noexcept
6986 {
6987 return mImpl->getOutput(index);
6988 }
6989
7012 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
7013 {
7014 return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);
7015 }
7016
7044 ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
7045 {
7046 return mImpl->addTopK(input, op, k, reduceAxes);
7047 }
7048
7060 IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept
7061 {
7062 return mImpl->addGather(data, indices, axis);
7063 }
7064
7076 IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept
7077 {
7078 return mImpl->addGatherV2(data, indices, mode);
7079 }
7080
7096 {
7097 return mImpl->addRaggedSoftMax(input, bounds);
7098 }
7099
7117 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
7118 {
7119 return mImpl->addMatrixMultiply(input0, op0, input1, op1);
7120 }
7121
7132 {
7133 return mImpl->addNonZero(input);
7134 }
7135
7155 IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept
7156 {
7157 return mImpl->addConstant(dimensions, weights);
7158 }
7159
7170 {
7171 return mImpl->addIdentity(input);
7172 }
7173
7184 ICastLayer* addCast(ITensor& input, DataType toType) noexcept
7185 {
7186 return mImpl->addCast(input, toType);
7187 }
7188
7199 void removeTensor(ITensor& tensor) noexcept
7200 {
7201 mImpl->removeTensor(tensor);
7202 }
7203
7211 void unmarkOutput(ITensor& tensor) noexcept
7212 {
7213 mImpl->unmarkOutput(tensor);
7214 }
7215
7232 TRT_DEPRECATED IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept
7233 {
7234 return mImpl->addPluginV2(inputs, nbInputs, plugin);
7235 }
7236
7250 IPluginV3Layer* addPluginV3(ITensor* const* inputs, int32_t nbInputs, ITensor* const* shapeInputs,
7251 int32_t nbShapeInputs, IPluginV3& plugin) noexcept
7252 {
7253 return mImpl->addPluginV3(inputs, nbInputs, shapeInputs, nbShapeInputs, plugin);
7254 }
7255
7270 ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept
7271 {
7272 return mImpl->addSlice(input, start, size, stride);
7273 }
7274
7294 void setName(char const* name) noexcept
7295 {
7296 mImpl->setName(name);
7297 }
7298
7308 char const* getName() const noexcept
7309 {
7310 return mImpl->getName();
7311 }
7312
7324 IShapeLayer* addShape(ITensor& input) noexcept
7325 {
7326 return mImpl->addShape(input);
7327 }
7328
7339 {
7340 return mImpl->hasImplicitBatchDimension();
7341 }
7342
7349 {
7350 return mImpl->getFlags();
7351 }
7352
7360 bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept
7361 {
7362 return mImpl->getFlag(networkDefinitionCreationFlag);
7363 }
7364
7377 bool markOutputForShapes(ITensor& tensor) noexcept
7378 {
7379 return mImpl->markOutputForShapes(tensor);
7380 }
7381
7389 bool unmarkOutputForShapes(ITensor& tensor) noexcept
7390 {
7391 return mImpl->unmarkOutputForShapes(tensor);
7392 }
7393
7408 {
7409 return mImpl->addParametricReLU(input, slope);
7410 }
7411
7430 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7431 {
7432 return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7433 }
7434
7449 IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept
7450 {
7451 return mImpl->addPoolingNd(input, type, windowSize);
7452 }
7453
7468 //
7472 ITensor& input, int64_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7473 {
7474 return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7475 }
7476
7509 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
7510 {
7511 return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);
7512 }
7513
7526 {
7527 return mImpl->addResize(input);
7528 }
7529
7539 ILoop* addLoop() noexcept
7540 {
7541 return mImpl->addLoop();
7542 }
7543
7555 {
7556 return mImpl->addIfConditional();
7557 }
7558
7593 ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept
7594 {
7595 return mImpl->addSelect(condition, thenInput, elseInput);
7596 }
7597
7610 IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept
7611 {
7612 return mImpl->addAssertion(condition, message);
7613 }
7614
7635 TRT_DEPRECATED IFillLayer* addFill(Dims const& dimensions, FillOperation op) noexcept
7636 {
7637 return mImpl->addFill(dimensions, op);
7638 }
7639
7661 IFillLayer* addFill(Dims const& dimensions, FillOperation op, DataType outputType) noexcept
7662 {
7663 return mImpl->addFillV2(dimensions, op, outputType);
7664 }
7665
7677 IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept
7678 {
7679 return mImpl->addPaddingNd(input, prePadding, postPadding);
7680 }
7681
7701 bool setWeightsName(Weights weights, char const* name) noexcept
7702 {
7703 return mImpl->setWeightsName(weights, name);
7704 }
7705
7717 //
7720 void setErrorRecorder(IErrorRecorder* recorder) noexcept
7721 {
7722 mImpl->setErrorRecorder(recorder);
7723 }
7724
7736 {
7737 return mImpl->getErrorRecorder();
7738 }
7739
7757 {
7758 return mImpl->addDequantize(input, scale);
7759 }
7760
7777 IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale, DataType outputType) noexcept
7778 {
7779 return mImpl->addDequantizeV2(input, scale, outputType);
7780 }
7781
7797 IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept
7798 {
7799 return mImpl->addScatter(data, indices, updates, mode);
7800 }
7801
7819 {
7820 return mImpl->addQuantize(input, scale);
7821 }
7822
7839 IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale, DataType outputType) noexcept
7840 {
7841 return mImpl->addQuantizeV2(input, scale, outputType);
7842 }
7843
7866 ITensor& input, int32_t axis, int32_t blockSize, DataType outputType, DataType scaleType) noexcept
7867 {
7868 return mImpl->addDynamicQuantize(input, axis, blockSize, outputType, scaleType);
7869 }
7870
7881 IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept
7882 {
7883 return mImpl->addEinsum(inputs, nbInputs, equation);
7884 }
7885
7900 {
7901 return mImpl->addGridSample(input, grid);
7902 }
7903
7917 INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept
7918 {
7919 return mImpl->addNMS(boxes, scores, maxOutputBoxesPerClass);
7920 }
7921
7935 {
7936 return mImpl->addReverseSequence(input, sequenceLens);
7937 }
7938
7960 INormalizationLayer* addNormalization(ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept
7961 {
7962 return mImpl->addNormalization(input, scale, bias, axesMask);
7963 }
7964
7982 ICumulativeLayer* addCumulative(ITensor& input, ITensor& axis, CumulativeOperation operation, bool exclusive, bool reverse) noexcept
7983 {
7984 return mImpl->addCumulative(input, axis, operation, exclusive, reverse);
7985 }
7986
7993 virtual IBuilder& getBuilder() const noexcept
7994 {
7995 return mImpl->getBuilder();
7996 }
7997
8006 bool markWeightsRefittable(char const* name) noexcept
8007 {
8008 return mImpl->markWeightsRefittable(name);
8009 }
8010
8018 bool unmarkWeightsRefittable(char const* name) noexcept
8019 {
8020 return mImpl->unmarkWeightsRefittable(name);
8021 }
8022
8031 bool areWeightsMarkedRefittable(char const* name) const noexcept
8032 {
8033 return mImpl->areWeightsMarkedRefittable(name);
8034 }
8035
8050 ISqueezeLayer* addSqueeze(ITensor& input, ITensor& axes) noexcept
8051 {
8052 return mImpl->addSqueeze(input, axes);
8053 }
8054
8072 {
8073 return mImpl->addUnsqueeze(input, axes);
8074 }
8075
8076protected:
8077 apiv::VNetworkDefinition* mImpl;
8078};
8079
8087enum class CalibrationAlgoType : int32_t
8088{
8093};
8094
8100template <>
8101constexpr inline int32_t EnumMax<CalibrationAlgoType>() noexcept
8102{
8103 return 4;
8104}
8105
8120{
8121public:
8129 TRT_DEPRECATED virtual int32_t getBatchSize() const noexcept = 0;
8130
8145 virtual bool getBatch(void* bindings[], char const* names[], int32_t nbBindings) noexcept = 0;
8146
8161 virtual void const* readCalibrationCache(std::size_t& length) noexcept = 0;
8162
8171 virtual void writeCalibrationCache(void const* ptr, std::size_t length) noexcept = 0;
8172
8178 virtual CalibrationAlgoType getAlgorithm() noexcept = 0;
8179
8180 ~IInt8Calibrator() noexcept override = default;
8181};
8182
8183namespace v_1_0
8184{
8186{
8187public:
8191 InterfaceInfo getInterfaceInfo() const noexcept override
8192 {
8193 return InterfaceInfo{"IInt8EntropyCalibrator", 1, 0};
8194 }
8195
8200 {
8202 }
8203
8204 ~IInt8EntropyCalibrator() noexcept override = default;
8205};
8206} // namespace v_1_0
8207
8222
8223namespace v_1_0
8224{
8226{
8227public:
8231 InterfaceInfo getInterfaceInfo() const noexcept override
8232 {
8233 return InterfaceInfo{"IInt8EntropyCalibrator2", 1, 0};
8234 }
8235
8240 {
8242 }
8243
8244 ~IInt8EntropyCalibrator2() noexcept override = default;
8245};
8246} // namespace v_1_0
8247
8262
8263namespace v_1_0
8264{
8266{
8267public:
8271 InterfaceInfo getInterfaceInfo() const noexcept override
8272 {
8273 return InterfaceInfo{"IInt8MinMaxCalibrator", 1, 0};
8274 }
8275
8280 {
8282 }
8283
8284 ~IInt8MinMaxCalibrator() noexcept override = default;
8285};
8286} // namespace v_1_0
8287
8301
8302namespace v_1_0
8303{
8305{
8306public:
8310 InterfaceInfo getInterfaceInfo() const noexcept override
8311 {
8312 return InterfaceInfo{"IInt8Calibrator", 1, 0};
8313 }
8314
8319 {
8321 }
8322
8329 virtual double getQuantile() const noexcept = 0;
8330
8337 virtual double getRegressionCutoff() const noexcept = 0;
8338
8351 virtual void const* readHistogramCache(std::size_t& length) noexcept = 0;
8352
8361 virtual void writeHistogramCache(void const* ptr, std::size_t length) noexcept = 0;
8362
8363 ~IInt8LegacyCalibrator() noexcept override = default;
8364};
8365} // namespace v_1_0
8366
8381
8395{
8396public:
8402 DataType getDataType() const noexcept
8403 {
8404 return mImpl->getDataType();
8405 }
8406
8413 Dims getStrides() const noexcept
8414 {
8415 return mImpl->getStrides();
8416 }
8417
8423 int64_t getVectorizedDim() const noexcept
8424 {
8425 return mImpl->getVectorizedDim();
8426 }
8427
8434 int64_t getComponentsPerElement() const noexcept
8435 {
8436 return mImpl->getComponentsPerElement();
8437 }
8438
8439protected:
8440 virtual ~IAlgorithmIOInfo() noexcept = default;
8441 apiv::VAlgorithmIOInfo* mImpl;
8442};
8443
8458{
8459public:
8463 int64_t getImplementation() const noexcept
8464 {
8465 return mImpl->getImplementation();
8466 }
8467
8471 int64_t getTactic() const noexcept
8472 {
8473 return mImpl->getTactic();
8474 }
8475
8476protected:
8477 virtual ~IAlgorithmVariant() noexcept = default;
8478 apiv::VAlgorithmVariant* mImpl;
8479};
8480
8492{
8493public:
8499 char const* getName() const noexcept
8500 {
8501 return mImpl->getName();
8502 }
8503
8511 Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
8512 {
8513 return mImpl->getDimensions(index, select);
8514 }
8515
8519 int32_t getNbInputs() const noexcept
8520 {
8521 return mImpl->getNbInputs();
8522 }
8523
8527 int32_t getNbOutputs() const noexcept
8528 {
8529 return mImpl->getNbOutputs();
8530 }
8531
8532protected:
8533 virtual ~IAlgorithmContext() noexcept = default;
8534 apiv::VAlgorithmContext* mImpl;
8535};
8536
8551{
8552public:
8557 {
8558 return mImpl->getAlgorithmVariant();
8559 }
8560
8564 float getTimingMSec() const noexcept
8565 {
8566 return mImpl->getTimingMSec();
8567 }
8568
8572 std::size_t getWorkspaceSize() const noexcept
8573 {
8574 return mImpl->getWorkspaceSize();
8575 }
8576
8586 IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept
8587 {
8588 return mImpl->getAlgorithmIOInfoByIndex(index);
8589 }
8590
8591protected:
8592 virtual ~IAlgorithm() noexcept = default;
8593 apiv::VAlgorithm* mImpl;
8594}; // IAlgorithm
8595
8596namespace v_1_0
8597{
8599{
8600public:
8604 InterfaceInfo getInterfaceInfo() const noexcept override
8605 {
8606 return InterfaceInfo{"IAlgorithmSelector", 1, 0};
8607 }
8622 virtual int32_t selectAlgorithms(IAlgorithmContext const& context, IAlgorithm const* const* choices,
8623 int32_t nbChoices, int32_t* selection) noexcept = 0;
8624
8635 virtual void reportAlgorithms(IAlgorithmContext const* const* algoContexts, IAlgorithm const* const* algoChoices,
8636 int32_t nbAlgorithms) noexcept = 0;
8637
8638 virtual ~IAlgorithmSelector() noexcept = default;
8639};
8640} // namespace v_1_0
8641
8655
8662using QuantizationFlags = uint32_t;
8663
8673enum class QuantizationFlag : int32_t
8674{
8679};
8680
8686template <>
8687constexpr inline int32_t EnumMax<QuantizationFlag>() noexcept
8688{
8689 return 1;
8690}
8691
8709enum class RuntimePlatform : int32_t
8710{
8713 kSAME_AS_BUILD = 0,
8714
8717 kWINDOWS_AMD64 = 1,
8718};
8719
8720namespace impl
8721{
8727template <>
8729{
8730 static constexpr int32_t kVALUE = 2;
8731};
8732} // namespace impl
8733
8740using BuilderFlags = uint32_t;
8741
8749enum class BuilderFlag : int32_t
8750{
8752 kFP16 = 0,
8753
8755 kINT8 = 1,
8756
8758 kDEBUG = 2,
8759
8761 kGPU_FALLBACK = 3,
8762
8764 kREFIT = 4,
8765
8768
8772 kTF32 = 6,
8773
8775 kSPARSE_WEIGHTS = 7,
8776
8783 kSAFETY_SCOPE = 8,
8784
8787
8791
8797
8800
8807
8813
8819 kFP8 = 15,
8820
8824
8827 kBF16 = 17,
8828
8834
8836 kSTRIP_PLAN = 19,
8837
8840
8847 kREFIT_IDENTICAL = 20,
8848
8874 kWEIGHT_STREAMING = 21,
8875
8877 kINT4 = 22,
8878
8883 kREFIT_INDIVIDUAL = 23,
8884
8893 kSTRICT_NANS = 24,
8894
8896 kMONITOR_MEMORY = 25,
8897
8899 kFP4 = 26,
8900
8903};
8904
8910template <>
8911constexpr inline int32_t EnumMax<BuilderFlag>() noexcept
8912{
8913 return 28;
8914}
8915
8916namespace v_1_0
8917{
8931{
8932 uint8_t data[16];
8933};
8934
8943{
8945 uint64_t tacticHash;
8949 static constexpr uint64_t kINVALID_TACTIC_HASH = UINT64_MAX;
8950};
8951} // namespace v_1_0
8952
8966class ITimingCache : public INoCopy
8967{
8968public:
8969 virtual ~ITimingCache() noexcept = default;
8970
8980 nvinfer1::IHostMemory* serialize() const noexcept
8981 {
8982 return mImpl->serialize();
8983 }
8984
9004 bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept
9005 {
9006 return mImpl->combine(inputCache, ignoreMismatch);
9007 }
9008
9014 bool reset() noexcept
9015 {
9016 return mImpl->reset();
9017 }
9018
9033 int64_t queryKeys(TimingCacheKey* keyBuffer, int64_t capacity) const noexcept
9034 {
9035 return mImpl->queryKeys(keyBuffer, capacity);
9036 }
9037
9050 TimingCacheValue query(TimingCacheKey const& key) const noexcept
9051 {
9052 return mImpl->query(key);
9053 }
9054
9072 bool update(TimingCacheKey const& key, TimingCacheValue const& value) noexcept
9073 {
9074 return mImpl->update(key, value);
9075 }
9076
9077protected:
9078 apiv::VTimingCache* mImpl;
9079};
9080
9088enum class MemoryPoolType : int32_t
9089{
9096 kWORKSPACE = 0,
9097
9105
9111 kDLA_LOCAL_DRAM = 2,
9112
9118 kDLA_GLOBAL_DRAM = 3,
9119
9127 kTACTIC_DRAM = 4,
9128
9142};
9143
9149template <>
9150constexpr inline int32_t EnumMax<MemoryPoolType>() noexcept
9151{
9152 return 6;
9153}
9154
9163enum class PreviewFeature : int32_t
9164{
9171
9176};
9177
9178namespace impl
9179{
9185template <>
9187{
9188 static constexpr int32_t kVALUE = 2;
9189};
9190} // namespace impl
9191
9202enum class HardwareCompatibilityLevel : int32_t
9203{
9206 kNONE = 0,
9207
9217 kAMPERE_PLUS = 1,
9218};
9219
9220namespace impl
9221{
9227template <>
9229{
9230 static constexpr int32_t kVALUE = 2;
9231};
9232} // namespace impl
9233
9242enum class TilingOptimizationLevel : int32_t
9243{
9245 kNONE = 0,
9246
9248 kFAST = 1,
9249
9252 kMODERATE = 2,
9253
9255 kFULL = 3
9256
9257};
9258
9259namespace impl
9260{
9266template <>
9268{
9269 static constexpr int32_t kVALUE = 4;
9270};
9271} // namespace impl
9272
9273namespace v_1_0
9274{
9276{
9277public:
9278 IProgressMonitor() = default;
9279 virtual ~IProgressMonitor() noexcept = default;
9280
9284 InterfaceInfo getInterfaceInfo() const noexcept override
9285 {
9286 return InterfaceInfo{"IProgressMonitor", 1, 0};
9287 }
9288
9308 virtual void phaseStart(char const* phaseName, char const* parentPhase, int32_t nbSteps) noexcept = 0;
9309
9322 virtual bool stepComplete(char const* phaseName, int32_t step) noexcept = 0;
9323
9335 virtual void phaseFinish(char const* phaseName) noexcept = 0;
9336
9337}; // class IProgressMonitor
9338} // namespace v_1_0
9339
9360
9369{
9370public:
9371 virtual ~IBuilderConfig() noexcept = default;
9372
9381 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
9382 {
9383 mImpl->setAvgTimingIterations(avgTiming);
9384 }
9385
9393 int32_t getAvgTimingIterations() const noexcept
9394 {
9395 return mImpl->getAvgTimingIterations();
9396 }
9397
9406 void setEngineCapability(EngineCapability capability) noexcept
9407 {
9408 mImpl->setEngineCapability(capability);
9409 }
9410
9419 {
9420 return mImpl->getEngineCapability();
9421 }
9422
9431 {
9432 mImpl->setInt8Calibrator(calibrator);
9433 }
9434
9441 {
9442 return mImpl->getInt8Calibrator();
9443 }
9444
9457 void setFlags(BuilderFlags builderFlags) noexcept
9458 {
9459 mImpl->setFlags(builderFlags);
9460 }
9461
9469 BuilderFlags getFlags() const noexcept
9470 {
9471 return mImpl->getFlags();
9472 }
9473
9481 void clearFlag(BuilderFlag builderFlag) noexcept
9482 {
9483 mImpl->clearFlag(builderFlag);
9484 }
9485
9493 void setFlag(BuilderFlag builderFlag) noexcept
9494 {
9495 mImpl->setFlag(builderFlag);
9496 }
9497
9505 bool getFlag(BuilderFlag builderFlag) const noexcept
9506 {
9507 return mImpl->getFlag(builderFlag);
9508 }
9509
9522 void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept
9523 {
9524 mImpl->setDeviceType(layer, deviceType);
9525 }
9526
9532 DeviceType getDeviceType(ILayer const* layer) const noexcept
9533 {
9534 return mImpl->getDeviceType(layer);
9535 }
9536
9544 bool isDeviceTypeSet(ILayer const* layer) const noexcept
9545 {
9546 return mImpl->isDeviceTypeSet(layer);
9547 }
9548
9554 void resetDeviceType(ILayer const* layer) noexcept
9555 {
9556 mImpl->resetDeviceType(layer);
9557 }
9558
9564 bool canRunOnDLA(ILayer const* layer) const noexcept
9565 {
9566 return mImpl->canRunOnDLA(layer);
9567 }
9568
9580 void setDLACore(int32_t dlaCore) noexcept
9581 {
9582 mImpl->setDLACore(dlaCore);
9583 }
9584
9590 int32_t getDLACore() const noexcept
9591 {
9592 return mImpl->getDLACore();
9593 }
9594
9601 void setDefaultDeviceType(DeviceType deviceType) noexcept
9602 {
9603 mImpl->setDefaultDeviceType(deviceType);
9604 }
9605
9612 {
9613 return mImpl->getDefaultDeviceType();
9614 }
9615
9621 void reset() noexcept
9622 {
9623 mImpl->reset();
9624 }
9625
9633 void setProfileStream(const cudaStream_t stream) noexcept
9634 {
9635 return mImpl->setProfileStream(stream);
9636 }
9637
9645 cudaStream_t getProfileStream() const noexcept
9646 {
9647 return mImpl->getProfileStream();
9648 }
9649
9662 int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept
9663 {
9664 return mImpl->addOptimizationProfile(profile);
9665 }
9666
9675 int32_t getNbOptimizationProfiles() const noexcept
9676 {
9677 return mImpl->getNbOptimizationProfiles();
9678 }
9679
9688 {
9689 mImpl->setProfilingVerbosity(verbosity);
9690 }
9691
9701 {
9702 return mImpl->getProfilingVerbosity();
9703 }
9704
9713 {
9714 mImpl->setAlgorithmSelector(selector);
9715 }
9716
9723 {
9724 return mImpl->getAlgorithmSelector();
9725 }
9726
9741 {
9742 return mImpl->setCalibrationProfile(profile);
9743 }
9744
9753 {
9754 return mImpl->getCalibrationProfile();
9755 }
9756
9770 {
9771 mImpl->setQuantizationFlags(flags);
9772 }
9773
9782 {
9783 return mImpl->getQuantizationFlags();
9784 }
9785
9794 {
9795 mImpl->clearQuantizationFlag(flag);
9796 }
9797
9806 {
9807 mImpl->setQuantizationFlag(flag);
9808 }
9809
9817 bool getQuantizationFlag(QuantizationFlag flag) const noexcept
9818 {
9819 return mImpl->getQuantizationFlag(flag);
9820 }
9821
9839 bool setTacticSources(TacticSources tacticSources) noexcept
9840 {
9841 return mImpl->setTacticSources(tacticSources);
9842 }
9843
9855 {
9856 return mImpl->getTacticSources();
9857 }
9858
9873 nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept
9874 {
9875 return mImpl->createTimingCache(blob, size);
9876 }
9877
9896 bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept
9897 {
9898 return mImpl->setTimingCache(cache, ignoreMismatch);
9899 }
9900
9907 {
9908 return mImpl->getTimingCache();
9909 }
9910
9938 void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
9939 {
9940 mImpl->setMemoryPoolLimit(pool, poolSize);
9941 }
9942
9957 std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
9958 {
9959 return mImpl->getMemoryPoolLimit(pool);
9960 }
9961
9975 void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
9976 {
9977 mImpl->setPreviewFeature(feature, enable);
9978 }
9979
9989 bool getPreviewFeature(PreviewFeature feature) const noexcept
9990 {
9991 return mImpl->getPreviewFeature(feature);
9992 }
9993
10022 void setBuilderOptimizationLevel(int32_t level) noexcept
10023 {
10024 mImpl->setBuilderOptimizationLevel(level);
10025 }
10026
10035 {
10036 return mImpl->getBuilderOptimizationLevel();
10037 }
10038
10051 void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept
10052 {
10053 mImpl->setHardwareCompatibilityLevel(hardwareCompatibilityLevel);
10054 }
10055
10065 {
10066 return mImpl->getHardwareCompatibilityLevel();
10067 }
10068
10077 void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept
10078 {
10079 mImpl->setPluginsToSerialize(paths, nbPaths);
10080 }
10081
10090 char const* getPluginToSerialize(int32_t index) const noexcept
10091 {
10092 return mImpl->getPluginToSerialize(index);
10093 }
10094
10100 int32_t getNbPluginsToSerialize() const noexcept
10101 {
10102 return mImpl->getNbPluginsToSerialize();
10103 }
10104
10129 void setMaxAuxStreams(int32_t nbStreams) noexcept
10130 {
10131 mImpl->setMaxAuxStreams(nbStreams);
10132 }
10133
10139 int32_t getMaxAuxStreams() const noexcept
10140 {
10141 return mImpl->getMaxAuxStreams();
10142 }
10143
10155 void setProgressMonitor(IProgressMonitor* monitor) noexcept
10156 {
10157 return mImpl->setProgressMonitor(monitor);
10158 }
10159
10166 {
10167 return mImpl->getProgressMonitor();
10168 }
10169
10181 void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept
10182 {
10183 mImpl->setRuntimePlatform(runtimePlatform);
10184 }
10185
10194 {
10195 return mImpl->getRuntimePlatform();
10196 }
10197
10205 void setMaxNbTactics(int32_t maxNbTactics) noexcept
10206 {
10207 mImpl->setMaxNbTactics(maxNbTactics);
10208 }
10209
10217 int32_t getMaxNbTactics() const noexcept
10218 {
10219 return mImpl->getMaxNbTactics();
10220 }
10221
10234 {
10235 return mImpl->setTilingOptimizationLevel(level);
10236 }
10237
10246 {
10247 return mImpl->getTilingOptimizationLevel();
10248 }
10249
10261 bool setL2LimitForTiling(int64_t size) noexcept
10262 {
10263 return mImpl->setL2LimitForTiling(size);
10264 }
10265
10273 int64_t getL2LimitForTiling() const noexcept
10274 {
10275 return mImpl->getL2LimitForTiling();
10276 }
10277
10278protected:
10279 apiv::VBuilderConfig* mImpl;
10280};
10281
10290
10300{
10305
10310 kSTRONGLY_TYPED = 1,
10311};
10312
10318template <>
10319constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>() noexcept
10320{
10321 return 2;
10322}
10323
10331class IBuilder : public INoCopy
10332{
10333public:
10334 virtual ~IBuilder() noexcept = default;
10335
10341 TRT_DEPRECATED bool platformHasFastFp16() const noexcept
10342 {
10343 return mImpl->platformHasFastFp16();
10344 }
10345
10352 {
10353 return mImpl->platformHasFastInt8();
10354 }
10355
10363 int32_t getMaxDLABatchSize() const noexcept
10364 {
10365 return mImpl->getMaxDLABatchSize();
10366 }
10367
10371 int32_t getNbDLACores() const noexcept
10372 {
10373 return mImpl->getNbDLACores();
10374 }
10375
10388 void setGpuAllocator(IGpuAllocator* allocator) noexcept
10389 {
10390 mImpl->setGpuAllocator(allocator);
10391 }
10392
10399 {
10400 return mImpl->createBuilderConfig();
10401 }
10402
10421 {
10422 return mImpl->createNetworkV2(flags);
10423 }
10424
10436 {
10437 return mImpl->createOptimizationProfile();
10438 }
10439
10454 void setErrorRecorder(IErrorRecorder* recorder) noexcept
10455 {
10456 mImpl->setErrorRecorder(recorder);
10457 }
10458
10470 {
10471 return mImpl->getErrorRecorder();
10472 }
10473
10477 void reset() noexcept
10478 {
10479 mImpl->reset();
10480 }
10481
10487 TRT_DEPRECATED bool platformHasTf32() const noexcept
10488 {
10489 return mImpl->platformHasTf32();
10490 }
10491
10507 {
10508 return mImpl->buildSerializedNetwork(network, config);
10509 }
10510
10527 {
10528 return mImpl->buildEngineWithConfig(network, config);
10529 }
10530
10548 bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
10549 {
10550 return mImpl->isNetworkSupported(network, config);
10551 }
10552
10558 ILogger* getLogger() const noexcept
10559 {
10560 return mImpl->getLogger();
10561 }
10562
10574 bool setMaxThreads(int32_t maxThreads) noexcept
10575 {
10576 return mImpl->setMaxThreads(maxThreads);
10577 }
10578
10588 int32_t getMaxThreads() const noexcept
10589 {
10590 return mImpl->getMaxThreads();
10591 }
10592
10599 {
10600 return mImpl->getPluginRegistry();
10601 }
10602
10603protected:
10604 apiv::VBuilder* mImpl;
10605};
10606
10607} // namespace nvinfer1
10608
10613extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept;
10614
10615namespace nvinfer1
10616{
10617namespace
10618{
10619
10627inline IBuilder* createInferBuilder(ILogger& logger) noexcept
10628{
10629 return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
10630}
10631
10632} // namespace
10633
10647 nvinfer1::EngineCapability capability) noexcept;
10648
10649namespace safe
10650{
10652class IPluginRegistry;
10653} // namespace safe
10654
10662extern "C" TENSORRTAPI nvinfer1::safe::IPluginRegistry* getBuilderSafePluginRegistry(
10663 nvinfer1::EngineCapability capability) noexcept;
10664
10665} // namespace nvinfer1
10666
10667#endif // NV_INFER_H
#define TENSORRTAPI
Definition: NvInferRuntimeBase.h:59
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeBase.h:91
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:45
#define TRT_DEPRECATED_ENUM
Definition: NvInferRuntimeBase.h:46
Definition: NvInferRuntimeBase.h:203
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeBase.h:206
An Activation layer in a network definition.
Definition: NvInfer.h:1347
void setBeta(float beta) noexcept
Set the beta parameter (must be finite).
Definition: NvInfer.h:1395
void setActivationType(ActivationType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1356
ActivationType getActivationType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1366
float getAlpha() const noexcept
Get the alpha parameter.
Definition: NvInfer.h:1404
virtual ~IActivationLayer() noexcept=default
float getBeta() const noexcept
Get the beta parameter.
Definition: NvInfer.h:1413
void setAlpha(float alpha) noexcept
Set the alpha parameter (must be finite).
Definition: NvInfer.h:1381
Describes the context and requirements, that could be fulfilled by one or more instances of IAlgorith...
Definition: NvInfer.h:8492
int32_t getNbOutputs() const noexcept
Return number of outputs of the algorithm.
Definition: NvInfer.h:8527
int32_t getNbInputs() const noexcept
Return number of inputs of the algorithm.
Definition: NvInfer.h:8519
char const * getName() const noexcept
Return name of the algorithm node.
Definition: NvInfer.h:8499
virtual ~IAlgorithmContext() noexcept=default
Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for input or output tensor.
Definition: NvInfer.h:8511
Describes a variation of execution of a layer. An algorithm is represented by IAlgorithmVariant and t...
Definition: NvInfer.h:8551
std::size_t getWorkspaceSize() const noexcept
The size of the GPU temporary memory in bytes which the algorithm uses at execution time.
Definition: NvInfer.h:8572
float getTimingMSec() const noexcept
The time in milliseconds to execute the algorithm.
Definition: NvInfer.h:8564
IAlgorithmIOInfo const * getAlgorithmIOInfoByIndex(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:8586
virtual ~IAlgorithm() noexcept=default
IAlgorithmVariant const & getAlgorithmVariant() const noexcept
Returns the algorithm variant.
Definition: NvInfer.h:8556
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:8395
virtual ~IAlgorithmIOInfo() noexcept=default
int64_t getVectorizedDim() const noexcept
Return the index of the vectorized dimension or -1 for non-vectorized formats.
Definition: NvInfer.h:8423
Dims getStrides() const noexcept
Return strides of the input/output tensor of algorithm. For vectorized formats, strides are given in ...
Definition: NvInfer.h:8413
DataType getDataType() const noexcept
Return DataType of the input/output of algorithm.
Definition: NvInfer.h:8402
int64_t getComponentsPerElement() const noexcept
Return the number of components per element. This is always 1 for non-vectorized formats.
Definition: NvInfer.h:8434
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:8458
virtual ~IAlgorithmVariant() noexcept=default
int64_t getTactic() const noexcept
Return tactic of the algorithm.
Definition: NvInfer.h:8471
int64_t getImplementation() const noexcept
Return implementation of the algorithm.
Definition: NvInfer.h:8463
An assertion layer in a network.
Definition: NvInfer.h:4940
void setMessage(char const *message) noexcept
Set the message to print if the assertion fails.
Definition: NvInfer.h:4950
char const * getMessage() const noexcept
Return the assertion message.
Definition: NvInfer.h:4960
virtual ~IAssertionLayer() noexcept=default
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:9369
void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
Set the memory size for the memory pool.
Definition: NvInfer.h:9938
void setQuantizationFlag(QuantizationFlag flag) noexcept
Set a single quantization flag.
Definition: NvInfer.h:9805
nvinfer1::ITimingCache * createTimingCache(void const *blob, std::size_t size) const noexcept
Create timing cache.
Definition: NvInfer.h:9873
void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
Enable or disable a specific preview feature.
Definition: NvInfer.h:9975
TRT_DEPRECATED void setAlgorithmSelector(IAlgorithmSelector *selector) noexcept
Set Algorithm Selector.
Definition: NvInfer.h:9712
TRT_DEPRECATED void setInt8Calibrator(IInt8Calibrator *calibrator) noexcept
Set Int8 Calibration interface.
Definition: NvInfer.h:9430
bool getPreviewFeature(PreviewFeature feature) const noexcept
Get status of preview feature.
Definition: NvInfer.h:9989
void clearQuantizationFlag(QuantizationFlag flag) noexcept
clear a quantization flag.
Definition: NvInfer.h:9793
int32_t getBuilderOptimizationLevel() noexcept
Get builder optimization level.
Definition: NvInfer.h:10034
bool setTacticSources(TacticSources tacticSources) noexcept
Set tactic sources.
Definition: NvInfer.h:9839
void setPluginsToSerialize(char const *const *paths, int32_t nbPaths) noexcept
Set the plugin libraries to be serialized with version-compatible engines.
Definition: NvInfer.h:10077
bool setTilingOptimizationLevel(TilingOptimizationLevel level) noexcept
Set the Tiling optimization level.
Definition: NvInfer.h:10233
bool setL2LimitForTiling(int64_t size) noexcept
Set the L2 cache usage limit for Tiling optimization.
Definition: NvInfer.h:10261
bool getQuantizationFlag(QuantizationFlag flag) const noexcept
Returns true if the quantization flag is set.
Definition: NvInfer.h:9817
TRT_DEPRECATED IInt8Calibrator * getInt8Calibrator() const noexcept
Get Int8 Calibration interface.
Definition: NvInfer.h:9440
std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
Get the memory size limit of the memory pool.
Definition: NvInfer.h:9957
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInfer.h:9590
int32_t getNbPluginsToSerialize() const noexcept
Get the number of plugin library paths to be serialized with version-compatible engines.
Definition: NvInfer.h:10100
void setDeviceType(ILayer const *layer, DeviceType deviceType) noexcept
Set the device that this layer must execute on.
Definition: NvInfer.h:9522
void setEngineCapability(EngineCapability capability) noexcept
Configure the builder to target specified EngineCapability flow.
Definition: NvInfer.h:9406
int32_t getMaxAuxStreams() const noexcept
Get the maximum number of auxiliary streams that TRT is allowed to use.
Definition: NvInfer.h:10139
bool getFlag(BuilderFlag builderFlag) const noexcept
Returns true if the build mode flag is set.
Definition: NvInfer.h:9505
void setQuantizationFlags(QuantizationFlags flags) noexcept
Set the quantization flags.
Definition: NvInfer.h:9769
void setMaxNbTactics(int32_t maxNbTactics) noexcept
Set the maximum number of tactics to time when there is a choice of tactics.
Definition: NvInfer.h:10205
int64_t getL2LimitForTiling() const noexcept
Get the L2 cache usage limit for tiling optimization.
Definition: NvInfer.h:10273
void setProgressMonitor(IProgressMonitor *monitor) noexcept
Sets the progress monitor for building a network.
Definition: NvInfer.h:10155
void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:9687
int32_t getNbOptimizationProfiles() const noexcept
Get number of optimization profiles.
Definition: NvInfer.h:9675
QuantizationFlags getQuantizationFlags() const noexcept
Get the quantization flags.
Definition: NvInfer.h:9781
nvinfer1::ITimingCache const * getTimingCache() const noexcept
Get the pointer to the timing cache from current IBuilderConfig.
Definition: NvInfer.h:9906
void reset() noexcept
Resets the builder configuration to defaults.
Definition: NvInfer.h:9621
bool setTimingCache(ITimingCache const &cache, bool ignoreMismatch) noexcept
Attach a timing cache to IBuilderConfig.
Definition: NvInfer.h:9896
char const * getPluginToSerialize(int32_t index) const noexcept
Get the plugin library path to be serialized with version-compatible engines.
Definition: NvInfer.h:10090
EngineCapability getEngineCapability() const noexcept
Query EngineCapability flow configured for the builder.
Definition: NvInfer.h:9418
RuntimePlatform getRuntimePlatform() const noexcept
Get the target platform for runtime execution.
Definition: NvInfer.h:10193
DeviceType getDefaultDeviceType() const noexcept
Get the default DeviceType which was set by setDefaultDeviceType.
Definition: NvInfer.h:9611
void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept
Set the target platform for runtime execution.
Definition: NvInfer.h:10181
int32_t getMaxNbTactics() const noexcept
Query the maximum number of tactics timed when there is a choice.
Definition: NvInfer.h:10217
BuilderFlags getFlags() const noexcept
Get the build mode flags for this builder config. Defaults to 0.
Definition: NvInfer.h:9469
void setFlags(BuilderFlags builderFlags) noexcept
Set the build mode flags to turn on builder options for this network.
Definition: NvInfer.h:9457
TacticSources getTacticSources() const noexcept
Get tactic sources.
Definition: NvInfer.h:9854
void resetDeviceType(ILayer const *layer) noexcept
reset the DeviceType for this layer
Definition: NvInfer.h:9554
void setDLACore(int32_t dlaCore) noexcept
Sets the DLA core used by the network. Defaults to -1.
Definition: NvInfer.h:9580
HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept
Get the hardware compatibility level.
Definition: NvInfer.h:10064
void clearFlag(BuilderFlag builderFlag) noexcept
clear a single build mode flag.
Definition: NvInfer.h:9481
int32_t addOptimizationProfile(IOptimizationProfile const *profile) noexcept
Add an optimization profile.
Definition: NvInfer.h:9662
IProgressMonitor * getProgressMonitor() const noexcept
Definition: NvInfer.h:10165
apiv::VBuilderConfig * mImpl
Definition: NvInfer.h:10279
TRT_DEPRECATED IOptimizationProfile const * getCalibrationProfile() noexcept
Get the current calibration profile.
Definition: NvInfer.h:9752
int32_t getAvgTimingIterations() const noexcept
Query the number of averaging iterations.
Definition: NvInfer.h:9393
void setDefaultDeviceType(DeviceType deviceType) noexcept
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
Definition: NvInfer.h:9601
void setFlag(BuilderFlag builderFlag) noexcept
Set a single build mode flag.
Definition: NvInfer.h:9493
TRT_DEPRECATED bool setCalibrationProfile(IOptimizationProfile const *profile) noexcept
Add a calibration profile.
Definition: NvInfer.h:9740
virtual ~IBuilderConfig() noexcept=default
DeviceType getDeviceType(ILayer const *layer) const noexcept
Get the device that this layer executes on.
Definition: NvInfer.h:9532
bool canRunOnDLA(ILayer const *layer) const noexcept
Checks if a layer can run on DLA.
Definition: NvInfer.h:9564
cudaStream_t getProfileStream() const noexcept
Get the CUDA stream that is used to profile this network.
Definition: NvInfer.h:9645
void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept
Set the hardware compatibility level.
Definition: NvInfer.h:10051
TilingOptimizationLevel getTilingOptimizationLevel() const noexcept
Get the Tiling optimization level.
Definition: NvInfer.h:10245
void setMaxAuxStreams(int32_t nbStreams) noexcept
Set the maximum number of auxiliary streams that TRT is allowed to use.
Definition: NvInfer.h:10129
ProfilingVerbosity getProfilingVerbosity() const noexcept
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:9700
bool isDeviceTypeSet(ILayer const *layer) const noexcept
whether the DeviceType has been explicitly set for this layer
Definition: NvInfer.h:9544
void setBuilderOptimizationLevel(int32_t level) noexcept
Set builder optimization level.
Definition: NvInfer.h:10022
void setProfileStream(const cudaStream_t stream) noexcept
Set the CUDA stream that is used to profile this network.
Definition: NvInfer.h:9633
TRT_DEPRECATED IAlgorithmSelector * getAlgorithmSelector() const noexcept
Get Algorithm Selector.
Definition: NvInfer.h:9722
Builds an engine from a network definition.
Definition: NvInfer.h:10332
int32_t getMaxDLABatchSize() const noexcept
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Definition: NvInfer.h:10363
int32_t getNbDLACores() const noexcept
Return the number of DLA engines available to this builder.
Definition: NvInfer.h:10371
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:10469
apiv::VBuilder * mImpl
Definition: NvInfer.h:10604
ILogger * getLogger() const noexcept
get the logger with which the builder was created
Definition: NvInfer.h:10558
bool isNetworkSupported(INetworkDefinition const &network, IBuilderConfig const &config) const noexcept
Checks that a network is within the scope of the IBuilderConfig settings.
Definition: NvInfer.h:10548
int32_t getMaxThreads() const noexcept
get the maximum number of threads that can be used by the builder.
Definition: NvInfer.h:10588
IPluginRegistry & getPluginRegistry() noexcept
get the local plugin registry that can be used by the builder.
Definition: NvInfer.h:10598
TRT_DEPRECATED bool platformHasFastInt8() const noexcept
Determine whether the platform has fast native int8.
Definition: NvInfer.h:10351
nvinfer1::IOptimizationProfile * createOptimizationProfile() noexcept
Create a new optimization profile.
Definition: NvInfer.h:10435
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInfer.h:10388
nvinfer1::INetworkDefinition * createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept
Create a network definition object.
Definition: NvInfer.h:10420
nvinfer1::IBuilderConfig * createBuilderConfig() noexcept
Create a builder configuration object.
Definition: NvInfer.h:10398
void reset() noexcept
Resets the builder state to default values.
Definition: NvInfer.h:10477
bool setMaxThreads(int32_t maxThreads) noexcept
Set the maximum number of threads.
Definition: NvInfer.h:10574
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:10454
nvinfer1::IHostMemory * buildSerializedNetwork(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds and serializes a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:10506
virtual ~IBuilder() noexcept=default
TRT_DEPRECATED bool platformHasTf32() const noexcept
Determine whether the platform has TF32 support.
Definition: NvInfer.h:10487
nvinfer1::ICudaEngine * buildEngineWithConfig(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:10526
A cast layer in a network.
Definition: NvInfer.h:3807
virtual ~ICastLayer() noexcept=default
apiv::VCastLayer * mImpl
Definition: NvInfer.h:3833
DataType getToType() const noexcept
Return cast layer output type.
Definition: NvInfer.h:3827
void setToType(DataType toType) noexcept
Set cast layer output type.
Definition: NvInfer.h:3816
A concatenation layer in a network definition.
Definition: NvInfer.h:2057
void setAxis(int32_t axis) noexcept
Set the axis along which concatenation occurs.
Definition: NvInfer.h:2070
int32_t getAxis() const noexcept
Get the axis along which concatenation occurs.
Definition: NvInfer.h:2080
virtual ~IConcatenationLayer() noexcept=default
This layer represents a condition input to an IIfConditional.
Definition: NvInfer.h:4466
virtual ~IConditionLayer() noexcept=default
Layer that represents a constant value.
Definition: NvInfer.h:3846
void setWeights(Weights weights) noexcept
Set the weights for the layer.
Definition: NvInfer.h:3856
Weights getWeights() const noexcept
Get the weights for the layer.
Definition: NvInfer.h:3866
void setDimensions(Dims const &dimensions) noexcept
Set the dimensions for the layer.
Definition: NvInfer.h:3878
apiv::VConstantLayer * mImpl
Definition: NvInfer.h:3896
virtual ~IConstantLayer() noexcept=default
Dims getDimensions() const noexcept
Get the dimensions for the layer.
Definition: NvInfer.h:3890
A convolution layer in a network definition.
Definition: NvInfer.h:1027
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1152
Weights getBiasWeights() const noexcept
Get the bias weights for the convolution.
Definition: NvInfer.h:1125
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1193
void setDilationNd(Dims const &dilation) noexcept
Set the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1297
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the convolution.
Definition: NvInfer.h:1283
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the convolution.
Definition: NvInfer.h:1253
Weights getKernelWeights() const noexcept
Get the kernel weights of the convolution.
Definition: NvInfer.h:1100
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride of the convolution.
Definition: NvInfer.h:1243
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1307
int64_t getNbOutputMaps() const noexcept
Get the number of output maps for the convolution.
Definition: NvInfer.h:1046
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the convolution.
Definition: NvInfer.h:1090
Dims getPostPadding() const noexcept
Get the post-padding.
Definition: NvInfer.h:1179
int64_t getNbGroups() const noexcept
Get the number of groups of the convolution.
Definition: NvInfer.h:1076
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1205
virtual ~IConvolutionLayer() noexcept=default
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups for a convolution.
Definition: NvInfer.h:1066
void setNbOutputMaps(int64_t nbOutputMaps) noexcept
Set the number of output maps for the convolution.
Definition: NvInfer.h:1036
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the convolution.
Definition: NvInfer.h:1115
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1228
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding of the convolution.
Definition: NvInfer.h:1271
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding of the convolution.
Definition: NvInfer.h:1142
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding of the convolution.
Definition: NvInfer.h:1169
void setKernelSizeNd(Dims const &kernelSize) noexcept
Set the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1218
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:3002
Layer that represents a cumulative operation across a tensor.
Definition: NvInfer.h:6529
bool setOperation(CumulativeOperation op) noexcept
Set the cumulative operation for the layer.
Definition: NvInfer.h:6540
void setReverse(bool reverse) noexcept
Specify whether the cumulative operation should be applied backward.
Definition: NvInfer.h:6588
apiv::VCumulativeLayer * mImpl
Definition: NvInfer.h:6606
bool getExclusive() const noexcept
Get whether it is exclusive accumulation or inclusive accumulation.
Definition: NvInfer.h:6576
virtual ~ICumulativeLayer() noexcept=default
bool getReverse() const noexcept
Get the boolean that specifies whether the cumulative operation should be applied backward.
Definition: NvInfer.h:6600
void setExclusive(bool exclusive) noexcept
Set whether it is an exclusive accumulation or inclusive accumulation.
Definition: NvInfer.h:6564
CumulativeOperation getOperation() const noexcept
Get the cumulative operation for the layer.
Definition: NvInfer.h:6552
A deconvolution layer in a network definition.
Definition: NvInfer.h:2098
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the deconvolution.
Definition: NvInfer.h:2186
int64_t getNbGroups() const noexcept
Get the number of groups for a deconvolution.
Definition: NvInfer.h:2147
Weights getKernelWeights() const noexcept
Get the kernel weights for the deconvolution.
Definition: NvInfer.h:2171
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding of the deconvolution.
Definition: NvInfer.h:2213
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2328
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2394
Weights getBiasWeights() const noexcept
Get the bias weights for the deconvolution.
Definition: NvInfer.h:2196
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the deconvolution.
Definition: NvInfer.h:2161
int64_t getNbOutputMaps() const noexcept
Get the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2117
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2318
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:2250
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2301
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding of the deconvolution.
Definition: NvInfer.h:2240
void setKernelSizeNd(Dims const &kernelSize) noexcept
Set the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2291
virtual ~IDeconvolutionLayer() noexcept=default
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2346
void setNbOutputMaps(int64_t nbOutputMaps) noexcept
Set the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2107
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2358
void setDilationNd(Dims const &dilation) noexcept
Set the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2384
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:2264
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups for a deconvolution.
Definition: NvInfer.h:2137
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:2223
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:2276
A Dequantize layer in a network definition.
Definition: NvInfer.h:5528
void setToType(DataType toType) noexcept
Set the Dequantize layer output type.
Definition: NvInfer.h:5565
virtual ~IDequantizeLayer() noexcept=default
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5538
DataType getToType() const noexcept
Return the Dequantize layer output type.
Definition: NvInfer.h:5577
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5549
A network layer to perform dynamic quantization.
Definition: NvInfer.h:5605
int32_t getAxis() const noexcept
Get the axis along which blocking occurs.
Definition: NvInfer.h:5693
int32_t getBlockSize() const noexcept
Get the size of the quantization block.
Definition: NvInfer.h:5716
DataType getScaleType() const noexcept
Return the scale factors data type.
Definition: NvInfer.h:5670
void setScaleType(DataType scaleType) noexcept
Set the data type of the scale factors used to quantize the data.
Definition: NvInfer.h:5657
DataType getToType() const noexcept
Return DynamicQuantizeLayer’s quantized output type.
Definition: NvInfer.h:5645
virtual ~IDynamicQuantizeLayer() noexcept=default
void setToType(DataType toType) noexcept
Set DynamicQuantizeLayer’s quantized output type.
Definition: NvInfer.h:5632
void setAxis(int32_t axis) noexcept
Set the axis along which block quantization occurs.
Definition: NvInfer.h:5683
void setBlockSize(int32_t size) noexcept
Set the size of the quantization block.
Definition: NvInfer.h:5706
An Einsum layer in a network.
Definition: NvInfer.h:5763
bool setEquation(char const *equation) noexcept
Set the equation. The equation is a comma-separated list of subscript labels, where each label refers...
Definition: NvInfer.h:5774
virtual ~IEinsumLayer() noexcept=default
char const * getEquation() const noexcept
Return the equation.
Definition: NvInfer.h:5784
A elementwise layer in a network definition.
Definition: NvInfer.h:2470
virtual ~IElementWiseLayer() noexcept=default
apiv::VElementWiseLayer * mImpl
Definition: NvInfer.h:2499
ElementWiseOperation getOperation() const noexcept
Get the binary operation for the layer.
Definition: NvInfer.h:2493
void setOperation(ElementWiseOperation op) noexcept
Set the binary operation for the layer.
Definition: NvInfer.h:2481
Generate a tensor according to a specified mode.
Definition: NvInfer.h:5051
bool isAlphaBetaInt64() const noexcept
Return true if alpha/beta have type int64, false if they have type double.
Definition: NvInfer.h:5283
FillOperation getOperation() const noexcept
Get the fill operation for the layer.
Definition: NvInfer.h:5097
void setOperation(FillOperation op) noexcept
Set the fill operation for the layer.
Definition: NvInfer.h:5087
DataType getToType() const noexcept
Get the fill layer output type.
Definition: NvInfer.h:5312
void setAlphaInt64(int64_t alpha) noexcept
Set the alpha parameter with int64 datatype.
Definition: NvInfer.h:5226
void setBetaInt64(int64_t beta) noexcept
Set the beta parameter with int64 datatype.
Definition: NvInfer.h:5260
void setBeta(double beta) noexcept
Set the beta parameter.
Definition: NvInfer.h:5150
int64_t getAlphaInt64() const noexcept
Get the value of alpha parameter with int64 datatype.
Definition: NvInfer.h:5241
int64_t getBetaInt64() const noexcept
Get the value of beta parameter with int64 datatype.
Definition: NvInfer.h:5275
double getAlpha() const noexcept
Get the value of alpha parameter.
Definition: NvInfer.h:5131
void setDimensions(Dims const &dimensions) noexcept
Set the output tensor's dimensions.
Definition: NvInfer.h:5062
void setAlpha(double alpha) noexcept
Set the alpha parameter.
Definition: NvInfer.h:5116
void setToType(DataType toType) noexcept
Set the fill layer output type.
Definition: NvInfer.h:5300
Dims getDimensions() const noexcept
Get the output tensor's dimensions.
Definition: NvInfer.h:5077
double getBeta() const noexcept
Get the value of beta parameter.
Definition: NvInfer.h:5165
virtual ~IFillLayer() noexcept=default
A Gather layer in a network definition. Supports several kinds of gathering.
Definition: NvInfer.h:2603
void setGatherAxis(int32_t axis) noexcept
Set the axis used by GatherMode::kELEMENTS and GatherMode::kDEFAULT The axis must be less than the nu...
Definition: NvInfer.h:2614
void setNbElementWiseDims(int32_t elementWiseDims) noexcept
Set the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:2649
apiv::VGatherLayer * mImpl
Definition: NvInfer.h:2685
int32_t getNbElementWiseDims() const noexcept
Get the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:2659
void setMode(GatherMode mode) noexcept
Set the gather mode.
Definition: NvInfer.h:2669
int32_t getGatherAxis() const noexcept
Get the axis to gather on.
Definition: NvInfer.h:2626
GatherMode getMode() const noexcept
Get the gather mode.
Definition: NvInfer.h:2679
virtual ~IGatherLayer() noexcept=default
A GridSample layer in a network definition.
Definition: NvInfer.h:5984
void setInterpolationMode(InterpolationMode mode) noexcept
Set the grid sample interpolation mode.
Definition: NvInfer.h:5991
bool setSampleMode(SampleMode mode) noexcept
Set the sample mode.
Definition: NvInfer.h:6037
void setAlignCorners(bool alignCorners) noexcept
Set the align corners mode.
Definition: NvInfer.h:6013
apiv::VGridSampleLayer * mImpl
Definition: NvInfer.h:6055
SampleMode getSampleMode() const noexcept
Get the sample mode.
Definition: NvInfer.h:6049
InterpolationMode getInterpolationMode() const noexcept
Get the grid sample interpolation mode.
Definition: NvInfer.h:6003
bool getAlignCorners() const noexcept
Get the align corners mode.
Definition: NvInfer.h:6025
virtual ~IGridSampleLayer() noexcept=default
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:142
A layer that represents the identity function.
Definition: NvInfer.h:3794
apiv::VIdentityLayer * mImpl
Definition: NvInfer.h:3796
virtual ~IIdentityLayer() noexcept=default
This is a base class for Conditional boundary layers.
Definition: NvInfer.h:4445
IIfConditional * getConditional() const noexcept
Get a pointer to the IIfConditional associated with this boundary layer.
Definition: NvInfer.h:4450
virtual ~IIfConditionalBoundaryLayer() noexcept=default
Helper for constructing conditionally-executed subgraphs.
Definition: NvInfer.h:4527
IIfConditionalInputLayer * addInput(ITensor &input) noexcept
Add an If-conditional input.
Definition: NvInfer.h:4568
char const * getName() const noexcept
Return the name of the conditional.
Definition: NvInfer.h:4593
virtual ~IIfConditional() noexcept=default
IConditionLayer * setCondition(ITensor &condition) noexcept
Set the condition tensor for this If-Conditional construct.
Definition: NvInfer.h:4538
IIfConditionalOutputLayer * addOutput(ITensor &trueSubgraphOutput, ITensor &falseSubgraphOutput) noexcept
Add an If-conditional output.
Definition: NvInfer.h:4556
void setName(char const *name) noexcept
Set the name of the conditional.
Definition: NvInfer.h:4583
This layer represents an input to an IIfConditional.
Definition: NvInfer.h:4496
virtual ~IIfConditionalInputLayer() noexcept=default
This layer represents an output of an IIfConditional.
Definition: NvInfer.h:4483
virtual ~IIfConditionalOutputLayer() noexcept=default
Application-implemented interface for calibration.
Definition: NvInfer.h:8120
virtual TRT_DEPRECATED int32_t getBatchSize() const noexcept=0
Get the batch size used for calibration batches.
A layer to do iterations.
Definition: NvInfer.h:4758
virtual ~IIteratorLayer() noexcept=default
void setReverse(bool reverse) noexcept
Set iteration order to be reverse.
Definition: NvInfer.h:4785
bool getReverse() const noexcept
Check if the iteration order is reverse.
Definition: NvInfer.h:4795
int32_t getAxis() const noexcept
Get axis being iterated over.
Definition: NvInfer.h:4771
void setAxis(int32_t axis) noexcept
Set axis to iterate over.
Definition: NvInfer.h:4763
A LRN layer in a network definition.
Definition: NvInfer.h:1712
int64_t getWindowSize() const noexcept
Get the LRN window size.
Definition: NvInfer.h:1733
float getAlpha() const noexcept
Get the LRN alpha value.
Definition: NvInfer.h:1755
void setWindowSize(int64_t windowSize) noexcept
Set the LRN window size.
Definition: NvInfer.h:1723
void setK(float k) noexcept
Set the LRN K value.
Definition: NvInfer.h:1789
void setAlpha(float alpha) noexcept
Set the LRN alpha value.
Definition: NvInfer.h:1745
void setBeta(float beta) noexcept
Set the LRN beta value.
Definition: NvInfer.h:1767
virtual ~ILRNLayer() noexcept=default
float getBeta() const noexcept
Get the LRN beta value.
Definition: NvInfer.h:1777
float getK() const noexcept
Get the LRN K value.
Definition: NvInfer.h:1799
Base class for all layer classes in a network definition.
Definition: NvInfer.h:555
bool precisionIsSet() const noexcept
whether the computational precision has been set for this layer
Definition: NvInfer.h:697
void setMetadata(char const *metadata) noexcept
Set the metadata for this layer.
Definition: NvInfer.h:813
void setPrecision(DataType dataType) noexcept
Set the preferred or required computational precision of this layer in a weakly-typed network.
Definition: NvInfer.h:673
void setName(char const *name) noexcept
Set the name of a layer.
Definition: NvInfer.h:576
void resetPrecision() noexcept
reset the computational precision for this layer
Definition: NvInfer.h:707
int32_t getNbInputs() const noexcept
Get the number of inputs of a layer.
Definition: NvInfer.h:594
char const * getMetadata() const noexcept
Get the metadata of the layer.
Definition: NvInfer.h:826
DataType getOutputType(int32_t index) const noexcept
get the output type of this layer
Definition: NvInfer.h:769
DataType getPrecision() const noexcept
get the computational precision of this layer
Definition: NvInfer.h:685
char const * getName() const noexcept
Return the name of a layer.
Definition: NvInfer.h:586
int32_t getNbOutputs() const noexcept
Get the number of outputs of a layer.
Definition: NvInfer.h:615
bool outputTypeIsSet(int32_t index) const noexcept
whether the output type has been set for this layer
Definition: NvInfer.h:783
ITensor * getOutput(int32_t index) const noexcept
Get the layer output corresponding to the given index.
Definition: NvInfer.h:625
void setInput(int32_t index, ITensor &tensor) noexcept
Replace an input of this layer with a specific tensor.
Definition: NvInfer.h:642
void resetOutputType(int32_t index) noexcept
reset the output type for this layer
Definition: NvInfer.h:795
ITensor * getInput(int32_t index) const noexcept
Get the layer input corresponding to the given index.
Definition: NvInfer.h:607
void setOutputType(int32_t index, DataType dataType) noexcept
Set the output type of this layer in a weakly-typed network.
Definition: NvInfer.h:754
LayerType getType() const noexcept
Return the type of a layer.
Definition: NvInfer.h:562
virtual ~ILayer() noexcept=default
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntime.h:1540
This is a base class for Loop boundary layers.
Definition: NvInfer.h:4422
virtual ~ILoopBoundaryLayer() noexcept=default
ILoop * getLoop() const noexcept
Get a pointer to ILoop associated with this boundary layer.
Definition: NvInfer.h:4427
Helper for creating a recurrent subgraph.
Definition: NvInfer.h:4815
void setName(char const *name) noexcept
Set the name of the loop.
Definition: NvInfer.h:4885
ITripLimitLayer * addTripLimit(ITensor &tensor, TripLimit limit) noexcept
Add a trip-count limiter, based on the given tensor.
Definition: NvInfer.h:4844
IIteratorLayer * addIterator(ITensor &tensor, int32_t axis=0, bool reverse=false) noexcept
Return layer that subscripts tensor by loop iteration.
Definition: NvInfer.h:4857
ILoopOutputLayer * addLoopOutput(ITensor &tensor, LoopOutput outputKind, int32_t axis=0) noexcept
Make an output for this loop, based on the given tensor.
Definition: NvInfer.h:4870
virtual ~ILoop() noexcept=default
char const * getName() const noexcept
Return the name of the loop.
Definition: NvInfer.h:4895
IRecurrenceLayer * addRecurrence(ITensor &initialValue) noexcept
Create a recurrence layer for this loop with initialValue as its first input.
Definition: NvInfer.h:4823
An ILoopOutputLayer is the sole way to get output from a loop.
Definition: NvInfer.h:4658
virtual ~ILoopOutputLayer() noexcept=default
int32_t getAxis() const noexcept
Get axis being concatenated over.
Definition: NvInfer.h:4688
LoopOutput getLoopOutput() const noexcept
Get which kind a loop output has.
Definition: NvInfer.h:4663
void setAxis(int32_t axis) noexcept
Set where to insert the contenation axis. Ignored if getLoopOutput() is kLAST_VALUE.
Definition: NvInfer.h:4680
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:3686
apiv::VMatrixMultiplyLayer * mImpl
Definition: NvInfer.h:3714
virtual ~IMatrixMultiplyLayer() noexcept=default
MatrixOperation getOperation(int32_t index) const noexcept
Get the operation for an input tensor.
Definition: NvInfer.h:3708
void setOperation(int32_t index, MatrixOperation op) noexcept
Set the operation for an input tensor.
Definition: NvInfer.h:3696
A non-maximum suppression layer in a network definition.
Definition: NvInfer.h:6132
virtual ~INMSLayer() noexcept=default
void setTopKBoxLimit(int32_t limit) noexcept
Set the TopK box limit parameter for the layer.
Definition: NvInfer.h:6169
void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept
Set the bounding box format parameter for the layer.
Definition: NvInfer.h:6143
BoundingBoxFormat getBoundingBoxFormat() const noexcept
Get the bounding box format parameter for the layer.
Definition: NvInfer.h:6155
apiv::VNMSLayer * mImpl
Definition: NvInfer.h:6205
int32_t getTopKBoxLimit() const noexcept
Get the TopK box limit parameter for the layer.
Definition: NvInfer.h:6179
A network definition for input to the builder.
Definition: NvInfer.h:6628
IConcatenationLayer * addConcatenation(ITensor *const *inputs, int32_t nbInputs) noexcept
Add a concatenation layer to the network.
Definition: NvInfer.h:6821
IShuffleLayer * addShuffle(ITensor &input) noexcept
Add a shuffle layer to the network.
Definition: NvInfer.h:6884
INormalizationLayer * addNormalization(ITensor &input, ITensor &scale, ITensor &bias, uint32_t axesMask) noexcept
Add a normalization layer to the network.
Definition: NvInfer.h:7960
void setName(char const *name) noexcept
Sets the name of the network.
Definition: NvInfer.h:7294
bool markDebug(ITensor &tensor) noexcept
Mark a tensor as a debug tensor.
Definition: NvInfer.h:6700
ILRNLayer * addLRN(ITensor &input, int64_t window, float alpha, float beta, float k) noexcept
Add a LRN layer to the network.
Definition: NvInfer.h:6765
ITopKLayer * addTopK(ITensor &input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
Add a TopK layer to the network.
Definition: NvInfer.h:7044
ICumulativeLayer * addCumulative(ITensor &input, ITensor &axis, CumulativeOperation operation, bool exclusive, bool reverse) noexcept
Add a cumulative layer to the network.
Definition: NvInfer.h:7982
IAssertionLayer * addAssertion(ITensor &condition, char const *message) noexcept
Add an assertion layer to the network.
Definition: NvInfer.h:7610
IConvolutionLayer * addConvolutionNd(ITensor &input, int64_t nbOutputMaps, Dims const &kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension convolution layer to the network.
Definition: NvInfer.h:7429
ICastLayer * addCast(ITensor &input, DataType toType) noexcept
Add a cast layer.
Definition: NvInfer.h:7184
IScaleLayer * addScaleNd(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
Add a multi-dimension scale layer to the network.
Definition: NvInfer.h:7508
char const * getName() const noexcept
Returns the name associated with the network.
Definition: NvInfer.h:7308
IParametricReLULayer * addParametricReLU(ITensor &input, ITensor &slope) noexcept
Add a parametric ReLU layer to the network.
Definition: NvInfer.h:7407
ITensor * getOutput(int32_t index) const noexcept
Get the output tensor specified by the given index.
Definition: NvInfer.h:6985
ITensor * getInput(int32_t index) const noexcept
Get the input tensor specified by the given index.
Definition: NvInfer.h:6955
IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale, DataType outputType) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7777
bool unmarkOutputForShapes(ITensor &tensor) noexcept
Undo markOutputForShapes.
Definition: NvInfer.h:7389
IFillLayer * addFill(Dims const &dimensions, FillOperation op, DataType outputType) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7661
ILoop * addLoop() noexcept
Add a loop to the network.
Definition: NvInfer.h:7539
IDynamicQuantizeLayer * addDynamicQuantize(ITensor &input, int32_t axis, int32_t blockSize, DataType outputType, DataType scaleType) noexcept
Add a dynamic quantization layer to the network.
Definition: NvInfer.h:7865
IActivationLayer * addActivation(ITensor &input, ActivationType type) noexcept
Add an activation layer to the network.
Definition: NvInfer.h:6746
TRT_DEPRECATED IFillLayer * addFill(Dims const &dimensions, FillOperation op) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7635
ISliceLayer * addSlice(ITensor &input, Dims const &start, Dims const &size, Dims const &stride) noexcept
Add a slice layer to the network.
Definition: NvInfer.h:7270
virtual ~INetworkDefinition() noexcept=default
TRT_DEPRECATED IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7818
virtual IBuilder & getBuilder() const noexcept
Return the builder from which this INetworkDefinition was created.
Definition: NvInfer.h:7993
INMSLayer * addNMS(ITensor &boxes, ITensor &scores, ITensor &maxOutputBoxesPerClass) noexcept
Add a non-maximum suppression layer to the network.
Definition: NvInfer.h:7917
ILayer * getLayer(int32_t index) const noexcept
Get the layer specified by the given index.
Definition: NvInfer.h:6927
bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept
Returns true if the network definition creation flag is set.
Definition: NvInfer.h:7360
IIfConditional * addIfConditional() noexcept
Add an if-then-else to the network.
Definition: NvInfer.h:7554
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:7735
ISqueezeLayer * addSqueeze(ITensor &input, ITensor &axes) noexcept
Add a squeeze layer to the network.
Definition: NvInfer.h:8050
IReverseSequenceLayer * addReverseSequence(ITensor &input, ITensor &sequenceLens) noexcept
Add a ReverseSequence layer to the network.
Definition: NvInfer.h:7934
int32_t getNbInputs() const noexcept
Get the number of inputs in the network.
Definition: NvInfer.h:6939
NetworkDefinitionCreationFlags getFlags() const noexcept
Get the network definition creation flags for this network definition object. Defaults to 0.
Definition: NvInfer.h:7348
IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale, DataType outputType) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7839
IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
Add a reduce layer to the network.
Definition: NvInfer.h:7011
IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation) noexcept
Add a unary layer to the network.
Definition: NvInfer.h:6870
IGridSampleLayer * addGridSample(ITensor &input, ITensor &grid) noexcept
Add a GridSample layer to the network.
Definition: NvInfer.h:7899
void removeTensor(ITensor &tensor) noexcept
remove a tensor from the network definition.
Definition: NvInfer.h:7199
bool areWeightsMarkedRefittable(char const *name) const noexcept
Whether the weight has been marked as refittable.
Definition: NvInfer.h:8031
ISelectLayer * addSelect(ITensor &condition, ITensor &thenInput, ITensor &elseInput) noexcept
Add a select layer to the network.
Definition: NvInfer.h:7593
IScatterLayer * addScatter(ITensor &data, ITensor &indices, ITensor &updates, ScatterMode mode) noexcept
Add a Scatter layer to the network with specified mode and axis=0.
Definition: NvInfer.h:7797
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInfer.h:6913
TRT_DEPRECATED bool hasImplicitBatchDimension() const noexcept
Query whether the network was created with an implicit batch dimension.
Definition: NvInfer.h:7338
apiv::VNetworkDefinition * mImpl
Definition: NvInfer.h:8077
bool markOutputForShapes(ITensor &tensor) noexcept
Enable tensor's value to be computed by IExecutionContext::getShapeBinding.
Definition: NvInfer.h:7377
IOneHotLayer * addOneHot(ITensor &indices, ITensor &values, ITensor &depth, int32_t axis) noexcept
Add a OneHot layer to the network.
Definition: NvInfer.h:6901
IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
Add a Scale layer to the network.
Definition: NvInfer.h:6791
IPluginV3Layer * addPluginV3(ITensor *const *inputs, int32_t nbInputs, ITensor *const *shapeInputs, int32_t nbShapeInputs, IPluginV3 &plugin) noexcept
Add a plugin layer implementing the IPluginV3 interface to the network.
Definition: NvInfer.h:7250
void unmarkOutput(ITensor &tensor) noexcept
unmark a tensor as a network output.
Definition: NvInfer.h:7211
IIdentityLayer * addIdentity(ITensor &input) noexcept
Add an identity layer.
Definition: NvInfer.h:7169
IGatherLayer * addGatherV2(ITensor &data, ITensor &indices, GatherMode mode) noexcept
Add gather with specified mode, axis=0 and nbElementWiseDims=0.
Definition: NvInfer.h:7076
IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op) noexcept
Add an elementwise layer to the network.
Definition: NvInfer.h:6848
IConstantLayer * addConstant(Dims const &dimensions, Weights weights) noexcept
Add a constant layer to the network.
Definition: NvInfer.h:7155
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:7720
IPoolingLayer * addPoolingNd(ITensor &input, PoolingType type, Dims const &windowSize) noexcept
Add a multi-dimension pooling layer to the network.
Definition: NvInfer.h:7449
IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds) noexcept
Add a RaggedSoftMax layer to the network.
Definition: NvInfer.h:7095
IShapeLayer * addShape(ITensor &input) noexcept
Add a shape layer to the network.
Definition: NvInfer.h:7324
IGatherLayer * addGather(ITensor &data, ITensor &indices, int32_t axis) noexcept
Add gather with mode GatherMode::kDEFAULT and specified axis and nbElementWiseDims=0.
Definition: NvInfer.h:7060
bool unmarkWeightsRefittable(char const *name) noexcept
Unmark weights as refittable when the builder flag kREFIT_INDIVIDUAL is set.
Definition: NvInfer.h:8018
bool markWeightsRefittable(char const *name) noexcept
Mark weights as refittable when the builder flag kREFIT_INDIVIDUAL is set.
Definition: NvInfer.h:8006
IDeconvolutionLayer * addDeconvolutionNd(ITensor &input, int64_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension deconvolution layer to the network.
Definition: NvInfer.h:7471
IResizeLayer * addResize(ITensor &input) noexcept
Add a resize layer to the network.
Definition: NvInfer.h:7525
IUnsqueezeLayer * addUnsqueeze(ITensor &input, ITensor &axes) noexcept
Add an unsqueeze layer to the network.
Definition: NvInfer.h:8071
IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1) noexcept
Add a MatrixMultiply layer to the network.
Definition: NvInfer.h:7116
ISoftMaxLayer * addSoftMax(ITensor &input) noexcept
Add a SoftMax layer to the network.
Definition: NvInfer.h:6804
bool isDebugTensor(nvinfer1::ITensor const &tensor) const noexcept
Check if a tensor is marked as debug tensor.
Definition: NvInfer.h:6726
bool unmarkDebug(ITensor &tensor) noexcept
Unmark a tensor as a debug tensor.
Definition: NvInfer.h:6716
IEinsumLayer * addEinsum(ITensor *const *inputs, int32_t nbInputs, char const *equation) noexcept
Add an Einsum layer to the network.
Definition: NvInfer.h:7881
void markOutput(ITensor &tensor) noexcept
Mark a tensor as a network output.
Definition: NvInfer.h:6682
TRT_DEPRECATED IPluginV2Layer * addPluginV2(ITensor *const *inputs, int32_t nbInputs, IPluginV2 &plugin) noexcept
Add a plugin layer to the network using the IPluginV2 interface.
Definition: NvInfer.h:7232
IPaddingLayer * addPaddingNd(ITensor &input, Dims const &prePadding, Dims const &postPadding) noexcept
Add a padding layer to the network. Only 2D padding is currently supported.
Definition: NvInfer.h:7677
INonZeroLayer * addNonZero(ITensor &input) noexcept
Add a nonzero layer to the network.
Definition: NvInfer.h:7131
TRT_DEPRECATED IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7756
int32_t getNbOutputs() const noexcept
Get the number of outputs in the network.
Definition: NvInfer.h:6969
bool setWeightsName(Weights weights, char const *name) noexcept
Associate a name with all current uses of the given weights.
Definition: NvInfer.h:7701
Forward declaration of IEngineInspector for use by other interfaces.
Definition: NvInferRuntime.h:51
Definition: NvInfer.h:3740
virtual ~INonZeroLayer() noexcept=default
A normalization layer in a network definition.
Definition: NvInfer.h:6294
float getEpsilon() const noexcept
Get the epsilon value used for the normalization calculation.
Definition: NvInfer.h:6313
uint32_t getAxes() const noexcept
Get the axes value used for the normalization calculation.
Definition: NvInfer.h:6333
virtual ~INormalizationLayer() noexcept=default
void setEpsilon(float eps) noexcept
Set the epsilon value used for the normalization calculation.
Definition: NvInfer.h:6303
DataType getComputePrecision() const noexcept
Get the compute precision of this layer.
Definition: NvInfer.h:6400
apiv::VNormalizationLayer * mImpl
Definition: NvInfer.h:6406
int64_t getNbGroups() const noexcept
Get the number of groups used to split the channels for the normalization calculation.
Definition: NvInfer.h:6364
void setAxes(uint32_t axesMask) noexcept
Set the reduction axes for the normalization calculation.
Definition: NvInfer.h:6323
void setComputePrecision(DataType type) noexcept
Set the compute precision of this layer.
Definition: NvInfer.h:6390
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups used to split the channels in the normalization calculation.
Definition: NvInfer.h:6354
A OneHot layer in a network definition.
Definition: NvInfer.h:5948
apiv::VOneHotLayer * mImpl
Definition: NvInfer.h:5969
void setAxis(int32_t axis) noexcept
Set the axis parameter.
Definition: NvInfer.h:5955
int32_t getAxis() const noexcept
Get the value of the axis parameter.
Definition: NvInfer.h:5963
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:2616
Layer that represents a padding operation.
Definition: NvInfer.h:2964
Dims getPostPaddingNd() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3013
void setPrePaddingNd(Dims const &padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:2975
virtual ~IPaddingLayer() noexcept=default
void setPostPaddingNd(Dims const &padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3001
Dims getPrePaddingNd() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:2987
apiv::VPaddingLayer * mImpl
Definition: NvInfer.h:3019
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:3910
apiv::VParametricReLULayer * mImpl
Definition: NvInfer.h:3912
virtual ~IParametricReLULayer() noexcept=default
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:56
Plugin class for user-implemented layers.
Definition: NvInferRuntimePlugin.h:133
Layer type for pluginV2.
Definition: NvInfer.h:2701
virtual ~IPluginV2Layer() noexcept=default
apiv::VPluginV2Layer * mImpl
Definition: NvInfer.h:2714
IPluginV2 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:2708
Layer type for V3 plugins.
Definition: NvInfer.h:2728
virtual ~IPluginV3Layer() noexcept=default
IPluginV3 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:2735
apiv::VPluginV3Layer * mImpl
Definition: NvInfer.h:2741
A Pooling layer in a network definition.
Definition: NvInfer.h:1461
PoolingType getPoolingType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1480
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1613
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:1589
bool getAverageCountExcludesPadding() const noexcept
Get whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1533
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1561
void setPoolingType(PoolingType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1470
void setWindowSizeNd(Dims const &windowSize) noexcept
Set the multi-dimension window size for pooling.
Definition: NvInfer.h:1626
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1602
Dims getWindowSizeNd() const noexcept
Get the multi-dimension window size for pooling.
Definition: NvInfer.h:1636
void setAverageCountExcludesPadding(bool exclusive) noexcept
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1522
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding for pooling.
Definition: NvInfer.h:1680
float getBlendFactor() const noexcept
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1508
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride for pooling.
Definition: NvInfer.h:1651
Dims getStrideNd() const noexcept
Get the multi-dimension stride for pooling.
Definition: NvInfer.h:1661
virtual ~IPoolingLayer() noexcept=default
Dims getPaddingNd() const noexcept
Get the multi-dimension padding for pooling.
Definition: NvInfer.h:1692
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding for pooling.
Definition: NvInfer.h:1579
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding for pooling.
Definition: NvInfer.h:1551
void setBlendFactor(float blendFactor) noexcept
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1495
A Quantize layer in a network definition.
Definition: NvInfer.h:5397
void setToType(DataType toType) noexcept
Set the Quantize layer output type.
Definition: NvInfer.h:5434
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5418
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5407
virtual ~IQuantizeLayer() noexcept=default
DataType getToType() const noexcept
Return the Quantize layer output type.
Definition: NvInfer.h:5446
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:3761
apiv::VRaggedSoftMaxLayer * mImpl
Definition: NvInfer.h:3763
virtual ~IRaggedSoftMaxLayer() noexcept=default
A recurrence layer in a network definition.
Definition: NvInfer.h:4611
virtual ~IRecurrenceLayer() noexcept=default
Layer that represents a reduction across a non-bool tensor.
Definition: NvInfer.h:2884
void setKeepDimensions(bool keepDimensions) noexcept
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:2931
void setOperation(ReduceOperation op) noexcept
Set the reduce operation for the layer.
Definition: NvInfer.h:2891
ReduceOperation getOperation() const noexcept
Get the reduce operation for the layer.
Definition: NvInfer.h:2901
virtual ~IReduceLayer() noexcept=default
uint32_t getReduceAxes() const noexcept
Get the axes over which to reduce for the layer.
Definition: NvInfer.h:2921
void setReduceAxes(uint32_t reduceAxes) noexcept
Set the axes over which to reduce.
Definition: NvInfer.h:2911
apiv::VReduceLayer * mImpl
Definition: NvInfer.h:2947
bool getKeepDimensions() const noexcept
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:2941
A resize layer in a network definition.
Definition: NvInfer.h:4099
void setSelectorForSinglePixel(ResizeSelector selector) noexcept
Set coordinate selector function when resized to single pixel.
Definition: NvInfer.h:4260
void setNearestRounding(ResizeRoundMode value) noexcept
Set rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4284
virtual ~IResizeLayer() noexcept=default
int32_t getScales(int32_t size, float *scales) const noexcept
Copies resize scales to scales[0, ..., nbScales-1], where nbScales is the number of scales that were ...
Definition: NvInfer.h:4178
void setOutputDimensions(Dims const &dimensions) noexcept
Set the output dimensions.
Definition: NvInfer.h:4119
void setCubicCoeff(float A) noexcept
Set the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:4316
void setScales(float const *scales, int32_t nbScales) noexcept
Set the resize scales.
Definition: NvInfer.h:4159
float getCubicCoeff() const noexcept
Get the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:4326
ResizeSelector getSelectorForSinglePixel() const noexcept
Get the coordinate selector function when resized to single pixel.
Definition: NvInfer.h:4270
InterpolationMode getResizeMode() const noexcept
Get resize mode for an input tensor.
Definition: NvInfer.h:4200
void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept
Set coordinate transformation function.
Definition: NvInfer.h:4235
void setExcludeOutside(bool excludeFlag) noexcept
Set the state for excluding outside pixels.
Definition: NvInfer.h:4339
void setResizeMode(InterpolationMode interpolationMode) noexcept
Set resize mode for an input tensor.
Definition: NvInfer.h:4190
Dims getOutputDimensions() const noexcept
Get the output dimensions.
Definition: NvInfer.h:4129
ResizeRoundMode getNearestRounding() const noexcept
Get rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4294
bool getExcludeOutside() const noexcept
Get the state for excluding outside pixels.
Definition: NvInfer.h:4349
ResizeCoordinateTransformation getCoordinateTransformation() const noexcept
Get coordinate transformation function.
Definition: NvInfer.h:4245
A ReverseSequence layer in a network definition.
Definition: NvInfer.h:6222
void setSequenceAxis(int32_t sequenceAxis) noexcept
Set the sequence axis. Default is 0.
Definition: NvInfer.h:6255
int32_t getBatchAxis() const noexcept
Return the batch axis. Return 1 if no batch axis was set.
Definition: NvInfer.h:6242
apiv::VReverseSequenceLayer * mImpl
Definition: NvInfer.h:6271
int32_t getSequenceAxis() const noexcept
Return the sequence axis. Return 0 if no sequence axis was set.
Definition: NvInfer.h:6265
void setBatchAxis(int32_t batchAxis) noexcept
Set the batch axis. Default is 1.
Definition: NvInfer.h:6232
virtual ~IReverseSequenceLayer() noexcept=default
A Scale layer in a network definition.
Definition: NvInfer.h:1858
Weights getScale() const noexcept
Get the scale value.
Definition: NvInfer.h:1915
Weights getPower() const noexcept
Get the power value.
Definition: NvInfer.h:1935
void setScale(Weights scale) noexcept
Set the scale value.
Definition: NvInfer.h:1905
void setPower(Weights power) noexcept
Set the power value.
Definition: NvInfer.h:1925
ScaleMode getMode() const noexcept
Get the scale mode.
Definition: NvInfer.h:1875
void setShift(Weights shift) noexcept
Set the shift value.
Definition: NvInfer.h:1885
void setChannelAxis(int32_t channelAxis) noexcept
Set the channel axis.
Definition: NvInfer.h:1971
Weights getShift() const noexcept
Get the shift value.
Definition: NvInfer.h:1895
virtual ~IScaleLayer() noexcept=default
void setMode(ScaleMode mode) noexcept
Set the scale mode.
Definition: NvInfer.h:1865
int32_t getChannelAxis() const noexcept
Get the channel axis.
Definition: NvInfer.h:1950
A scatter layer in a network definition. Supports several kinds of scattering.
Definition: NvInfer.h:5876
void setMode(ScatterMode mode) noexcept
Set the scatter mode.
Definition: NvInfer.h:5883
apiv::VScatterLayer * mImpl
Definition: NvInfer.h:5917
void setAxis(int32_t axis) noexcept
Set the axis used by ScatterMode::kELEMENTS.
Definition: NvInfer.h:5903
int32_t getAxis() const noexcept
Get the axis.
Definition: NvInfer.h:5911
ScatterMode getMode() const noexcept
Get the scatter mode.
Definition: NvInfer.h:5893
virtual ~IScatterLayer() noexcept=default
Select elements from two data tensors based on a condition tensor.
Definition: NvInfer.h:4918
virtual ~ISelectLayer() noexcept=default
Layer type for getting shape of a tensor.
Definition: NvInfer.h:3489
virtual ~IShapeLayer() noexcept=default
apiv::VShapeLayer * mImpl
Definition: NvInfer.h:3491
Layer type for shuffling data.
Definition: NvInfer.h:3052
apiv::VShuffleLayer * mImpl
Definition: NvInfer.h:3210
void setFirstTranspose(Permutation permutation) noexcept
Set the permutation applied by the first transpose operation.
Definition: NvInfer.h:3063
void setSecondTranspose(Permutation permutation) noexcept
Set the permutation applied by the second transpose operation.
Definition: NvInfer.h:3163
Dims getReshapeDimensions() const noexcept
Get the reshaped dimensions.
Definition: NvInfer.h:3116
void setReshapeDimensions(Dims const &dimensions) noexcept
Set the reshaped dimensions.
Definition: NvInfer.h:3103
Permutation getFirstTranspose() const noexcept
Get the permutation applied by the first transpose operation.
Definition: NvInfer.h:3075
virtual ~IShuffleLayer() noexcept=default
Permutation getSecondTranspose() const noexcept
Get the permutation applied by the second transpose operation.
Definition: NvInfer.h:3175
bool getZeroIsPlaceholder() const noexcept
Get meaning of 0 in reshape dimensions.
Definition: NvInfer.h:3204
void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
Set meaning of 0 in reshape dimensions.
Definition: NvInfer.h:3191
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:3304
void setStride(Dims const &stride) noexcept
Set the stride for computing the output slice data.
Definition: NvInfer.h:3373
apiv::VSliceLayer * mImpl
Definition: NvInfer.h:3472
virtual ~ISliceLayer() noexcept=default
void setSize(Dims const &size) noexcept
Set the dimensions of the output slice.
Definition: NvInfer.h:3344
void setAxes(Dims const &axes) noexcept
Set the axes for this ISliceLayer.
Definition: NvInfer.h:3451
void setStart(Dims const &start) noexcept
Set the start offset that the slice layer uses to create the output slice.
Definition: NvInfer.h:3315
Dims getStart() const noexcept
Get the start offset for the slice layer.
Definition: NvInfer.h:3330
void setMode(SampleMode mode) noexcept
Set the slice mode.
Definition: NvInfer.h:3398
Dims getSize() const noexcept
Get dimensions of the output slice.
Definition: NvInfer.h:3359
SampleMode getMode() const noexcept
Get the slice mode.
Definition: NvInfer.h:3408
Dims getStride() const noexcept
Get the stride for the output slice.
Definition: NvInfer.h:3388
Dims getAxes() const noexcept
Get the axes for this ISliceLayer.
Definition: NvInfer.h:3466
A Softmax layer in a network definition.
Definition: NvInfer.h:2002
void setAxes(uint32_t axes) noexcept
Set the axis along which softmax is computed. Currently, only one axis can be set.
Definition: NvInfer.h:2024
uint32_t getAxes() const noexcept
Get the axis along which softmax occurs.
Definition: NvInfer.h:2034
virtual ~ISoftMaxLayer() noexcept=default
Layer that represents a squeeze operation, removing unit dimensions of the input tensor on a set of a...
Definition: NvInfer.h:6419
virtual ~ISqueezeLayer() noexcept=default
apiv::VSqueezeLayer * mImpl
Definition: NvInfer.h:6436
A tensor in a network definition.
Definition: NvInfer.h:185
void setAllowedFormats(TensorFormats formats) noexcept
Set allowed formats for an input or output tensor. By default all formats are allowed....
Definition: NvInfer.h:431
TensorLocation getLocation() const noexcept
Get the storage location of a tensor.
Definition: NvInfer.h:350
void setDimensions(Dims const &dimensions) noexcept
Set the dimensions of a tensor.
Definition: NvInfer.h:233
void resetDynamicRange() noexcept
Undo effect of setDynamicRange.
Definition: NvInfer.h:389
void setName(char const *name) noexcept
Set the tensor name.
Definition: NvInfer.h:202
bool isExecutionTensor() const noexcept
Whether the tensor is an execution tensor.
Definition: NvInfer.h:496
void setType(DataType type) noexcept
Set the data type of a tensor.
Definition: NvInfer.h:262
TRT_DEPRECATED bool dynamicRangeIsSet() const noexcept
Query whether dynamic range is set.
Definition: NvInfer.h:381
char const * getName() const noexcept
Get the tensor name.
Definition: NvInfer.h:214
bool isShapeTensor() const noexcept
Whether the tensor is a shape tensor.
Definition: NvInfer.h:475
float getDynamicRangeMax() const noexcept
Get maximum of dynamic range.
Definition: NvInfer.h:409
bool isNetworkInput() const noexcept
Whether the tensor is a network input.
Definition: NvInfer.h:299
TRT_DEPRECATED void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
Set whether to enable broadcast of tensor across the implicit batch dimension.
Definition: NvInfer.h:324
TRT_DEPRECATED bool setDynamicRange(float min, float max) noexcept
Set dynamic range for the tensor.
Definition: NvInfer.h:291
TRT_DEPRECATED bool getBroadcastAcrossBatch() const noexcept
Check if tensor is broadcast across the implicit batch dimension.
Definition: NvInfer.h:338
bool isNetworkOutput() const noexcept
Whether the tensor is a network output.
Definition: NvInfer.h:307
DataType getType() const noexcept
Get the data type of a tensor.
Definition: NvInfer.h:274
apiv::VTensor * mImpl
Definition: NvInfer.h:543
float getDynamicRangeMin() const noexcept
Get minimum of dynamic range.
Definition: NvInfer.h:399
virtual ~ITensor() noexcept=default
void setDimensionName(int32_t index, char const *name) noexcept
Name a dimension of an input tensor.
Definition: NvInfer.h:522
char const * getDimensionName(int32_t index) const noexcept
Get the name of an input dimension.
Definition: NvInfer.h:537
TRT_DEPRECATED void setLocation(TensorLocation location) noexcept
Set the storage location of a tensor.
Definition: NvInfer.h:369
Dims getDimensions() const noexcept
Get the dimensions of a tensor.
Definition: NvInfer.h:247
TensorFormats getAllowedFormats() const noexcept
Get a bitmask of TensorFormat values that the tensor supports. For a shape tensor,...
Definition: NvInfer.h:444
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:8967
int64_t queryKeys(TimingCacheKey *keyBuffer, int64_t capacity) const noexcept
Query cache keys from Timing Cache.
Definition: NvInfer.h:9033
bool combine(ITimingCache const &inputCache, bool ignoreMismatch) noexcept
Combine input timing cache into local instance.
Definition: NvInfer.h:9004
TimingCacheValue query(TimingCacheKey const &key) const noexcept
Query value in a cache entry.
Definition: NvInfer.h:9050
virtual ~ITimingCache() noexcept=default
bool update(TimingCacheKey const &key, TimingCacheValue const &value) noexcept
Update values in a cache entry.
Definition: NvInfer.h:9072
apiv::VTimingCache * mImpl
Definition: NvInfer.h:9078
bool reset() noexcept
Empty the timing cache.
Definition: NvInfer.h:9014
Layer that represents a TopK reduction.
Definition: NvInfer.h:3529
void setK(int32_t k) noexcept
Set the static k value for the layer.
Definition: NvInfer.h:3560
void setReduceAxes(uint32_t reduceAxes) noexcept
Set which axes to reduce for the layer.
Definition: NvInfer.h:3584
TopKOperation getOperation() const noexcept
Get the operation for the layer.
Definition: NvInfer.h:3546
apiv::VTopKLayer * mImpl
Definition: NvInfer.h:3616
void setOperation(TopKOperation op) noexcept
Set the operation for the layer.
Definition: NvInfer.h:3536
int32_t getK() const noexcept
Get the k value for the layer.
Definition: NvInfer.h:3574
uint32_t getReduceAxes() const noexcept
Get the axes to reduce for the layer.
Definition: NvInfer.h:3594
virtual ~ITopKLayer() noexcept=default
A layer that represents a trip-count limiter.
Definition: NvInfer.h:4732
TripLimit getTripLimit() const noexcept
Get a trip limiter type.
Definition: NvInfer.h:4737
virtual ~ITripLimitLayer() noexcept=default
Layer that represents an unary operation.
Definition: NvInfer.h:2809
void setOperation(UnaryOperation op) noexcept
Set the unary operation for the layer.
Definition: NvInfer.h:2818
apiv::VUnaryLayer * mImpl
Definition: NvInfer.h:2834
UnaryOperation getOperation() const noexcept
Get the unary operation for the layer.
Definition: NvInfer.h:2828
virtual ~IUnaryLayer() noexcept=default
Layer that represents an unsqueeze operation, which reshapes the input tensor by inserting unit-lengt...
Definition: NvInfer.h:6448
virtual ~IUnsqueezeLayer() noexcept=default
apiv::VUnsqueezeLayer * mImpl
Definition: NvInfer.h:6465
An Interface class for version control.
Definition: NvInferRuntimeBase.h:263
Version information associated with a TRT interface.
Definition: NvInferRuntimeBase.h:228
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:124
Definition: NvInfer.h:8599
virtual int32_t selectAlgorithms(IAlgorithmContext const &context, IAlgorithm const *const *choices, int32_t nbChoices, int32_t *selection) noexcept=0
Select Algorithms for a layer from the given list of algorithm choices.
virtual void reportAlgorithms(IAlgorithmContext const *const *algoContexts, IAlgorithm const *const *algoChoices, int32_t nbAlgorithms) noexcept=0
Called by TensorRT to report choices it made.
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8604
virtual ~IAlgorithmSelector() noexcept=default
Definition: NvInferRuntimeBase.h:400
Definition: NvInferRuntime.h:1608
~IInt8EntropyCalibrator2() noexcept override=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8239
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8231
Definition: NvInfer.h:8186
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8199
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8191
~IInt8EntropyCalibrator() noexcept override=default
Definition: NvInfer.h:8305
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8318
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8310
virtual double getQuantile() const noexcept=0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
Definition: NvInfer.h:8266
~IInt8MinMaxCalibrator() noexcept override=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:8279
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8271
Definition: NvInferPluginBase.h:206
Definition: NvInfer.h:9276
virtual bool stepComplete(char const *phaseName, int32_t step) noexcept=0
Signal that a step of an optimizer phase has finished.
virtual ~IProgressMonitor() noexcept=default
virtual void phaseFinish(char const *phaseName) noexcept=0
Signal that a phase of the optimizer has finished.
virtual void phaseStart(char const *phaseName, char const *parentPhase, int32_t nbSteps) noexcept=0
Signal that a phase of the optimizer has started.
IBuilder * createInferBuilder(ILogger &logger) noexcept
Create an instance of an IBuilder class.
Definition: NvInfer.h:10627
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2832
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4004
@ kFORMULA
Use formula to map the original index.
@ kUPPER
Select the upper left pixel.