TensorRT 10.0.0
NvInfer.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_H
19#define NV_INFER_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntime.h"
23
32//
35
41
47namespace nvinfer1
48{
49
57enum class LayerType : int32_t
58{
59 kCONVOLUTION = 0,
60 kCAST = 1,
61 kACTIVATION = 2,
62 kPOOLING = 3,
63 kLRN = 4,
64 kSCALE = 5,
65 kSOFTMAX = 6,
66 kDECONVOLUTION = 7,
67 kCONCATENATION = 8,
68 kELEMENTWISE = 9,
69 kPLUGIN = 10,
70 kUNARY = 11,
71 kPADDING = 12,
72 kSHUFFLE = 13,
73 kREDUCE = 14,
74 kTOPK = 15,
75 kGATHER = 16,
76 kMATRIX_MULTIPLY = 17,
77 kRAGGED_SOFTMAX = 18,
78 kCONSTANT = 19,
79 kIDENTITY = 20,
80 kPLUGIN_V2 = 21,
81 kSLICE = 22,
82 kSHAPE = 23,
83 kPARAMETRIC_RELU = 24,
84 kRESIZE = 25,
85 kTRIP_LIMIT = 26,
86 kRECURRENCE = 27,
87 kITERATOR = 28,
88 kLOOP_OUTPUT = 29,
89 kSELECT = 30,
90 kFILL = 31,
91 kQUANTIZE = 32,
92 kDEQUANTIZE = 33,
93 kCONDITION = 34,
96 kSCATTER = 37,
97 kEINSUM = 38,
98 kASSERTION = 39,
99 kONE_HOT = 40,
100 kNON_ZERO = 41,
101 kGRID_SAMPLE = 42,
102 kNMS = 43,
103 kREVERSE_SEQUENCE = 44,
104 kNORMALIZATION = 45,
105 kPLUGIN_V3 = 46
106};
107
113template <>
114constexpr inline int32_t EnumMax<LayerType>() noexcept
115{
116 return 47;
117}
118
125using TensorFormats = uint32_t;
126
132enum class ActivationType : int32_t
133{
134 kRELU = 0,
135 kSIGMOID = 1,
136 kTANH = 2,
137 kLEAKY_RELU = 3,
138 kELU = 4,
139 kSELU = 5,
140 kSOFTSIGN = 6,
141 kSOFTPLUS = 7,
142 kCLIP = 8,
143 kHARD_SIGMOID = 9,
144 kSCALED_TANH = 10,
145 kTHRESHOLDED_RELU = 11,
146 kGELU_ERF = 12,
147 kGELU_TANH = 13
148};
149
150namespace impl
151{
157template <>
159{
160 static constexpr int32_t kVALUE = 14;
161};
162} // namespace impl
163
180class ITensor : public INoCopy
181{
182public:
197 void setName(char const* name) noexcept
198 {
199 mImpl->setName(name);
200 }
201
209 char const* getName() const noexcept
210 {
211 return mImpl->getName();
212 }
213
228 void setDimensions(Dims const& dimensions) noexcept
229 {
230 mImpl->setDimensions(dimensions);
231 }
232
242 Dims getDimensions() const noexcept
243 {
244 return mImpl->getDimensions();
245 }
246
257 void setType(DataType type) noexcept
258 {
259 mImpl->setType(type);
260 }
261
269 DataType getType() const noexcept
270 {
271 return mImpl->getType();
272 }
273
284 bool setDynamicRange(float min, float max) noexcept
285 {
286 return mImpl->setDynamicRange(min, max);
287 }
288
292 bool isNetworkInput() const noexcept
293 {
294 return mImpl->isNetworkInput();
295 }
296
300 bool isNetworkOutput() const noexcept
301 {
302 return mImpl->isNetworkOutput();
303 }
304
317 TRT_DEPRECATED void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
318 {
319 mImpl->setBroadcastAcrossBatch(broadcastAcrossBatch);
320 }
321
332 {
333 return mImpl->getBroadcastAcrossBatch();
334 }
335
344 {
345 return mImpl->getLocation();
346 }
347
363 {
364 mImpl->setLocation(location);
365 }
366
372 bool dynamicRangeIsSet() const noexcept
373 {
374 return mImpl->dynamicRangeIsSet();
375 }
376
380 void resetDynamicRange() noexcept
381 {
382 mImpl->resetDynamicRange();
383 }
384
390 float getDynamicRangeMin() const noexcept
391 {
392 return mImpl->getDynamicRangeMin();
393 }
394
400 float getDynamicRangeMax() const noexcept
401 {
402 return mImpl->getDynamicRangeMax();
403 }
404
420 void setAllowedFormats(TensorFormats formats) noexcept
421 {
422 mImpl->setAllowedFormats(formats);
423 }
424
434 {
435 return mImpl->getAllowedFormats();
436 }
437
464 bool isShapeTensor() const noexcept
465 {
466 return mImpl->isShapeTensor();
467 }
468
485 bool isExecutionTensor() const noexcept
486 {
487 return mImpl->isExecutionTensor();
488 }
489
511 void setDimensionName(int32_t index, char const* name) noexcept
512 {
513 mImpl->setDimensionName(index, name);
514 }
515
526 char const* getDimensionName(int32_t index) const noexcept
527 {
528 return mImpl->getDimensionName(index);
529 }
530
531protected:
532 apiv::VTensor* mImpl;
533 virtual ~ITensor() noexcept = default;
534};
535
543class ILayer : public INoCopy
544{
545public:
551 LayerType getType() const noexcept
552 {
553 return mLayer->getType();
554 }
555
565 void setName(char const* name) noexcept
566 {
567 mLayer->setName(name);
568 }
569
575 char const* getName() const noexcept
576 {
577 return mLayer->getName();
578 }
579
583 int32_t getNbInputs() const noexcept
584 {
585 return mLayer->getNbInputs();
586 }
587
596 ITensor* getInput(int32_t index) const noexcept
597 {
598 return mLayer->getInput(index);
599 }
600
604 int32_t getNbOutputs() const noexcept
605 {
606 return mLayer->getNbOutputs();
607 }
608
614 ITensor* getOutput(int32_t index) const noexcept
615 {
616 return mLayer->getOutput(index);
617 }
618
631 void setInput(int32_t index, ITensor& tensor) noexcept
632 {
633 return mLayer->setInput(index, tensor);
634 }
635
663 void setPrecision(DataType dataType) noexcept
664 {
665 mLayer->setPrecision(dataType);
666 }
667
675 DataType getPrecision() const noexcept
676 {
677 return mLayer->getPrecision();
678 }
679
687 bool precisionIsSet() const noexcept
688 {
689 return mLayer->precisionIsSet();
690 }
691
697 void resetPrecision() noexcept
698 {
699 mLayer->resetPrecision();
700 }
701
744 void setOutputType(int32_t index, DataType dataType) noexcept
745 {
746 mLayer->setOutputType(index, dataType);
747 }
748
759 DataType getOutputType(int32_t index) const noexcept
760 {
761 return mLayer->getOutputType(index);
762 }
763
773 bool outputTypeIsSet(int32_t index) const noexcept
774 {
775 return mLayer->outputTypeIsSet(index);
776 }
777
785 void resetOutputType(int32_t index) noexcept
786 {
787 return mLayer->resetOutputType(index);
788 }
789
803 void setMetadata(char const* metadata) noexcept
804 {
805 mLayer->setMetadata(metadata);
806 }
807
816 char const* getMetadata() const noexcept
817 {
818 return mLayer->getMetadata();
819 }
820
821protected:
822 virtual ~ILayer() noexcept = default;
823 apiv::VLayer* mLayer;
824};
825
982enum class PaddingMode : int32_t
983{
986 kSAME_UPPER = 2,
987 kSAME_LOWER = 3,
988};
989
990namespace impl
991{
997template <>
999{
1000 static constexpr int32_t kVALUE = 4;
1001};
1002} // namespace impl
1003
1017{
1018public:
1026 void setNbOutputMaps(int64_t nbOutputMaps) noexcept
1027 {
1028 mImpl->setNbOutputMaps(nbOutputMaps);
1029 }
1030
1036 int64_t getNbOutputMaps() const noexcept
1037 {
1038 return mImpl->getNbOutputMaps();
1039 }
1040
1056 void setNbGroups(int64_t nbGroups) noexcept
1057 {
1058 mImpl->setNbGroups(nbGroups);
1059 }
1060
1066 int64_t getNbGroups() const noexcept
1067 {
1068 return mImpl->getNbGroups();
1069 }
1070
1080 void setKernelWeights(Weights weights) noexcept
1081 {
1082 mImpl->setKernelWeights(weights);
1083 }
1084
1090 Weights getKernelWeights() const noexcept
1091 {
1092 return mImpl->getKernelWeights();
1093 }
1094
1105 void setBiasWeights(Weights weights) noexcept
1106 {
1107 mImpl->setBiasWeights(weights);
1108 }
1109
1115 Weights getBiasWeights() const noexcept
1116 {
1117 return mImpl->getBiasWeights();
1118 }
1119
1132 void setPrePadding(Dims const& padding) noexcept
1133 {
1134 mImpl->setPrePadding(padding);
1135 }
1136
1142 Dims getPrePadding() const noexcept
1143 {
1144 return mImpl->getPrePadding();
1145 }
1146
1159 void setPostPadding(Dims const& padding) noexcept
1160 {
1161 mImpl->setPostPadding(padding);
1162 }
1163
1169 Dims getPostPadding() const noexcept
1170 {
1171 return mImpl->getPostPadding();
1172 }
1173
1183 void setPaddingMode(PaddingMode paddingMode) noexcept
1184 {
1185 mImpl->setPaddingMode(paddingMode);
1186 }
1187
1196 {
1197 return mImpl->getPaddingMode();
1198 }
1199
1208 void setKernelSizeNd(Dims const& kernelSize) noexcept
1209 {
1210 mImpl->setKernelSizeNd(kernelSize);
1211 }
1212
1218 Dims getKernelSizeNd() const noexcept
1219 {
1220 return mImpl->getKernelSizeNd();
1221 }
1222
1233 void setStrideNd(Dims const& stride) noexcept
1234 {
1235 mImpl->setStrideNd(stride);
1236 }
1237
1243 Dims getStrideNd() const noexcept
1244 {
1245 return mImpl->getStrideNd();
1246 }
1247
1261 void setPaddingNd(Dims const& padding) noexcept
1262 {
1263 mImpl->setPaddingNd(padding);
1264 }
1265
1273 Dims getPaddingNd() const noexcept
1274 {
1275 return mImpl->getPaddingNd();
1276 }
1277
1287 void setDilationNd(Dims const& dilation) noexcept
1288 {
1289 mImpl->setDilationNd(dilation);
1290 }
1291
1297 Dims getDilationNd() const noexcept
1298 {
1299 return mImpl->getDilationNd();
1300 }
1301
1316 using ILayer::setInput;
1317
1318protected:
1319 virtual ~IConvolutionLayer() noexcept = default;
1320 apiv::VConvolutionLayer* mImpl;
1321};
1322
1337{
1338public:
1347 {
1348 mImpl->setActivationType(type);
1349 }
1350
1357 {
1358 return mImpl->getActivationType();
1359 }
1360
1371 void setAlpha(float alpha) noexcept
1372 {
1373 mImpl->setAlpha(alpha);
1374 }
1375
1385 void setBeta(float beta) noexcept
1386 {
1387 mImpl->setBeta(beta);
1388 }
1389
1394 float getAlpha() const noexcept
1395 {
1396 return mImpl->getAlpha();
1397 }
1398
1403 float getBeta() const noexcept
1404 {
1405 return mImpl->getBeta();
1406 }
1407
1408protected:
1409 virtual ~IActivationLayer() noexcept = default;
1410 apiv::VActivationLayer* mImpl;
1411};
1412
1418enum class PoolingType : int32_t
1419{
1420 kMAX = 0,
1421 kAVERAGE = 1,
1423};
1424
1425namespace impl
1426{
1432template <>
1434{
1435 static constexpr int32_t kVALUE = 3;
1436};
1437} // namespace impl
1438
1450class IPoolingLayer : public ILayer
1451{
1452public:
1460 void setPoolingType(PoolingType type) noexcept
1461 {
1462 mImpl->setPoolingType(type);
1463 }
1464
1471 {
1472 return mImpl->getPoolingType();
1473 }
1474
1485 void setBlendFactor(float blendFactor) noexcept
1486 {
1487 mImpl->setBlendFactor(blendFactor);
1488 }
1489
1498 float getBlendFactor() const noexcept
1499 {
1500 return mImpl->getBlendFactor();
1501 }
1502
1512 void setAverageCountExcludesPadding(bool exclusive) noexcept
1513 {
1514 mImpl->setAverageCountExcludesPadding(exclusive);
1515 }
1516
1524 {
1525 return mImpl->getAverageCountExcludesPadding();
1526 }
1527
1541 void setPrePadding(Dims const& padding) noexcept
1542 {
1543 mImpl->setPrePadding(padding);
1544 }
1545
1551 Dims getPrePadding() const noexcept
1552 {
1553 return mImpl->getPrePadding();
1554 }
1555
1569 void setPostPadding(Dims const& padding) noexcept
1570 {
1571 mImpl->setPostPadding(padding);
1572 }
1573
1579 Dims getPostPadding() const noexcept
1580 {
1581 return mImpl->getPostPadding();
1582 }
1583
1592 void setPaddingMode(PaddingMode paddingMode) noexcept
1593 {
1594 mImpl->setPaddingMode(paddingMode);
1595 }
1596
1604 {
1605 return mImpl->getPaddingMode();
1606 }
1607
1616 void setWindowSizeNd(Dims const& windowSize) noexcept
1617 {
1618 mImpl->setWindowSizeNd(windowSize);
1619 }
1620
1626 Dims getWindowSizeNd() const noexcept
1627 {
1628 return mImpl->getWindowSizeNd();
1629 }
1630
1641 void setStrideNd(Dims const& stride) noexcept
1642 {
1643 mImpl->setStrideNd(stride);
1644 }
1645
1651 Dims getStrideNd() const noexcept
1652 {
1653 return mImpl->getStrideNd();
1654 }
1655
1670 void setPaddingNd(Dims const& padding) noexcept
1671 {
1672 mImpl->setPaddingNd(padding);
1673 }
1674
1682 Dims getPaddingNd() const noexcept
1683 {
1684 return mImpl->getPaddingNd();
1685 }
1686
1687protected:
1688 virtual ~IPoolingLayer() noexcept = default;
1689 apiv::VPoolingLayer* mImpl;
1690};
1691
1701class ILRNLayer : public ILayer
1702{
1703public:
1713 void setWindowSize(int64_t windowSize) noexcept
1714 {
1715 mImpl->setWindowSize(windowSize);
1716 }
1717
1723 int64_t getWindowSize() const noexcept
1724 {
1725 return mImpl->getWindowSize();
1726 }
1727
1735 void setAlpha(float alpha) noexcept
1736 {
1737 mImpl->setAlpha(alpha);
1738 }
1739
1745 float getAlpha() const noexcept
1746 {
1747 return mImpl->getAlpha();
1748 }
1749
1757 void setBeta(float beta) noexcept
1758 {
1759 mImpl->setBeta(beta);
1760 }
1761
1767 float getBeta() const noexcept
1768 {
1769 return mImpl->getBeta();
1770 }
1771
1779 void setK(float k) noexcept
1780 {
1781 mImpl->setK(k);
1782 }
1783
1789 float getK() const noexcept
1790 {
1791 return mImpl->getK();
1792 }
1793
1794protected:
1795 virtual ~ILRNLayer() noexcept = default;
1796 apiv::VLRNLayer* mImpl;
1797};
1798
1804enum class ScaleMode : int32_t
1805{
1806 kUNIFORM = 0,
1807 kCHANNEL = 1,
1808 kELEMENTWISE = 2
1809};
1810
1816template <>
1817constexpr inline int32_t EnumMax<ScaleMode>() noexcept
1818{
1819 return 3;
1820}
1821
1847class IScaleLayer : public ILayer
1848{
1849public:
1855 void setMode(ScaleMode mode) noexcept
1856 {
1857 mImpl->setMode(mode);
1858 }
1859
1865 ScaleMode getMode() const noexcept
1866 {
1867 return mImpl->getMode();
1868 }
1869
1875 void setShift(Weights shift) noexcept
1876 {
1877 mImpl->setShift(shift);
1878 }
1879
1885 Weights getShift() const noexcept
1886 {
1887 return mImpl->getShift();
1888 }
1889
1895 void setScale(Weights scale) noexcept
1896 {
1897 mImpl->setScale(scale);
1898 }
1899
1905 Weights getScale() const noexcept
1906 {
1907 return mImpl->getScale();
1908 }
1909
1915 void setPower(Weights power) noexcept
1916 {
1917 mImpl->setPower(power);
1918 }
1919
1925 Weights getPower() const noexcept
1926 {
1927 return mImpl->getPower();
1928 }
1929
1940 int32_t getChannelAxis() const noexcept
1941 {
1942 return mImpl->getChannelAxis();
1943 }
1944
1961 void setChannelAxis(int32_t channelAxis) noexcept
1962 {
1963 mImpl->setChannelAxis(channelAxis);
1964 }
1965
1966protected:
1967 virtual ~IScaleLayer() noexcept = default;
1968 apiv::VScaleLayer* mImpl;
1969};
1970
1991class ISoftMaxLayer : public ILayer
1992{
1993public:
2014 void setAxes(uint32_t axes) noexcept
2015 {
2016 mImpl->setAxes(axes);
2017 }
2018
2024 uint32_t getAxes() const noexcept
2025 {
2026 return mImpl->getAxes();
2027 }
2028
2029protected:
2030 virtual ~ISoftMaxLayer() noexcept = default;
2031 apiv::VSoftMaxLayer* mImpl;
2032};
2033
2047{
2048public:
2060 void setAxis(int32_t axis) noexcept
2061 {
2062 mImpl->setAxis(axis);
2063 }
2064
2070 int32_t getAxis() const noexcept
2071 {
2072 return mImpl->getAxis();
2073 }
2074
2075protected:
2076 virtual ~IConcatenationLayer() noexcept = default;
2077 apiv::VConcatenationLayer* mImpl;
2078};
2079
2088{
2089public:
2097 void setNbOutputMaps(int64_t nbOutputMaps) noexcept
2098 {
2099 mImpl->setNbOutputMaps(nbOutputMaps);
2100 }
2101
2107 int64_t getNbOutputMaps() const noexcept
2108 {
2109 return mImpl->getNbOutputMaps();
2110 }
2111
2127 void setNbGroups(int64_t nbGroups) noexcept
2128 {
2129 mImpl->setNbGroups(nbGroups);
2130 }
2131
2137 int64_t getNbGroups() const noexcept
2138 {
2139 return mImpl->getNbGroups();
2140 }
2141
2151 void setKernelWeights(Weights weights) noexcept
2152 {
2153 mImpl->setKernelWeights(weights);
2154 }
2155
2161 Weights getKernelWeights() const noexcept
2162 {
2163 return mImpl->getKernelWeights();
2164 }
2165
2176 void setBiasWeights(Weights weights) noexcept
2177 {
2178 mImpl->setBiasWeights(weights);
2179 }
2180
2186 Weights getBiasWeights() const noexcept
2187 {
2188 return mImpl->getBiasWeights();
2189 }
2190
2204 void setPrePadding(Dims const& padding) noexcept
2205 {
2206 mImpl->setPrePadding(padding);
2207 }
2208
2214 Dims getPrePadding() const noexcept
2215 {
2216 return mImpl->getPrePadding();
2217 }
2218
2232 void setPostPadding(Dims const& padding) noexcept
2233 {
2234 mImpl->setPostPadding(padding);
2235 }
2236
2242 Dims getPostPadding() const noexcept
2243 {
2244 return mImpl->getPostPadding();
2245 }
2246
2256 void setPaddingMode(PaddingMode paddingMode) noexcept
2257 {
2258 mImpl->setPaddingMode(paddingMode);
2259 }
2260
2269 {
2270 return mImpl->getPaddingMode();
2271 }
2272
2283 void setKernelSizeNd(Dims const& kernelSize) noexcept
2284 {
2285 mImpl->setKernelSizeNd(kernelSize);
2286 }
2287
2293 Dims getKernelSizeNd() const noexcept
2294 {
2295 return mImpl->getKernelSizeNd();
2296 }
2297
2310 void setStrideNd(Dims const& stride) noexcept
2311 {
2312 mImpl->setStrideNd(stride);
2313 }
2314
2320 Dims getStrideNd() const noexcept
2321 {
2322 return mImpl->getStrideNd();
2323 }
2324
2338 void setPaddingNd(Dims const& padding) noexcept
2339 {
2340 mImpl->setPaddingNd(padding);
2341 }
2342
2350 Dims getPaddingNd() const noexcept
2351 {
2352 return mImpl->getPaddingNd();
2353 }
2354
2367 using ILayer::setInput;
2368
2376 void setDilationNd(Dims const& dilation) noexcept
2377 {
2378 mImpl->setDilationNd(dilation);
2379 }
2380
2386 Dims getDilationNd() const noexcept
2387 {
2388 return mImpl->getDilationNd();
2389 }
2390
2391protected:
2392 virtual ~IDeconvolutionLayer() noexcept = default;
2393 apiv::VDeconvolutionLayer* mImpl;
2394};
2395
2410enum class ElementWiseOperation : int32_t
2411{
2412 kSUM = 0,
2413 kPROD = 1,
2414 kMAX = 2,
2415 kMIN = 3,
2416 kSUB = 4,
2417 kDIV = 5,
2418 kPOW = 6,
2419 kFLOOR_DIV = 7,
2420 kAND = 8,
2421 kOR = 9,
2422 kXOR = 10,
2423 kEQUAL = 11,
2424 kGREATER = 12,
2425 kLESS = 13
2426};
2427
2428namespace impl
2429{
2435template <>
2437{
2438 static constexpr int32_t kVALUE = 14;
2439};
2440} // namespace impl
2441
2462{
2463public:
2474 {
2475 return mImpl->setOperation(op);
2476 }
2477
2486 {
2487 return mImpl->getOperation();
2488 }
2489
2490protected:
2491 apiv::VElementWiseLayer* mImpl;
2492 virtual ~IElementWiseLayer() noexcept = default;
2493};
2494
2500enum class GatherMode : int32_t
2501{
2502 kDEFAULT = 0,
2503 kELEMENT = 1,
2504 kND = 2
2505};
2506
2512template <>
2513constexpr inline int32_t EnumMax<GatherMode>() noexcept
2514{
2515 return 3;
2516}
2517
2595class IGatherLayer : public ILayer
2596{
2597public:
2607 void setGatherAxis(int32_t axis) noexcept
2608 {
2609 mImpl->setGatherAxis(axis);
2610 }
2611
2619 int32_t getGatherAxis() const noexcept
2620 {
2621 return mImpl->getGatherAxis();
2622 }
2623
2642 void setNbElementWiseDims(int32_t elementWiseDims) noexcept
2643 {
2644 mImpl->setNbElementWiseDims(elementWiseDims);
2645 }
2646
2652 int32_t getNbElementWiseDims() const noexcept
2653 {
2654 return mImpl->getNbElementWiseDims();
2655 }
2656
2662 void setMode(GatherMode mode) noexcept
2663 {
2664 mImpl->setMode(mode);
2665 }
2666
2672 GatherMode getMode() const noexcept
2673 {
2674 return mImpl->getMode();
2675 }
2676
2677protected:
2678 apiv::VGatherLayer* mImpl;
2679 virtual ~IGatherLayer() noexcept = default;
2680};
2681
2692{
2693public:
2700 {
2701 return mImpl->getPlugin();
2702 }
2703
2704protected:
2705 apiv::VPluginV2Layer* mImpl;
2706 virtual ~IPluginV2Layer() noexcept = default;
2707};
2708
2719{
2720public:
2727 {
2728 return mImpl->getPlugin();
2729 }
2730
2731protected:
2732 apiv::VPluginV3Layer* mImpl;
2733 virtual ~IPluginV3Layer() noexcept = default;
2734};
2735
2752enum class UnaryOperation : int32_t
2753{
2754 kEXP = 0,
2755 kLOG = 1,
2756 kSQRT = 2,
2757 kRECIP = 3,
2758 kABS = 4,
2759 kNEG = 5,
2760 kSIN = 6,
2761 kCOS = 7,
2762 kTAN = 8,
2763 kSINH = 9,
2764 kCOSH = 10,
2765 kASIN = 11,
2766 kACOS = 12,
2767 kATAN = 13,
2768 kASINH = 14,
2769 kACOSH = 15,
2770 kATANH = 16,
2771 kCEIL = 17,
2772 kFLOOR = 18,
2773 kERF = 19,
2774 kNOT = 20,
2775 kSIGN = 21,
2776 kROUND = 22,
2777 kISINF = 23,
2778};
2779
2785template <>
2786constexpr inline int32_t EnumMax<UnaryOperation>() noexcept
2787{
2788 return 24;
2789}
2790
2798class IUnaryLayer : public ILayer
2799{
2800public:
2809 {
2810 mImpl->setOperation(op);
2811 }
2812
2819 {
2820 return mImpl->getOperation();
2821 }
2822
2823protected:
2824 apiv::VUnaryLayer* mImpl;
2825 virtual ~IUnaryLayer() noexcept = default;
2826};
2827
2846enum class ReduceOperation : int32_t
2847{
2848 kSUM = 0,
2849 kPROD = 1,
2850 kMAX = 2,
2851 kMIN = 3,
2852 kAVG = 4
2853};
2854
2860template <>
2861constexpr inline int32_t EnumMax<ReduceOperation>() noexcept
2862{
2863 return 5;
2864}
2865
2873class IReduceLayer : public ILayer
2874{
2875public:
2882 {
2883 mImpl->setOperation(op);
2884 }
2885
2892 {
2893 return mImpl->getOperation();
2894 }
2895
2901 void setReduceAxes(uint32_t reduceAxes) noexcept
2902 {
2903 mImpl->setReduceAxes(reduceAxes);
2904 }
2905
2911 uint32_t getReduceAxes() const noexcept
2912 {
2913 return mImpl->getReduceAxes();
2914 }
2915
2921 void setKeepDimensions(bool keepDimensions) noexcept
2922 {
2923 mImpl->setKeepDimensions(keepDimensions);
2924 }
2925
2931 bool getKeepDimensions() const noexcept
2932 {
2933 return mImpl->getKeepDimensions();
2934 }
2935
2936protected:
2937 apiv::VReduceLayer* mImpl;
2938 virtual ~IReduceLayer() noexcept = default;
2939};
2940
2951class IPaddingLayer : public ILayer
2952{
2953public:
2963 void setPrePaddingNd(Dims const& padding) noexcept
2964 {
2965 mImpl->setPrePaddingNd(padding);
2966 }
2967
2975 Dims getPrePaddingNd() const noexcept
2976 {
2977 return mImpl->getPrePaddingNd();
2978 }
2979
2989 void setPostPaddingNd(Dims const& padding) noexcept
2990 {
2991 mImpl->setPostPaddingNd(padding);
2992 }
2993
3001 Dims getPostPaddingNd() const noexcept
3002 {
3003 return mImpl->getPostPaddingNd();
3004 }
3005
3006protected:
3007 apiv::VPaddingLayer* mImpl;
3008 virtual ~IPaddingLayer() noexcept = default;
3009};
3010
3017{
3024 int32_t order[Dims::MAX_DIMS];
3025};
3026
3039class IShuffleLayer : public ILayer
3040{
3041public:
3051 void setFirstTranspose(Permutation permutation) noexcept
3052 {
3053 mImpl->setFirstTranspose(permutation);
3054 }
3055
3064 {
3065 return mImpl->getFirstTranspose();
3066 }
3067
3088 void setReshapeDimensions(Dims const& dimensions) noexcept
3089 {
3090 mImpl->setReshapeDimensions(dimensions);
3091 }
3092
3102 {
3103 return mImpl->getReshapeDimensions();
3104 }
3105
3111 //
3134 using ILayer::setInput;
3135
3148 void setSecondTranspose(Permutation permutation) noexcept
3149 {
3150 mImpl->setSecondTranspose(permutation);
3151 }
3152
3161 {
3162 return mImpl->getSecondTranspose();
3163 }
3164
3176 void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
3177 {
3178 return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);
3179 }
3180
3189 bool getZeroIsPlaceholder() const noexcept
3190 {
3191 return mImpl->getZeroIsPlaceholder();
3192 }
3193
3194protected:
3195 apiv::VShuffleLayer* mImpl;
3196 virtual ~IShuffleLayer() noexcept = default;
3197};
3198
3204enum class SampleMode : int32_t
3205{
3206 kSTRICT_BOUNDS = 0,
3207 kWRAP = 1,
3208 kCLAMP = 2,
3209 kFILL = 3,
3210 kREFLECT = 4,
3213};
3214
3220template <>
3221constexpr inline int32_t EnumMax<SampleMode>() noexcept
3222{
3223 return 5;
3224}
3225
3268class ISliceLayer : public ILayer
3269{
3270public:
3280 void setStart(Dims const& start) noexcept
3281 {
3282 mImpl->setStart(start);
3283 }
3284
3295 Dims getStart() const noexcept
3296 {
3297 return mImpl->getStart();
3298 }
3299
3309 void setSize(Dims const& size) noexcept
3310 {
3311 return mImpl->setSize(size);
3312 }
3313
3324 Dims getSize() const noexcept
3325 {
3326 return mImpl->getSize();
3327 }
3328
3338 void setStride(Dims const& stride) noexcept
3339 {
3340 mImpl->setStride(stride);
3341 }
3342
3353 Dims getStride() const noexcept
3354 {
3355 return mImpl->getStride();
3356 }
3357
3363 void setMode(SampleMode mode) noexcept
3364 {
3365 mImpl->setMode(mode);
3366 }
3367
3373 SampleMode getMode() const noexcept
3374 {
3375 return mImpl->getMode();
3376 }
3377
3401 using ILayer::setInput;
3402
3403protected:
3404 apiv::VSliceLayer* mImpl;
3405 virtual ~ISliceLayer() noexcept = default;
3406};
3407
3420class IShapeLayer : public ILayer
3421{
3422protected:
3423 apiv::VShapeLayer* mImpl;
3424 virtual ~IShapeLayer() noexcept = default;
3425};
3426
3432enum class TopKOperation : int32_t
3433{
3434 kMAX = 0,
3435 kMIN = 1,
3436};
3437
3443template <>
3444constexpr inline int32_t EnumMax<TopKOperation>() noexcept
3445{
3446 return 2;
3447}
3448
3460class ITopKLayer : public ILayer
3461{
3462public:
3468 void setOperation(TopKOperation op) noexcept
3469 {
3470 mImpl->setOperation(op);
3471 }
3472
3479 {
3480 return mImpl->getOperation();
3481 }
3482
3492 void setK(int32_t k) noexcept
3493 {
3494 mImpl->setK(k);
3495 }
3496
3506 int32_t getK() const noexcept
3507 {
3508 return mImpl->getK();
3509 }
3510
3516 void setReduceAxes(uint32_t reduceAxes) noexcept
3517 {
3518 mImpl->setReduceAxes(reduceAxes);
3519 }
3520
3526 uint32_t getReduceAxes() const noexcept
3527 {
3528 return mImpl->getReduceAxes();
3529 }
3530
3545 using ILayer::setInput;
3546
3547protected:
3548 apiv::VTopKLayer* mImpl;
3549 virtual ~ITopKLayer() noexcept = default;
3550};
3551
3558enum class MatrixOperation : int32_t
3559{
3563 kNONE = 0,
3564
3566 kTRANSPOSE = 1,
3567
3578 kVECTOR = 2,
3579};
3580
3586template <>
3587constexpr inline int32_t EnumMax<MatrixOperation>() noexcept
3588{
3589 return 3;
3590}
3591
3618{
3619public:
3628 void setOperation(int32_t index, MatrixOperation op) noexcept
3629 {
3630 mImpl->setOperation(index, op);
3631 }
3632
3640 MatrixOperation getOperation(int32_t index) const noexcept
3641 {
3642 return mImpl->getOperation(index);
3643 }
3644
3645protected:
3646 apiv::VMatrixMultiplyLayer* mImpl;
3647 virtual ~IMatrixMultiplyLayer() noexcept = default;
3648};
3649
3671class INonZeroLayer : public ILayer
3672{
3673protected:
3674 virtual ~INonZeroLayer() noexcept = default;
3675 apiv::VNonZeroLayer* mImpl;
3676};
3677
3693{
3694protected:
3695 apiv::VRaggedSoftMaxLayer* mImpl;
3696 virtual ~IRaggedSoftMaxLayer() noexcept = default;
3697};
3698
3727{
3728protected:
3729 apiv::VIdentityLayer* mImpl;
3730 virtual ~IIdentityLayer() noexcept = default;
3731};
3732
3739class ICastLayer : public ILayer
3740{
3741public:
3749 void setToType(DataType toType) noexcept
3750 {
3751 mImpl->setToType(toType);
3752 }
3753
3760 DataType getToType() const noexcept
3761 {
3762 return mImpl->getToType();
3763 }
3764
3765protected:
3766 apiv::VCastLayer* mImpl;
3767 virtual ~ICastLayer() noexcept = default;
3768};
3769
3779{
3780public:
3789 void setWeights(Weights weights) noexcept
3790 {
3791 mImpl->setWeights(weights);
3792 }
3793
3799 Weights getWeights() const noexcept
3800 {
3801 return mImpl->getWeights();
3802 }
3803
3811 void setDimensions(Dims const& dimensions) noexcept
3812 {
3813 mImpl->setDimensions(dimensions);
3814 }
3815
3823 Dims getDimensions() const noexcept
3824 {
3825 return mImpl->getDimensions();
3826 }
3827
3828protected:
3829 apiv::VConstantLayer* mImpl;
3830 virtual ~IConstantLayer() noexcept = default;
3831};
3832
3843{
3844protected:
3845 apiv::VParametricReLULayer* mImpl;
3846 virtual ~IParametricReLULayer() noexcept = default;
3847};
3848
3854enum class InterpolationMode : int32_t
3855{
3856 kNEAREST = 0,
3857 kLINEAR = 1,
3858 kCUBIC = 2
3859};
3860
3861namespace impl
3862{
3868template <>
3870{
3871 static constexpr int32_t kVALUE = 3;
3872};
3873} // namespace impl
3874
3883{
3896 kALIGN_CORNERS = 0,
3897
3904 kASYMMETRIC = 1,
3905
3912 kHALF_PIXEL = 2,
3913};
3914
3915namespace impl
3916{
3922template <>
3924{
3925 static constexpr int32_t kVALUE = 3;
3926};
3927} // namespace impl
3928
3936enum class ResizeSelector : int32_t
3937{
3939 kFORMULA = 0,
3940
3942 kUPPER = 1,
3943};
3944
3945namespace impl
3946{
3952template <>
3954{
3955 static constexpr int32_t kVALUE = 2;
3956};
3957} // namespace impl
3958
3966enum class ResizeRoundMode : int32_t
3967{
3969 kHALF_UP = 0,
3970
3972 kHALF_DOWN = 1,
3973
3975 kFLOOR = 2,
3976
3978 kCEIL = 3,
3979};
3980
3981namespace impl
3982{
3988template <>
3990{
3991 static constexpr int32_t kVALUE = 4;
3992};
3993} // namespace impl
3994
4031class IResizeLayer : public ILayer
4032{
4033public:
4052 void setOutputDimensions(Dims const& dimensions) noexcept
4053 {
4054 return mImpl->setOutputDimensions(dimensions);
4055 }
4056
4062 Dims getOutputDimensions() const noexcept
4063 {
4064 return mImpl->getOutputDimensions();
4065 }
4066
4092 void setScales(float const* scales, int32_t nbScales) noexcept
4093 {
4094 mImpl->setScales(scales, nbScales);
4095 }
4096
4111 int32_t getScales(int32_t size, float* scales) const noexcept
4112 {
4113 return mImpl->getScales(size, scales);
4114 }
4115
4123 void setResizeMode(InterpolationMode interpolationMode) noexcept
4124 {
4125 mImpl->setResizeMode(interpolationMode);
4126 }
4127
4134 {
4135 return mImpl->getResizeMode();
4136 }
4137
4157 using ILayer::setInput;
4158
4169 {
4170 mImpl->setCoordinateTransformation(coordTransform);
4171 }
4172
4179 {
4180 return mImpl->getCoordinateTransformation();
4181 }
4182
4194 {
4195 mImpl->setSelectorForSinglePixel(selector);
4196 }
4197
4204 {
4205 return mImpl->getSelectorForSinglePixel();
4206 }
4207
4218 {
4219 mImpl->setNearestRounding(value);
4220 }
4221
4228 {
4229 return mImpl->getNearestRounding();
4230 }
4231
4249 void setCubicCoeff(float A) noexcept
4250 {
4251 mImpl->setCubicCoeff(A);
4252 }
4253
4259 float getCubicCoeff() const noexcept
4260 {
4261 return mImpl->getCubicCoeff();
4262 }
4263
4272 void setExcludeOutside(bool excludeFlag) noexcept
4273 {
4274 mImpl->setExcludeOutside(excludeFlag);
4275 }
4276
4282 bool getExcludeOutside() const noexcept
4283 {
4284 return mImpl->getExcludeOutside();
4285 }
4286
4287protected:
4288 virtual ~IResizeLayer() noexcept = default;
4289 apiv::VResizeLayer* mImpl;
4290};
4291
4295enum class LoopOutput : int32_t
4296{
4298 kLAST_VALUE = 0,
4299
4301 kCONCATENATE = 1,
4302
4304 kREVERSE = 2
4305};
4306
4312template <>
4313constexpr inline int32_t EnumMax<LoopOutput>() noexcept
4314{
4315 return 3;
4316}
4317
4321enum class TripLimit : int32_t
4322{
4323
4324 kCOUNT = 0,
4325 kWHILE = 1
4326};
4327
4333template <>
4334constexpr inline int32_t EnumMax<TripLimit>() noexcept
4335{
4336 return 2;
4337}
4338
4339class ILoop;
4340
4347{
4348public:
4352 ILoop* getLoop() const noexcept
4353 {
4354 return mBoundary->getLoop();
4355 }
4356
4357protected:
4358 virtual ~ILoopBoundaryLayer() noexcept = default;
4359 apiv::VLoopBoundaryLayer* mBoundary;
4360};
4361
4370{
4371public:
4376 {
4377 return mBoundary->getConditional();
4378 }
4379
4380protected:
4381 virtual ~IIfConditionalBoundaryLayer() noexcept = default;
4382 apiv::VConditionalBoundaryLayer* mBoundary;
4383};
4384
4391{
4392public:
4393protected:
4394 virtual ~IConditionLayer() noexcept = default;
4395 apiv::VConditionLayer* mImpl;
4396};
4397
4406{
4407public:
4408protected:
4409 virtual ~IIfConditionalOutputLayer() noexcept = default;
4410 apiv::VConditionalOutputLayer* mImpl;
4411};
4412
4419{
4420public:
4421protected:
4422 virtual ~IIfConditionalInputLayer() noexcept = default;
4423 apiv::VConditionalInputLayer* mImpl;
4424};
4425
4450{
4451public:
4462 {
4463 return mImpl->setCondition(condition);
4464 }
4465
4477 IIfConditionalOutputLayer* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput) noexcept
4478 {
4479 return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);
4480 }
4481
4490 {
4491 return mImpl->addInput(input);
4492 }
4493
4504 void setName(char const* name) noexcept
4505 {
4506 mImpl->setName(name);
4507 }
4508
4514 char const* getName() const noexcept
4515 {
4516 return mImpl->getName();
4517 }
4518
4519protected:
4520 virtual ~IIfConditional() noexcept = default;
4521 apiv::VIfConditional* mImpl;
4522};
4523
4530{
4531public:
4537 //
4550 using ILayer::setInput;
4551
4552protected:
4553 virtual ~IRecurrenceLayer() noexcept = default;
4554 apiv::VRecurrenceLayer* mImpl;
4555};
4556
4577{
4578public:
4582 LoopOutput getLoopOutput() const noexcept
4583 {
4584 return mImpl->getLoopOutput();
4585 }
4586
4599 void setAxis(int32_t axis) noexcept
4600 {
4601 mImpl->setAxis(axis);
4602 }
4603
4607 int32_t getAxis() const noexcept
4608 {
4609 return mImpl->getAxis();
4610 }
4611
4617 //
4632 using ILayer::setInput;
4633
4634protected:
4635 virtual ~ILoopOutputLayer() noexcept = default;
4636 apiv::VLoopOutputLayer* mImpl;
4637};
4638
4645{
4646public:
4650 TripLimit getTripLimit() const noexcept
4651 {
4652 return mImpl->getTripLimit();
4653 }
4654
4655protected:
4656 virtual ~ITripLimitLayer() noexcept = default;
4657 apiv::VTripLimitLayer* mImpl;
4658};
4659
4666{
4667public:
4671 void setAxis(int32_t axis) noexcept
4672 {
4673 mImpl->setAxis(axis);
4674 }
4675
4679 int32_t getAxis() const noexcept
4680 {
4681 return mImpl->getAxis();
4682 }
4683
4693 void setReverse(bool reverse) noexcept
4694 {
4695 mImpl->setReverse(reverse);
4696 }
4697
4703 bool getReverse() const noexcept
4704 {
4705 return mImpl->getReverse();
4706 }
4707
4708protected:
4709 virtual ~IIteratorLayer() noexcept = default;
4710 apiv::VIteratorLayer* mImpl;
4711};
4712
4718class ILoop : public INoCopy
4719{
4720public:
4727 IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept
4728 {
4729 return mImpl->addRecurrence(initialValue);
4730 }
4731
4749 {
4750 return mImpl->addTripLimit(tensor, limit);
4751 }
4752
4761 IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept
4762 {
4763 return mImpl->addIterator(tensor, axis, reverse);
4764 }
4765
4774 ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept
4775 {
4776 return mImpl->addLoopOutput(tensor, outputKind, axis);
4777 }
4778
4789 void setName(char const* name) noexcept
4790 {
4791 mImpl->setName(name);
4792 }
4793
4799 char const* getName() const noexcept
4800 {
4801 return mImpl->getName();
4802 }
4803
4804protected:
4805 virtual ~ILoop() noexcept = default;
4806 apiv::VLoop* mImpl;
4807};
4808
4816class ISelectLayer : public ILayer
4817{
4818protected:
4819 virtual ~ISelectLayer() noexcept = default;
4820 apiv::VSelectLayer* mImpl;
4821};
4822
4839{
4840public:
4849 void setMessage(char const* message) noexcept
4850 {
4851 mImpl->setMessage(message);
4852 }
4853
4859 char const* getMessage() const noexcept
4860 {
4861 return mImpl->getMessage();
4862 }
4863
4864protected:
4865 virtual ~IAssertionLayer() noexcept = default;
4866
4867 apiv::VAssertionLayer* mImpl;
4868};
4869
4877enum class FillOperation : int32_t
4878{
4894 kLINSPACE = 0,
4895
4897 kRANDOM_UNIFORM = 1,
4898
4900 kRANDOM_NORMAL = 2
4901};
4902
4908template <>
4909constexpr inline int32_t EnumMax<FillOperation>() noexcept
4910{
4911 return 3;
4912}
4913
4949class IFillLayer : public ILayer
4950{
4951public:
4960 //
4961 void setDimensions(Dims const& dimensions) noexcept
4962 {
4963 mImpl->setDimensions(dimensions);
4964 }
4965
4976 Dims getDimensions() const noexcept
4977 {
4978 return mImpl->getDimensions();
4979 }
4980
4986 void setOperation(FillOperation op) noexcept
4987 {
4988 mImpl->setOperation(op);
4989 }
4990
4997 {
4998 return mImpl->getOperation();
4999 }
5000
5014 //
5015 void setAlpha(double alpha) noexcept
5016 {
5017 mImpl->setAlpha(alpha);
5018 }
5019
5030 double getAlpha() const noexcept
5031 {
5032 return mImpl->getAlpha();
5033 }
5034
5049 void setBeta(double beta) noexcept
5050 {
5051 mImpl->setBeta(beta);
5052 }
5053
5064 double getBeta() const noexcept
5065 {
5066 return mImpl->getBeta();
5067 }
5068
5109 using ILayer::setInput;
5110
5124 //
5125 void setAlphaInt64(int64_t alpha) noexcept
5126 {
5127 mImpl->setAlphaInt64(alpha);
5128 }
5129
5140 int64_t getAlphaInt64() const noexcept
5141 {
5142 return mImpl->getAlphaInt64();
5143 }
5144
5159 void setBetaInt64(int64_t beta) noexcept
5160 {
5161 mImpl->setBetaInt64(beta);
5162 }
5163
5174 int64_t getBetaInt64() const noexcept
5175 {
5176 return mImpl->getBetaInt64();
5177 }
5178
5182 bool isAlphaBetaInt64() const noexcept
5183 {
5184 return mImpl->isAlphaBetaInt64();
5185 }
5186
5199 void setToType(DataType toType) noexcept
5200 {
5201 mImpl->setToType(toType);
5202 }
5203
5211 DataType getToType() const noexcept
5212 {
5213 return mImpl->getToType();
5214 }
5215
5216protected:
5217 virtual ~IFillLayer() noexcept = default;
5218 apiv::VFillLayer* mImpl;
5219};
5220
5295{
5296public:
5305 int32_t getAxis() const noexcept
5306 {
5307 return mImpl->getAxis();
5308 }
5316 void setAxis(int32_t axis) noexcept
5317 {
5318 mImpl->setAxis(axis);
5319 }
5320
5332 void setToType(DataType toType) noexcept
5333 {
5334 mImpl->setToType(toType);
5335 }
5336
5344 DataType getToType() const noexcept
5345 {
5346 return mImpl->getToType();
5347 }
5348
5349protected:
5350 virtual ~IQuantizeLayer() noexcept = default;
5351 apiv::VQuantizeLayer* mImpl;
5352};
5353
5425{
5426public:
5435 int32_t getAxis() const noexcept
5436 {
5437 return mImpl->getAxis();
5438 }
5446 void setAxis(int32_t axis) noexcept
5447 {
5448 mImpl->setAxis(axis);
5449 }
5450
5462 void setToType(DataType toType) noexcept
5463 {
5464 mImpl->setToType(toType);
5465 }
5466
5474 DataType getToType() const noexcept
5475 {
5476 return mImpl->getToType();
5477 }
5478
5479protected:
5480 virtual ~IDequantizeLayer() noexcept = default;
5481 apiv::VDequantizeLayer* mImpl;
5482};
5483
5520class IEinsumLayer : public ILayer
5521{
5522public:
5532 bool setEquation(char const* equation) noexcept
5533 {
5534 return mImpl->setEquation(equation);
5535 }
5536
5542 char const* getEquation() const noexcept
5543 {
5544 return mImpl->getEquation();
5545 }
5546
5547protected:
5548 virtual ~IEinsumLayer() noexcept = default;
5549 apiv::VEinsumLayer* mImpl;
5550};
5551
5559enum class ScatterMode : int32_t
5560{
5561 kELEMENT = 0,
5562 kND = 1,
5563};
5564
5570template <>
5571constexpr inline int32_t EnumMax<ScatterMode>() noexcept
5572{
5573 return 2;
5574}
5575
5632class IScatterLayer : public ILayer
5633{
5634public:
5640 void setMode(ScatterMode mode) noexcept
5641 {
5642 mImpl->setMode(mode);
5643 }
5644
5650 ScatterMode getMode() const noexcept
5651 {
5652 return mImpl->getMode();
5653 }
5654
5660 void setAxis(int32_t axis) noexcept
5661 {
5662 mImpl->setAxis(axis);
5663 }
5664
5668 int32_t getAxis() const noexcept
5669 {
5670 return mImpl->getAxis();
5671 }
5672
5673protected:
5674 apiv::VScatterLayer* mImpl;
5675 virtual ~IScatterLayer() noexcept = default;
5676}; // class IScatterLayer
5677
5704class IOneHotLayer : public ILayer
5705{
5706public:
5712 void setAxis(int32_t axis) noexcept
5713 {
5714 mImpl->setAxis(axis);
5715 }
5716
5720 int32_t getAxis() const noexcept
5721 {
5722 return mImpl->getAxis();
5723 }
5724
5725protected:
5726 apiv::VOneHotLayer* mImpl;
5727};
5728
5741{
5742public:
5749 {
5750 mImpl->setInterpolationMode(mode);
5751 }
5752
5761 {
5762 return mImpl->getInterpolationMode();
5763 }
5764
5770 void setAlignCorners(bool alignCorners) noexcept
5771 {
5772 mImpl->setAlignCorners(alignCorners);
5773 }
5774
5782 bool getAlignCorners() const noexcept
5783 {
5784 return mImpl->getAlignCorners();
5785 }
5786
5794 bool setSampleMode(SampleMode mode) noexcept
5795 {
5796 return mImpl->setSampleMode(mode);
5797 }
5798
5806 SampleMode getSampleMode() const noexcept
5807 {
5808 return mImpl->getSampleMode();
5809 }
5810
5811protected:
5812 apiv::VGridSampleLayer* mImpl;
5813 virtual ~IGridSampleLayer() noexcept = default;
5814}; // class IGridSampleLayer
5815
5823enum class BoundingBoxFormat : int32_t
5824{
5826 kCORNER_PAIRS = 0,
5828 kCENTER_SIZES = 1
5829};
5830
5836template <>
5837constexpr inline int32_t EnumMax<BoundingBoxFormat>() noexcept
5838{
5839 return 2;
5840}
5841
5888class INMSLayer : public ILayer
5889{
5890public:
5901 {
5902 mImpl->setBoundingBoxFormat(fmt);
5903 }
5904
5913 {
5914 return mImpl->getBoundingBoxFormat();
5915 }
5916
5926 void setTopKBoxLimit(int32_t limit) noexcept
5927 {
5928 mImpl->setTopKBoxLimit(limit);
5929 }
5930
5936 int32_t getTopKBoxLimit() const noexcept
5937 {
5938 return mImpl->getTopKBoxLimit();
5939 }
5940
5959 using ILayer::setInput;
5960
5961protected:
5962 apiv::VNMSLayer* mImpl;
5963 virtual ~INMSLayer() noexcept = default;
5964}; // class INMSLayer
5965
5979{
5980public:
5989 void setBatchAxis(int32_t batchAxis) noexcept
5990 {
5991 mImpl->setBatchAxis(batchAxis);
5992 }
5993
5999 int32_t getBatchAxis() const noexcept
6000 {
6001 return mImpl->getBatchAxis();
6002 }
6003
6012 void setSequenceAxis(int32_t sequenceAxis) noexcept
6013 {
6014 mImpl->setSequenceAxis(sequenceAxis);
6015 }
6016
6022 int32_t getSequenceAxis() const noexcept
6023 {
6024 return mImpl->getSequenceAxis();
6025 }
6026
6027protected:
6028 apiv::VReverseSequenceLayer* mImpl;
6029 virtual ~IReverseSequenceLayer() noexcept = default;
6030}; // class IReverseSequenceLayer
6031
6051{
6052public:
6060 void setEpsilon(float eps) noexcept
6061 {
6062 return mImpl->setEpsilon(eps);
6063 }
6064
6070 float getEpsilon() const noexcept
6071 {
6072 return mImpl->getEpsilon();
6073 }
6074
6080 void setAxes(uint32_t axesMask) noexcept
6081 {
6082 return mImpl->setAxes(axesMask);
6083 }
6084
6090 uint32_t getAxes() const noexcept
6091 {
6092 return mImpl->getAxes();
6093 }
6094
6111 void setNbGroups(int64_t nbGroups) noexcept
6112 {
6113 return mImpl->setNbGroups(nbGroups);
6114 }
6115
6121 int64_t getNbGroups() const noexcept
6122 {
6123 return mImpl->getNbGroups();
6124 }
6125
6146 void setComputePrecision(DataType type) noexcept
6147 {
6148 return mImpl->setComputePrecision(type);
6149 }
6150
6157 {
6158 return mImpl->getComputePrecision();
6159 }
6160
6161protected:
6162 apiv::VNormalizationLayer* mImpl;
6163 virtual ~INormalizationLayer() noexcept = default;
6164};
6165
6184{
6185public:
6186 virtual ~INetworkDefinition() noexcept = default;
6187
6224 ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept
6225 {
6226 return mImpl->addInput(name, type, dimensions);
6227 }
6228
6238 void markOutput(ITensor& tensor) noexcept
6239 {
6240 mImpl->markOutput(tensor);
6241 }
6242
6256 bool markDebug(ITensor& tensor) noexcept
6257 {
6258 return mImpl->markDebug(tensor);
6259 }
6260
6272 bool unmarkDebug(ITensor& tensor) noexcept
6273 {
6274 return mImpl->unmarkDebug(tensor);
6275 }
6276
6282 bool isDebugTensor(nvinfer1::ITensor const& tensor) const noexcept
6283 {
6284 return mImpl->isDebugTensor(tensor);
6285 }
6286
6303 {
6304 return mImpl->addActivation(input, type);
6305 }
6306
6321 ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept
6322 {
6323 return mImpl->addLRN(input, window, alpha, beta, k);
6324 }
6325
6347 IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
6348 {
6349 return mImpl->addScale(input, mode, shift, scale, power);
6350 }
6351
6361 {
6362 return mImpl->addSoftMax(input);
6363 }
6364
6377 IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept
6378 {
6379 return mImpl->addConcatenation(inputs, nbInputs);
6380 }
6381
6405 {
6406 return mImpl->addElementWise(input1, input2, op);
6407 }
6408
6426 IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept
6427 {
6428 return mImpl->addUnary(input, operation);
6429 }
6430
6441 {
6442 return mImpl->addShuffle(input);
6443 }
6444
6457 IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept
6458 {
6459 return mImpl->addOneHot(indices, values, depth, axis);
6460 }
6461
6469 int32_t getNbLayers() const noexcept
6470 {
6471 return mImpl->getNbLayers();
6472 }
6473
6483 ILayer* getLayer(int32_t index) const noexcept
6484 {
6485 return mImpl->getLayer(index);
6486 }
6487
6495 int32_t getNbInputs() const noexcept
6496 {
6497 return mImpl->getNbInputs();
6498 }
6499
6511 ITensor* getInput(int32_t index) const noexcept
6512 {
6513 return mImpl->getInput(index);
6514 }
6515
6525 int32_t getNbOutputs() const noexcept
6526 {
6527 return mImpl->getNbOutputs();
6528 }
6529
6541 ITensor* getOutput(int32_t index) const noexcept
6542 {
6543 return mImpl->getOutput(index);
6544 }
6545
6568 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
6569 {
6570 return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);
6571 }
6572
6600 ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
6601 {
6602 return mImpl->addTopK(input, op, k, reduceAxes);
6603 }
6604
6616 IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept
6617 {
6618 return mImpl->addGather(data, indices, axis);
6619 }
6620
6632 IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept
6633 {
6634 return mImpl->addGatherV2(data, indices, mode);
6635 }
6636
6652 {
6653 return mImpl->addRaggedSoftMax(input, bounds);
6654 }
6655
6673 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
6674 {
6675 return mImpl->addMatrixMultiply(input0, op0, input1, op1);
6676 }
6677
6688 {
6689 return mImpl->addNonZero(input);
6690 }
6691
6711 IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept
6712 {
6713 return mImpl->addConstant(dimensions, weights);
6714 }
6715
6726 {
6727 return mImpl->addIdentity(input);
6728 }
6729
6740 ICastLayer* addCast(ITensor& input, DataType toType) noexcept
6741 {
6742 return mImpl->addCast(input, toType);
6743 }
6744
6755 void removeTensor(ITensor& tensor) noexcept
6756 {
6757 mImpl->removeTensor(tensor);
6758 }
6759
6767 void unmarkOutput(ITensor& tensor) noexcept
6768 {
6769 mImpl->unmarkOutput(tensor);
6770 }
6771
6786 IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept
6787 {
6788 return mImpl->addPluginV2(inputs, nbInputs, plugin);
6789 }
6790
6804 IPluginV3Layer* addPluginV3(ITensor* const* inputs, int32_t nbInputs, ITensor* const* shapeInputs,
6805 int32_t nbShapeInputs, IPluginV3& plugin) noexcept
6806 {
6807 return mImpl->addPluginV3(inputs, nbInputs, shapeInputs, nbShapeInputs, plugin);
6808 }
6809
6824 ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept
6825 {
6826 return mImpl->addSlice(input, start, size, stride);
6827 }
6828
6848 void setName(char const* name) noexcept
6849 {
6850 mImpl->setName(name);
6851 }
6852
6862 char const* getName() const noexcept
6863 {
6864 return mImpl->getName();
6865 }
6866
6878 IShapeLayer* addShape(ITensor& input) noexcept
6879 {
6880 return mImpl->addShape(input);
6881 }
6882
6893 {
6894 return mImpl->hasImplicitBatchDimension();
6895 }
6896
6903 {
6904 return mImpl->getFlags();
6905 }
6906
6914 bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept
6915 {
6916 return mImpl->getFlag(networkDefinitionCreationFlag);
6917 }
6918
6931 bool markOutputForShapes(ITensor& tensor) noexcept
6932 {
6933 return mImpl->markOutputForShapes(tensor);
6934 }
6935
6943 bool unmarkOutputForShapes(ITensor& tensor) noexcept
6944 {
6945 return mImpl->unmarkOutputForShapes(tensor);
6946 }
6947
6962 {
6963 return mImpl->addParametricReLU(input, slope);
6964 }
6965
6984 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6985 {
6986 return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6987 }
6988
7003 IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept
7004 {
7005 return mImpl->addPoolingNd(input, type, windowSize);
7006 }
7007
7022 //
7026 ITensor& input, int64_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7027 {
7028 return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7029 }
7030
7063 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
7064 {
7065 return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);
7066 }
7067
7080 {
7081 return mImpl->addResize(input);
7082 }
7083
7093 ILoop* addLoop() noexcept
7094 {
7095 return mImpl->addLoop();
7096 }
7097
7109 {
7110 return mImpl->addIfConditional();
7111 }
7112
7147 ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept
7148 {
7149 return mImpl->addSelect(condition, thenInput, elseInput);
7150 }
7151
7164 IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept
7165 {
7166 return mImpl->addAssertion(condition, message);
7167 }
7168
7189 TRT_DEPRECATED IFillLayer* addFill(Dims const& dimensions, FillOperation op) noexcept
7190 {
7191 return mImpl->addFill(dimensions, op);
7192 }
7193
7215 IFillLayer* addFill(Dims const& dimensions, FillOperation op, DataType outputType) noexcept
7216 {
7217 return mImpl->addFillV2(dimensions, op, outputType);
7218 }
7219
7231 IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept
7232 {
7233 return mImpl->addPaddingNd(input, prePadding, postPadding);
7234 }
7235
7255 bool setWeightsName(Weights weights, char const* name) noexcept
7256 {
7257 return mImpl->setWeightsName(weights, name);
7258 }
7259
7271 //
7274 void setErrorRecorder(IErrorRecorder* recorder) noexcept
7275 {
7276 mImpl->setErrorRecorder(recorder);
7277 }
7278
7290 {
7291 return mImpl->getErrorRecorder();
7292 }
7293
7311 {
7312 return mImpl->addDequantize(input, scale);
7313 }
7314
7331 IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale, DataType outputType) noexcept
7332 {
7333 return mImpl->addDequantizeV2(input, scale, outputType);
7334 }
7335
7351 IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept
7352 {
7353 return mImpl->addScatter(data, indices, updates, mode);
7354 }
7355
7373 {
7374 return mImpl->addQuantize(input, scale);
7375 }
7376
7393 IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale, DataType outputType) noexcept
7394 {
7395 return mImpl->addQuantizeV2(input, scale, outputType);
7396 }
7397
7408 IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept
7409 {
7410 return mImpl->addEinsum(inputs, nbInputs, equation);
7411 }
7412
7427 {
7428 return mImpl->addGridSample(input, grid);
7429 }
7430
7444 INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept
7445 {
7446 return mImpl->addNMS(boxes, scores, maxOutputBoxesPerClass);
7447 }
7448
7462 {
7463 return mImpl->addReverseSequence(input, sequenceLens);
7464 }
7465
7487 INormalizationLayer* addNormalization(ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept
7488 {
7489 return mImpl->addNormalization(input, scale, bias, axesMask);
7490 }
7491
7498 virtual IBuilder& getBuilder() const noexcept
7499 {
7500 return mImpl->getBuilder();
7501 }
7502
7503protected:
7504 apiv::VNetworkDefinition* mImpl;
7505};
7506
7512enum class CalibrationAlgoType : int32_t
7513{
7518};
7519
7525template <>
7526constexpr inline int32_t EnumMax<CalibrationAlgoType>() noexcept
7527{
7528 return 4;
7529}
7530
7543{
7544public:
7552 TRT_DEPRECATED virtual int32_t getBatchSize() const noexcept = 0;
7553
7568 virtual bool getBatch(void* bindings[], char const* names[], int32_t nbBindings) noexcept = 0;
7569
7584 virtual void const* readCalibrationCache(std::size_t& length) noexcept = 0;
7585
7594 virtual void writeCalibrationCache(void const* ptr, std::size_t length) noexcept = 0;
7595
7601 virtual CalibrationAlgoType getAlgorithm() noexcept = 0;
7602
7603 ~IInt8Calibrator() noexcept override = default;
7604};
7605
7606namespace v_1_0
7607{
7609{
7610public:
7614 InterfaceInfo getInterfaceInfo() const noexcept override
7615 {
7616 return InterfaceInfo{"IInt8EntropyCalibrator", 1, 0};
7617 }
7618
7623 {
7625 }
7626
7627 ~IInt8EntropyCalibrator() noexcept override = default;
7628};
7629} // namespace v_1_0
7630
7643
7644namespace v_1_0
7645{
7647{
7648public:
7652 InterfaceInfo getInterfaceInfo() const noexcept override
7653 {
7654 return InterfaceInfo{"IInt8EntropyCalibrator2", 1, 0};
7655 }
7656
7661 {
7663 }
7664
7665 ~IInt8EntropyCalibrator2() noexcept override = default;
7666};
7667} // namespace v_1_0
7668
7681
7682namespace v_1_0
7683{
7685{
7686public:
7690 InterfaceInfo getInterfaceInfo() const noexcept override
7691 {
7692 return InterfaceInfo{"IInt8MinMaxCalibrator", 1, 0};
7693 }
7694
7699 {
7701 }
7702
7703 ~IInt8MinMaxCalibrator() noexcept override = default;
7704};
7705} // namespace v_1_0
7706
7718
7719namespace v_1_0
7720{
7722{
7723public:
7727 InterfaceInfo getInterfaceInfo() const noexcept override
7728 {
7729 return InterfaceInfo{"IInt8Calibrator", 1, 0};
7730 }
7731
7736 {
7738 }
7739
7746 virtual double getQuantile() const noexcept = 0;
7747
7754 virtual double getRegressionCutoff() const noexcept = 0;
7755
7768 virtual void const* readHistogramCache(std::size_t& length) noexcept = 0;
7769
7778 virtual void writeHistogramCache(void const* ptr, std::size_t length) noexcept = 0;
7779
7780 ~IInt8LegacyCalibrator() noexcept override = default;
7781};
7782} // namespace v_1_0
7783
7796
7808{
7809public:
7815 DataType getDataType() const noexcept
7816 {
7817 return mImpl->getDataType();
7818 }
7819
7826 Dims getStrides() const noexcept
7827 {
7828 return mImpl->getStrides();
7829 }
7830
7836 int64_t getVectorizedDim() const noexcept
7837 {
7838 return mImpl->getVectorizedDim();
7839 }
7840
7847 int64_t getComponentsPerElement() const noexcept
7848 {
7849 return mImpl->getComponentsPerElement();
7850 }
7851
7852protected:
7853 virtual ~IAlgorithmIOInfo() noexcept = default;
7854 apiv::VAlgorithmIOInfo* mImpl;
7855};
7856
7869{
7870public:
7874 int64_t getImplementation() const noexcept
7875 {
7876 return mImpl->getImplementation();
7877 }
7878
7882 int64_t getTactic() const noexcept
7883 {
7884 return mImpl->getTactic();
7885 }
7886
7887protected:
7888 virtual ~IAlgorithmVariant() noexcept = default;
7889 apiv::VAlgorithmVariant* mImpl;
7890};
7891
7901{
7902public:
7908 char const* getName() const noexcept
7909 {
7910 return mImpl->getName();
7911 }
7912
7920 Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
7921 {
7922 return mImpl->getDimensions(index, select);
7923 }
7924
7928 int32_t getNbInputs() const noexcept
7929 {
7930 return mImpl->getNbInputs();
7931 }
7932
7936 int32_t getNbOutputs() const noexcept
7937 {
7938 return mImpl->getNbOutputs();
7939 }
7940
7941protected:
7942 virtual ~IAlgorithmContext() noexcept = default;
7943 apiv::VAlgorithmContext* mImpl;
7944};
7945
7957class IAlgorithm : public INoCopy
7958{
7959public:
7964 {
7965 return mImpl->getAlgorithmVariant();
7966 }
7967
7971 float getTimingMSec() const noexcept
7972 {
7973 return mImpl->getTimingMSec();
7974 }
7975
7979 std::size_t getWorkspaceSize() const noexcept
7980 {
7981 return mImpl->getWorkspaceSize();
7982 }
7983
7993 IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept
7994 {
7995 return mImpl->getAlgorithmIOInfoByIndex(index);
7996 }
7997
7998protected:
7999 virtual ~IAlgorithm() noexcept = default;
8000 apiv::VAlgorithm* mImpl;
8001}; // IAlgorithm
8002
8003namespace v_1_0
8004{
8006{
8007public:
8011 InterfaceInfo getInterfaceInfo() const noexcept override
8012 {
8013 return InterfaceInfo{"IAlgorithmSelector", 1, 0};
8014 }
8029 virtual int32_t selectAlgorithms(IAlgorithmContext const& context, IAlgorithm const* const* choices,
8030 int32_t nbChoices, int32_t* selection) noexcept = 0;
8031
8042 virtual void reportAlgorithms(IAlgorithmContext const* const* algoContexts, IAlgorithm const* const* algoChoices,
8043 int32_t nbAlgorithms) noexcept = 0;
8044
8045 virtual ~IAlgorithmSelector() noexcept = default;
8046};
8047} // namespace v_1_0
8048
8060
8067using QuantizationFlags = uint32_t;
8068
8076enum class QuantizationFlag : int32_t
8077{
8082};
8083
8089template <>
8090constexpr inline int32_t EnumMax<QuantizationFlag>() noexcept
8091{
8092 return 1;
8093}
8094
8101using BuilderFlags = uint32_t;
8102
8110enum class BuilderFlag : int32_t
8111{
8113 kFP16 = 0,
8114
8116 kINT8 = 1,
8117
8119 kDEBUG = 2,
8120
8122 kGPU_FALLBACK = 3,
8123
8125 kREFIT = 4,
8126
8129
8133 kTF32 = 6,
8134
8136 kSPARSE_WEIGHTS = 7,
8137
8144 kSAFETY_SCOPE = 8,
8145
8148
8152
8156 kDIRECT_IO = 11,
8157
8160
8167
8173
8179 kFP8 = 15,
8180
8184
8187 kBF16 = 17,
8188
8194
8196 kSTRIP_PLAN = 19,
8197
8200
8207 kREFIT_IDENTICAL = 20,
8208
8234 kWEIGHT_STREAMING = 21,
8235};
8236
8242template <>
8243constexpr inline int32_t EnumMax<BuilderFlag>() noexcept
8244{
8245 return 22;
8246}
8247
8258class ITimingCache : public INoCopy
8259{
8260public:
8261 virtual ~ITimingCache() noexcept = default;
8262
8272 nvinfer1::IHostMemory* serialize() const noexcept
8273 {
8274 return mImpl->serialize();
8275 }
8276
8296 bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept
8297 {
8298 return mImpl->combine(inputCache, ignoreMismatch);
8299 }
8300
8306 bool reset() noexcept
8307 {
8308 return mImpl->reset();
8309 }
8310
8311protected:
8312 apiv::VTimingCache* mImpl;
8313};
8314
8322enum class MemoryPoolType : int32_t
8323{
8330 kWORKSPACE = 0,
8331
8339
8345 kDLA_LOCAL_DRAM = 2,
8346
8352 kDLA_GLOBAL_DRAM = 3,
8353
8361 kTACTIC_DRAM = 4,
8362
8373};
8374
8380template <>
8381constexpr inline int32_t EnumMax<MemoryPoolType>() noexcept
8382{
8383 return 6;
8384}
8385
8394enum class PreviewFeature : int32_t
8395{
8402};
8403namespace impl
8404{
8410template <>
8412{
8413 static constexpr int32_t kVALUE = 1;
8414};
8415} // namespace impl
8416
8427enum class HardwareCompatibilityLevel : int32_t
8428{
8431 kNONE = 0,
8432
8437 kAMPERE_PLUS = 1,
8438};
8439
8440namespace impl
8441{
8447template <>
8449{
8450 static constexpr int32_t kVALUE = 2;
8451};
8452} // namespace impl
8453
8454namespace v_1_0
8455{
8457{
8458public:
8459 IProgressMonitor() = default;
8460 virtual ~IProgressMonitor() noexcept = default;
8461
8465 InterfaceInfo getInterfaceInfo() const noexcept override
8466 {
8467 return InterfaceInfo{"IProgressMonitor", 1, 0};
8468 }
8469
8489 virtual void phaseStart(char const* phaseName, char const* parentPhase, int32_t nbSteps) noexcept = 0;
8490
8503 virtual bool stepComplete(char const* phaseName, int32_t step) noexcept = 0;
8504
8516 virtual void phaseFinish(char const* phaseName) noexcept = 0;
8517
8518}; // class IProgressMonitor
8519} // namespace v_1_0
8520
8541
8550{
8551public:
8552 virtual ~IBuilderConfig() noexcept = default;
8553
8562 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
8563 {
8564 mImpl->setAvgTimingIterations(avgTiming);
8565 }
8566
8574 int32_t getAvgTimingIterations() const noexcept
8575 {
8576 return mImpl->getAvgTimingIterations();
8577 }
8578
8587 void setEngineCapability(EngineCapability capability) noexcept
8588 {
8589 mImpl->setEngineCapability(capability);
8590 }
8591
8600 {
8601 return mImpl->getEngineCapability();
8602 }
8603
8609 void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept
8610 {
8611 mImpl->setInt8Calibrator(calibrator);
8612 }
8613
8618 {
8619 return mImpl->getInt8Calibrator();
8620 }
8621
8634 void setFlags(BuilderFlags builderFlags) noexcept
8635 {
8636 mImpl->setFlags(builderFlags);
8637 }
8638
8646 BuilderFlags getFlags() const noexcept
8647 {
8648 return mImpl->getFlags();
8649 }
8650
8658 void clearFlag(BuilderFlag builderFlag) noexcept
8659 {
8660 mImpl->clearFlag(builderFlag);
8661 }
8662
8670 void setFlag(BuilderFlag builderFlag) noexcept
8671 {
8672 mImpl->setFlag(builderFlag);
8673 }
8674
8682 bool getFlag(BuilderFlag builderFlag) const noexcept
8683 {
8684 return mImpl->getFlag(builderFlag);
8685 }
8686
8699 void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept
8700 {
8701 mImpl->setDeviceType(layer, deviceType);
8702 }
8703
8709 DeviceType getDeviceType(ILayer const* layer) const noexcept
8710 {
8711 return mImpl->getDeviceType(layer);
8712 }
8713
8721 bool isDeviceTypeSet(ILayer const* layer) const noexcept
8722 {
8723 return mImpl->isDeviceTypeSet(layer);
8724 }
8725
8731 void resetDeviceType(ILayer const* layer) noexcept
8732 {
8733 mImpl->resetDeviceType(layer);
8734 }
8735
8741 bool canRunOnDLA(ILayer const* layer) const noexcept
8742 {
8743 return mImpl->canRunOnDLA(layer);
8744 }
8745
8757 void setDLACore(int32_t dlaCore) noexcept
8758 {
8759 mImpl->setDLACore(dlaCore);
8760 }
8761
8767 int32_t getDLACore() const noexcept
8768 {
8769 return mImpl->getDLACore();
8770 }
8771
8778 void setDefaultDeviceType(DeviceType deviceType) noexcept
8779 {
8780 mImpl->setDefaultDeviceType(deviceType);
8781 }
8782
8789 {
8790 return mImpl->getDefaultDeviceType();
8791 }
8792
8798 void reset() noexcept
8799 {
8800 mImpl->reset();
8801 }
8802
8810 void setProfileStream(const cudaStream_t stream) noexcept
8811 {
8812 return mImpl->setProfileStream(stream);
8813 }
8814
8822 cudaStream_t getProfileStream() const noexcept
8823 {
8824 return mImpl->getProfileStream();
8825 }
8826
8839 int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept
8840 {
8841 return mImpl->addOptimizationProfile(profile);
8842 }
8843
8852 int32_t getNbOptimizationProfiles() const noexcept
8853 {
8854 return mImpl->getNbOptimizationProfiles();
8855 }
8856
8865 {
8866 mImpl->setProfilingVerbosity(verbosity);
8867 }
8868
8878 {
8879 return mImpl->getProfilingVerbosity();
8880 }
8881
8887 {
8888 mImpl->setAlgorithmSelector(selector);
8889 }
8890
8895 {
8896 return mImpl->getAlgorithmSelector();
8897 }
8898
8910 bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept
8911 {
8912 return mImpl->setCalibrationProfile(profile);
8913 }
8914
8921 {
8922 return mImpl->getCalibrationProfile();
8923 }
8924
8938 {
8939 mImpl->setQuantizationFlags(flags);
8940 }
8941
8950 {
8951 return mImpl->getQuantizationFlags();
8952 }
8953
8962 {
8963 mImpl->clearQuantizationFlag(flag);
8964 }
8965
8974 {
8975 mImpl->setQuantizationFlag(flag);
8976 }
8977
8985 bool getQuantizationFlag(QuantizationFlag flag) const noexcept
8986 {
8987 return mImpl->getQuantizationFlag(flag);
8988 }
8989
9007 bool setTacticSources(TacticSources tacticSources) noexcept
9008 {
9009 return mImpl->setTacticSources(tacticSources);
9010 }
9011
9023 {
9024 return mImpl->getTacticSources();
9025 }
9026
9041 nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept
9042 {
9043 return mImpl->createTimingCache(blob, size);
9044 }
9045
9064 bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept
9065 {
9066 return mImpl->setTimingCache(cache, ignoreMismatch);
9067 }
9068
9075 {
9076 return mImpl->getTimingCache();
9077 }
9078
9106 void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
9107 {
9108 mImpl->setMemoryPoolLimit(pool, poolSize);
9109 }
9110
9125 std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
9126 {
9127 return mImpl->getMemoryPoolLimit(pool);
9128 }
9129
9143 void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
9144 {
9145 mImpl->setPreviewFeature(feature, enable);
9146 }
9147
9157 bool getPreviewFeature(PreviewFeature feature) const noexcept
9158 {
9159 return mImpl->getPreviewFeature(feature);
9160 }
9161
9190 void setBuilderOptimizationLevel(int32_t level) noexcept
9191 {
9192 mImpl->setBuilderOptimizationLevel(level);
9193 }
9194
9203 {
9204 return mImpl->getBuilderOptimizationLevel();
9205 }
9206
9219 void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept
9220 {
9221 mImpl->setHardwareCompatibilityLevel(hardwareCompatibilityLevel);
9222 }
9223
9233 {
9234 return mImpl->getHardwareCompatibilityLevel();
9235 }
9236
9245 void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept
9246 {
9247 mImpl->setPluginsToSerialize(paths, nbPaths);
9248 }
9249
9258 char const* getPluginToSerialize(int32_t index) const noexcept
9259 {
9260 return mImpl->getPluginToSerialize(index);
9261 }
9262
9268 int32_t getNbPluginsToSerialize() const noexcept
9269 {
9270 return mImpl->getNbPluginsToSerialize();
9271 }
9272
9297 void setMaxAuxStreams(int32_t nbStreams) noexcept
9298 {
9299 mImpl->setMaxAuxStreams(nbStreams);
9300 }
9301
9307 int32_t getMaxAuxStreams() const noexcept
9308 {
9309 return mImpl->getMaxAuxStreams();
9310 }
9311
9323 void setProgressMonitor(IProgressMonitor* monitor) noexcept
9324 {
9325 return mImpl->setProgressMonitor(monitor);
9326 }
9327
9334 {
9335 return mImpl->getProgressMonitor();
9336 }
9337
9338protected:
9339 apiv::VBuilderConfig* mImpl;
9340};
9341
9350
9360{
9365
9370 kSTRONGLY_TYPED = 1,
9371};
9372
9378template <>
9379constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>() noexcept
9380{
9381 return 2;
9382}
9383
9391class IBuilder : public INoCopy
9392{
9393public:
9394 virtual ~IBuilder() noexcept = default;
9395
9399 bool platformHasFastFp16() const noexcept
9400 {
9401 return mImpl->platformHasFastFp16();
9402 }
9403
9407 bool platformHasFastInt8() const noexcept
9408 {
9409 return mImpl->platformHasFastInt8();
9410 }
9411
9419 int32_t getMaxDLABatchSize() const noexcept
9420 {
9421 return mImpl->getMaxDLABatchSize();
9422 }
9423
9427 int32_t getNbDLACores() const noexcept
9428 {
9429 return mImpl->getNbDLACores();
9430 }
9431
9444 void setGpuAllocator(IGpuAllocator* allocator) noexcept
9445 {
9446 mImpl->setGpuAllocator(allocator);
9447 }
9448
9455 {
9456 return mImpl->createBuilderConfig();
9457 }
9458
9477 {
9478 return mImpl->createNetworkV2(flags);
9479 }
9480
9492 {
9493 return mImpl->createOptimizationProfile();
9494 }
9495
9510 void setErrorRecorder(IErrorRecorder* recorder) noexcept
9511 {
9512 mImpl->setErrorRecorder(recorder);
9513 }
9514
9526 {
9527 return mImpl->getErrorRecorder();
9528 }
9529
9533 void reset() noexcept
9534 {
9535 mImpl->reset();
9536 }
9537
9541 bool platformHasTf32() const noexcept
9542 {
9543 return mImpl->platformHasTf32();
9544 }
9545
9561 {
9562 return mImpl->buildSerializedNetwork(network, config);
9563 }
9564
9582 bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
9583 {
9584 return mImpl->isNetworkSupported(network, config);
9585 }
9586
9592 ILogger* getLogger() const noexcept
9593 {
9594 return mImpl->getLogger();
9595 }
9596
9608 bool setMaxThreads(int32_t maxThreads) noexcept
9609 {
9610 return mImpl->setMaxThreads(maxThreads);
9611 }
9612
9622 int32_t getMaxThreads() const noexcept
9623 {
9624 return mImpl->getMaxThreads();
9625 }
9626
9633 {
9634 return mImpl->getPluginRegistry();
9635 }
9636
9637protected:
9638 apiv::VBuilder* mImpl;
9639};
9640
9641} // namespace nvinfer1
9642
9647extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept;
9648
9649namespace nvinfer1
9650{
9651namespace
9652{
9653
9661inline IBuilder* createInferBuilder(ILogger& logger) noexcept
9662{
9663 return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
9664}
9665
9666} // namespace
9667
9681 nvinfer1::EngineCapability capability) noexcept;
9682
9683namespace safe
9684{
9686class IPluginRegistry;
9687} // namespace safe
9688
9697 nvinfer1::EngineCapability capability) noexcept;
9698
9699} // namespace nvinfer1
9700
9701#endif // NV_INFER_H
#define TENSORRTAPI
Definition: NvInferRuntimeBase.h:59
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeBase.h:87
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:45
#define TRT_DEPRECATED_ENUM
Definition: NvInferRuntimeBase.h:46
Definition: NvInferRuntimeBase.h:195
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeBase.h:198
An Activation layer in a network definition.
Definition: NvInfer.h:1337
void setBeta(float beta) noexcept
Set the beta parameter (must be finite).
Definition: NvInfer.h:1385
void setActivationType(ActivationType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1346
ActivationType getActivationType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1356
float getAlpha() const noexcept
Get the alpha parameter.
Definition: NvInfer.h:1394
virtual ~IActivationLayer() noexcept=default
float getBeta() const noexcept
Get the beta parameter.
Definition: NvInfer.h:1403
void setAlpha(float alpha) noexcept
Set the alpha parameter (must be finite).
Definition: NvInfer.h:1371
Describes the context and requirements, that could be fulfilled by one or more instances of IAlgorith...
Definition: NvInfer.h:7901
int32_t getNbOutputs() const noexcept
Return number of outputs of the algorithm.
Definition: NvInfer.h:7936
int32_t getNbInputs() const noexcept
Return number of inputs of the algorithm.
Definition: NvInfer.h:7928
char const * getName() const noexcept
Return name of the algorithm node.
Definition: NvInfer.h:7908
virtual ~IAlgorithmContext() noexcept=default
Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for input or output tensor.
Definition: NvInfer.h:7920
Describes a variation of execution of a layer. An algorithm is represented by IAlgorithmVariant and t...
Definition: NvInfer.h:7958
std::size_t getWorkspaceSize() const noexcept
The size of the GPU temporary memory in bytes which the algorithm uses at execution time.
Definition: NvInfer.h:7979
float getTimingMSec() const noexcept
The time in milliseconds to execute the algorithm.
Definition: NvInfer.h:7971
IAlgorithmIOInfo const * getAlgorithmIOInfoByIndex(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:7993
virtual ~IAlgorithm() noexcept=default
IAlgorithmVariant const & getAlgorithmVariant() const noexcept
Returns the algorithm variant.
Definition: NvInfer.h:7963
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:7808
virtual ~IAlgorithmIOInfo() noexcept=default
int64_t getVectorizedDim() const noexcept
Return the index of the vectorized dimension or -1 for non-vectorized formats.
Definition: NvInfer.h:7836
Dims getStrides() const noexcept
Return strides of the input/output tensor of algorithm. For vectorized formats, strides are given in ...
Definition: NvInfer.h:7826
DataType getDataType() const noexcept
Return DataType of the input/output of algorithm.
Definition: NvInfer.h:7815
int64_t getComponentsPerElement() const noexcept
Return the number of components per element. This is always 1 for non-vectorized formats.
Definition: NvInfer.h:7847
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:7869
virtual ~IAlgorithmVariant() noexcept=default
int64_t getTactic() const noexcept
Return tactic of the algorithm.
Definition: NvInfer.h:7882
int64_t getImplementation() const noexcept
Return implementation of the algorithm.
Definition: NvInfer.h:7874
An assertion layer in a network.
Definition: NvInfer.h:4839
void setMessage(char const *message) noexcept
Set the message to print if the assertion fails.
Definition: NvInfer.h:4849
char const * getMessage() const noexcept
Return the assertion message.
Definition: NvInfer.h:4859
virtual ~IAssertionLayer() noexcept=default
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:8550
IOptimizationProfile const * getCalibrationProfile() noexcept
Get the current calibration profile.
Definition: NvInfer.h:8920
void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
Set the memory size for the memory pool.
Definition: NvInfer.h:9106
void setQuantizationFlag(QuantizationFlag flag) noexcept
Set a single quantization flag.
Definition: NvInfer.h:8973
nvinfer1::ITimingCache * createTimingCache(void const *blob, std::size_t size) const noexcept
Create timing cache.
Definition: NvInfer.h:9041
void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
Enable or disable a specific preview feature.
Definition: NvInfer.h:9143
void setInt8Calibrator(IInt8Calibrator *calibrator) noexcept
Set Int8 Calibration interface.
Definition: NvInfer.h:8609
bool getPreviewFeature(PreviewFeature feature) const noexcept
Get status of preview feature.
Definition: NvInfer.h:9157
void clearQuantizationFlag(QuantizationFlag flag) noexcept
clear a quantization flag.
Definition: NvInfer.h:8961
int32_t getBuilderOptimizationLevel() noexcept
Get builder optimization level.
Definition: NvInfer.h:9202
bool setTacticSources(TacticSources tacticSources) noexcept
Set tactic sources.
Definition: NvInfer.h:9007
void setPluginsToSerialize(char const *const *paths, int32_t nbPaths) noexcept
Set the plugin libraries to be serialized with version-compatible engines.
Definition: NvInfer.h:9245
bool getQuantizationFlag(QuantizationFlag flag) const noexcept
Returns true if the quantization flag is set.
Definition: NvInfer.h:8985
std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
Get the memory size limit of the memory pool.
Definition: NvInfer.h:9125
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInfer.h:8767
int32_t getNbPluginsToSerialize() const noexcept
Get the number of plugin library paths to be serialized with version-compatible engines.
Definition: NvInfer.h:9268
void setDeviceType(ILayer const *layer, DeviceType deviceType) noexcept
Set the device that this layer must execute on.
Definition: NvInfer.h:8699
void setEngineCapability(EngineCapability capability) noexcept
Configure the builder to target specified EngineCapability flow.
Definition: NvInfer.h:8587
int32_t getMaxAuxStreams() const noexcept
Get the maximum number of auxiliary streams that TRT is allowed to use.
Definition: NvInfer.h:9307
bool getFlag(BuilderFlag builderFlag) const noexcept
Returns true if the build mode flag is set.
Definition: NvInfer.h:8682
void setQuantizationFlags(QuantizationFlags flags) noexcept
Set the quantization flags.
Definition: NvInfer.h:8937
bool setCalibrationProfile(IOptimizationProfile const *profile) noexcept
Add a calibration profile.
Definition: NvInfer.h:8910
void setProgressMonitor(IProgressMonitor *monitor) noexcept
Sets the progress monitor for building a network.
Definition: NvInfer.h:9323
void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:8864
IAlgorithmSelector * getAlgorithmSelector() const noexcept
Get Algorithm Selector.
Definition: NvInfer.h:8894
int32_t getNbOptimizationProfiles() const noexcept
Get number of optimization profiles.
Definition: NvInfer.h:8852
QuantizationFlags getQuantizationFlags() const noexcept
Get the quantization flags.
Definition: NvInfer.h:8949
nvinfer1::ITimingCache const * getTimingCache() const noexcept
Get the pointer to the timing cache from current IBuilderConfig.
Definition: NvInfer.h:9074
void reset() noexcept
Resets the builder configuration to defaults.
Definition: NvInfer.h:8798
bool setTimingCache(ITimingCache const &cache, bool ignoreMismatch) noexcept
Attach a timing cache to IBuilderConfig.
Definition: NvInfer.h:9064
char const * getPluginToSerialize(int32_t index) const noexcept
Get the plugin library path to be serialized with version-compatible engines.
Definition: NvInfer.h:9258
EngineCapability getEngineCapability() const noexcept
Query EngineCapability flow configured for the builder.
Definition: NvInfer.h:8599
void setAlgorithmSelector(IAlgorithmSelector *selector) noexcept
Set Algorithm Selector.
Definition: NvInfer.h:8886
DeviceType getDefaultDeviceType() const noexcept
Get the default DeviceType which was set by setDefaultDeviceType.
Definition: NvInfer.h:8788
BuilderFlags getFlags() const noexcept
Get the build mode flags for this builder config. Defaults to 0.
Definition: NvInfer.h:8646
void setFlags(BuilderFlags builderFlags) noexcept
Set the build mode flags to turn on builder options for this network.
Definition: NvInfer.h:8634
TacticSources getTacticSources() const noexcept
Get tactic sources.
Definition: NvInfer.h:9022
void resetDeviceType(ILayer const *layer) noexcept
reset the DeviceType for this layer
Definition: NvInfer.h:8731
void setDLACore(int32_t dlaCore) noexcept
Sets the DLA core used by the network. Defaults to -1.
Definition: NvInfer.h:8757
HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept
Get the hardware compatibility level.
Definition: NvInfer.h:9232
void clearFlag(BuilderFlag builderFlag) noexcept
clear a single build mode flag.
Definition: NvInfer.h:8658
int32_t addOptimizationProfile(IOptimizationProfile const *profile) noexcept
Add an optimization profile.
Definition: NvInfer.h:8839
IProgressMonitor * getProgressMonitor() const noexcept
Definition: NvInfer.h:9333
apiv::VBuilderConfig * mImpl
Definition: NvInfer.h:9339
int32_t getAvgTimingIterations() const noexcept
Query the number of averaging iterations.
Definition: NvInfer.h:8574
void setDefaultDeviceType(DeviceType deviceType) noexcept
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
Definition: NvInfer.h:8778
void setFlag(BuilderFlag builderFlag) noexcept
Set a single build mode flag.
Definition: NvInfer.h:8670
virtual ~IBuilderConfig() noexcept=default
DeviceType getDeviceType(ILayer const *layer) const noexcept
Get the device that this layer executes on.
Definition: NvInfer.h:8709
bool canRunOnDLA(ILayer const *layer) const noexcept
Checks if a layer can run on DLA.
Definition: NvInfer.h:8741
cudaStream_t getProfileStream() const noexcept
Get the cuda stream that is used to profile this network.
Definition: NvInfer.h:8822
void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept
Set the hardware compatibility level.
Definition: NvInfer.h:9219
IInt8Calibrator * getInt8Calibrator() const noexcept
Get Int8 Calibration interface.
Definition: NvInfer.h:8617
void setMaxAuxStreams(int32_t nbStreams) noexcept
Set the maximum number of auxiliary streams that TRT is allowed to use.
Definition: NvInfer.h:9297
ProfilingVerbosity getProfilingVerbosity() const noexcept
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:8877
bool isDeviceTypeSet(ILayer const *layer) const noexcept
whether the DeviceType has been explicitly set for this layer
Definition: NvInfer.h:8721
void setBuilderOptimizationLevel(int32_t level) noexcept
Set builder optimization level.
Definition: NvInfer.h:9190
void setProfileStream(const cudaStream_t stream) noexcept
Set the cuda stream that is used to profile this network.
Definition: NvInfer.h:8810
Builds an engine from a network definition.
Definition: NvInfer.h:9392
int32_t getMaxDLABatchSize() const noexcept
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Definition: NvInfer.h:9419
int32_t getNbDLACores() const noexcept
Return the number of DLA engines available to this builder.
Definition: NvInfer.h:9427
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:9525
apiv::VBuilder * mImpl
Definition: NvInfer.h:9638
ILogger * getLogger() const noexcept
get the logger with which the builder was created
Definition: NvInfer.h:9592
bool isNetworkSupported(INetworkDefinition const &network, IBuilderConfig const &config) const noexcept
Checks that a network is within the scope of the IBuilderConfig settings.
Definition: NvInfer.h:9582
bool platformHasTf32() const noexcept
Determine whether the platform has TF32 support.
Definition: NvInfer.h:9541
int32_t getMaxThreads() const noexcept
get the maximum number of threads that can be used by the builder.
Definition: NvInfer.h:9622
IPluginRegistry & getPluginRegistry() noexcept
get the local plugin registry that can be used by the builder.
Definition: NvInfer.h:9632
nvinfer1::IOptimizationProfile * createOptimizationProfile() noexcept
Create a new optimization profile.
Definition: NvInfer.h:9491
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInfer.h:9444
nvinfer1::INetworkDefinition * createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept
Create a network definition object.
Definition: NvInfer.h:9476
nvinfer1::IBuilderConfig * createBuilderConfig() noexcept
Create a builder configuration object.
Definition: NvInfer.h:9454
void reset() noexcept
Resets the builder state to default values.
Definition: NvInfer.h:9533
bool setMaxThreads(int32_t maxThreads) noexcept
Set the maximum number of threads.
Definition: NvInfer.h:9608
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:9510
bool platformHasFastInt8() const noexcept
Determine whether the platform has fast native int8.
Definition: NvInfer.h:9407
nvinfer1::IHostMemory * buildSerializedNetwork(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds and serializes a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:9560
virtual ~IBuilder() noexcept=default
A cast layer in a network.
Definition: NvInfer.h:3740
virtual ~ICastLayer() noexcept=default
apiv::VCastLayer * mImpl
Definition: NvInfer.h:3766
DataType getToType() const noexcept
Return cast layer output type.
Definition: NvInfer.h:3760
void setToType(DataType toType) noexcept
Set cast layer output type.
Definition: NvInfer.h:3749
A concatenation layer in a network definition.
Definition: NvInfer.h:2047
void setAxis(int32_t axis) noexcept
Set the axis along which concatenation occurs.
Definition: NvInfer.h:2060
int32_t getAxis() const noexcept
Get the axis along which concatenation occurs.
Definition: NvInfer.h:2070
virtual ~IConcatenationLayer() noexcept=default
This layer represents a condition input to an IIfConditional.
Definition: NvInfer.h:4391
virtual ~IConditionLayer() noexcept=default
Layer that represents a constant value.
Definition: NvInfer.h:3779
void setWeights(Weights weights) noexcept
Set the weights for the layer.
Definition: NvInfer.h:3789
Weights getWeights() const noexcept
Get the weights for the layer.
Definition: NvInfer.h:3799
void setDimensions(Dims const &dimensions) noexcept
Set the dimensions for the layer.
Definition: NvInfer.h:3811
apiv::VConstantLayer * mImpl
Definition: NvInfer.h:3829
virtual ~IConstantLayer() noexcept=default
Dims getDimensions() const noexcept
Get the dimensions for the layer.
Definition: NvInfer.h:3823
A convolution layer in a network definition.
Definition: NvInfer.h:1017
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1142
Weights getBiasWeights() const noexcept
Get the bias weights for the convolution.
Definition: NvInfer.h:1115
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1183
void setDilationNd(Dims const &dilation) noexcept
Set the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1287
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the convolution.
Definition: NvInfer.h:1273
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the convolution.
Definition: NvInfer.h:1243
Weights getKernelWeights() const noexcept
Get the kernel weights of the convolution.
Definition: NvInfer.h:1090
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride of the convolution.
Definition: NvInfer.h:1233
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1297
int64_t getNbOutputMaps() const noexcept
Get the number of output maps for the convolution.
Definition: NvInfer.h:1036
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the convolution.
Definition: NvInfer.h:1080
Dims getPostPadding() const noexcept
Get the post-padding.
Definition: NvInfer.h:1169
int64_t getNbGroups() const noexcept
Get the number of groups of the convolution.
Definition: NvInfer.h:1066
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1195
virtual ~IConvolutionLayer() noexcept=default
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups for a convolution.
Definition: NvInfer.h:1056
void setNbOutputMaps(int64_t nbOutputMaps) noexcept
Set the number of output maps for the convolution.
Definition: NvInfer.h:1026
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the convolution.
Definition: NvInfer.h:1105
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1218
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding of the convolution.
Definition: NvInfer.h:1261
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding of the convolution.
Definition: NvInfer.h:1132
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding of the convolution.
Definition: NvInfer.h:1159
void setKernelSizeNd(Dims const &kernelSize) noexcept
Set the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1208
A deconvolution layer in a network definition.
Definition: NvInfer.h:2088
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the deconvolution.
Definition: NvInfer.h:2176
int64_t getNbGroups() const noexcept
Get the number of groups for a deconvolution.
Definition: NvInfer.h:2137
Weights getKernelWeights() const noexcept
Get the kernel weights for the deconvolution.
Definition: NvInfer.h:2161
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding of the deconvolution.
Definition: NvInfer.h:2204
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2320
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2386
Weights getBiasWeights() const noexcept
Get the bias weights for the deconvolution.
Definition: NvInfer.h:2186
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the deconvolution.
Definition: NvInfer.h:2151
int64_t getNbOutputMaps() const noexcept
Get the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2107
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2310
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:2242
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2293
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding of the deconvolution.
Definition: NvInfer.h:2232
void setKernelSizeNd(Dims const &kernelSize) noexcept
Set the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2283
virtual ~IDeconvolutionLayer() noexcept=default
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2338
void setNbOutputMaps(int64_t nbOutputMaps) noexcept
Set the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2097
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2350
void setDilationNd(Dims const &dilation) noexcept
Set the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2376
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:2256
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups for a deconvolution.
Definition: NvInfer.h:2127
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:2214
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:2268
A Dequantize layer in a network definition.
Definition: NvInfer.h:5425
void setToType(DataType toType) noexcept
Set the Dequantize layer output type.
Definition: NvInfer.h:5462
virtual ~IDequantizeLayer() noexcept=default
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5435
DataType getToType() const noexcept
Return the Dequantize layer output type.
Definition: NvInfer.h:5474
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5446
An Einsum layer in a network.
Definition: NvInfer.h:5521
bool setEquation(char const *equation) noexcept
Set the equation. The equation is a comma-separated list of subscript labels, where each label refers...
Definition: NvInfer.h:5532
virtual ~IEinsumLayer() noexcept=default
char const * getEquation() const noexcept
Return the equation.
Definition: NvInfer.h:5542
A elementwise layer in a network definition.
Definition: NvInfer.h:2462
virtual ~IElementWiseLayer() noexcept=default
apiv::VElementWiseLayer * mImpl
Definition: NvInfer.h:2491
ElementWiseOperation getOperation() const noexcept
Get the binary operation for the layer.
Definition: NvInfer.h:2485
void setOperation(ElementWiseOperation op) noexcept
Set the binary operation for the layer.
Definition: NvInfer.h:2473
Generate a tensor according to a specified mode.
Definition: NvInfer.h:4950
bool isAlphaBetaInt64() const noexcept
Return true if alpha/beta have type int64, false if they have type double.
Definition: NvInfer.h:5182
FillOperation getOperation() const noexcept
Get the fill operation for the layer.
Definition: NvInfer.h:4996
void setOperation(FillOperation op) noexcept
Set the fill operation for the layer.
Definition: NvInfer.h:4986
DataType getToType() const noexcept
Get the fill layer output type.
Definition: NvInfer.h:5211
void setAlphaInt64(int64_t alpha) noexcept
Set the alpha parameter with int64 datatype.
Definition: NvInfer.h:5125
void setBetaInt64(int64_t beta) noexcept
Set the beta parameter with int64 datatype.
Definition: NvInfer.h:5159
void setBeta(double beta) noexcept
Set the beta parameter.
Definition: NvInfer.h:5049
int64_t getAlphaInt64() const noexcept
Get the value of alpha parameter with int64 datatype.
Definition: NvInfer.h:5140
int64_t getBetaInt64() const noexcept
Get the value of beta parameter with int64 datatype.
Definition: NvInfer.h:5174
double getAlpha() const noexcept
Get the value of alpha parameter.
Definition: NvInfer.h:5030
void setDimensions(Dims const &dimensions) noexcept
Set the output tensor's dimensions.
Definition: NvInfer.h:4961
void setAlpha(double alpha) noexcept
Set the alpha parameter.
Definition: NvInfer.h:5015
void setToType(DataType toType) noexcept
Set the fill layer output type.
Definition: NvInfer.h:5199
Dims getDimensions() const noexcept
Get the output tensor's dimensions.
Definition: NvInfer.h:4976
double getBeta() const noexcept
Get the value of beta parameter.
Definition: NvInfer.h:5064
virtual ~IFillLayer() noexcept=default
A Gather layer in a network definition. Supports several kinds of gathering.
Definition: NvInfer.h:2596
void setGatherAxis(int32_t axis) noexcept
Set the axis used by GatherMode::kELEMENTS and GatherMode::kDEFAULT The axis must be less than the nu...
Definition: NvInfer.h:2607
void setNbElementWiseDims(int32_t elementWiseDims) noexcept
Set the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:2642
apiv::VGatherLayer * mImpl
Definition: NvInfer.h:2678
int32_t getNbElementWiseDims() const noexcept
Get the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:2652
void setMode(GatherMode mode) noexcept
Set the gather mode.
Definition: NvInfer.h:2662
int32_t getGatherAxis() const noexcept
Get the axis to gather on.
Definition: NvInfer.h:2619
GatherMode getMode() const noexcept
Get the gather mode.
Definition: NvInfer.h:2672
virtual ~IGatherLayer() noexcept=default
A GridSample layer in a network definition.
Definition: NvInfer.h:5741
void setInterpolationMode(InterpolationMode mode) noexcept
Set the grid sample interpolation mode.
Definition: NvInfer.h:5748
bool setSampleMode(SampleMode mode) noexcept
Set the sample mode.
Definition: NvInfer.h:5794
void setAlignCorners(bool alignCorners) noexcept
Set the align corners mode.
Definition: NvInfer.h:5770
apiv::VGridSampleLayer * mImpl
Definition: NvInfer.h:5812
SampleMode getSampleMode() const noexcept
Get the sample mode.
Definition: NvInfer.h:5806
InterpolationMode getInterpolationMode() const noexcept
Get the grid sample interpolation mode.
Definition: NvInfer.h:5760
bool getAlignCorners() const noexcept
Get the align corners mode.
Definition: NvInfer.h:5782
virtual ~IGridSampleLayer() noexcept=default
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:139
A layer that represents the identity function.
Definition: NvInfer.h:3727
apiv::VIdentityLayer * mImpl
Definition: NvInfer.h:3729
virtual ~IIdentityLayer() noexcept=default
This is a base class for Conditional boundary layers.
Definition: NvInfer.h:4370
IIfConditional * getConditional() const noexcept
Get a pointer to the IIfConditional associated with this boundary layer.
Definition: NvInfer.h:4375
virtual ~IIfConditionalBoundaryLayer() noexcept=default
Helper for constructing conditionally-executed subgraphs.
Definition: NvInfer.h:4450
IIfConditionalInputLayer * addInput(ITensor &input) noexcept
Add an If-conditional input.
Definition: NvInfer.h:4489
char const * getName() const noexcept
Return the name of the conditional.
Definition: NvInfer.h:4514
virtual ~IIfConditional() noexcept=default
IConditionLayer * setCondition(ITensor &condition) noexcept
Set the condition tensor for this If-Conditional construct.
Definition: NvInfer.h:4461
IIfConditionalOutputLayer * addOutput(ITensor &trueSubgraphOutput, ITensor &falseSubgraphOutput) noexcept
Add an If-conditional output.
Definition: NvInfer.h:4477
void setName(char const *name) noexcept
Set the name of the conditional.
Definition: NvInfer.h:4504
This layer represents an input to an IIfConditional.
Definition: NvInfer.h:4419
virtual ~IIfConditionalInputLayer() noexcept=default
This layer represents an output of an IIfConditional.
Definition: NvInfer.h:4406
virtual ~IIfConditionalOutputLayer() noexcept=default
Application-implemented interface for calibration.
Definition: NvInfer.h:7543
virtual TRT_DEPRECATED int32_t getBatchSize() const noexcept=0
Get the batch size used for calibration batches.
A layer to do iterations.
Definition: NvInfer.h:4666
virtual ~IIteratorLayer() noexcept=default
void setReverse(bool reverse) noexcept
Set iteration order to be reverse.
Definition: NvInfer.h:4693
bool getReverse() const noexcept
Check if the iteration order is reverse.
Definition: NvInfer.h:4703
int32_t getAxis() const noexcept
Get axis being iterated over.
Definition: NvInfer.h:4679
void setAxis(int32_t axis) noexcept
Set axis to iterate over.
Definition: NvInfer.h:4671
A LRN layer in a network definition.
Definition: NvInfer.h:1702
int64_t getWindowSize() const noexcept
Get the LRN window size.
Definition: NvInfer.h:1723
float getAlpha() const noexcept
Get the LRN alpha value.
Definition: NvInfer.h:1745
void setWindowSize(int64_t windowSize) noexcept
Set the LRN window size.
Definition: NvInfer.h:1713
void setK(float k) noexcept
Set the LRN K value.
Definition: NvInfer.h:1779
void setAlpha(float alpha) noexcept
Set the LRN alpha value.
Definition: NvInfer.h:1735
void setBeta(float beta) noexcept
Set the LRN beta value.
Definition: NvInfer.h:1757
virtual ~ILRNLayer() noexcept=default
float getBeta() const noexcept
Get the LRN beta value.
Definition: NvInfer.h:1767
float getK() const noexcept
Get the LRN K value.
Definition: NvInfer.h:1789
Base class for all layer classes in a network definition.
Definition: NvInfer.h:544
bool precisionIsSet() const noexcept
whether the computational precision has been set for this layer
Definition: NvInfer.h:687
void setMetadata(char const *metadata) noexcept
Set the metadata for this layer.
Definition: NvInfer.h:803
void setPrecision(DataType dataType) noexcept
Set the preferred or required computational precision of this layer in a weakly-typed network.
Definition: NvInfer.h:663
void setName(char const *name) noexcept
Set the name of a layer.
Definition: NvInfer.h:565
void resetPrecision() noexcept
reset the computational precision for this layer
Definition: NvInfer.h:697
int32_t getNbInputs() const noexcept
Get the number of inputs of a layer.
Definition: NvInfer.h:583
char const * getMetadata() const noexcept
Get the metadata of the layer.
Definition: NvInfer.h:816
DataType getOutputType(int32_t index) const noexcept
get the output type of this layer
Definition: NvInfer.h:759
DataType getPrecision() const noexcept
get the computational precision of this layer
Definition: NvInfer.h:675
char const * getName() const noexcept
Return the name of a layer.
Definition: NvInfer.h:575
int32_t getNbOutputs() const noexcept
Get the number of outputs of a layer.
Definition: NvInfer.h:604
bool outputTypeIsSet(int32_t index) const noexcept
whether the output type has been set for this layer
Definition: NvInfer.h:773
ITensor * getOutput(int32_t index) const noexcept
Get the layer output corresponding to the given index.
Definition: NvInfer.h:614
void setInput(int32_t index, ITensor &tensor) noexcept
Replace an input of this layer with a specific tensor.
Definition: NvInfer.h:631
void resetOutputType(int32_t index) noexcept
reset the output type for this layer
Definition: NvInfer.h:785
ITensor * getInput(int32_t index) const noexcept
Get the layer input corresponding to the given index.
Definition: NvInfer.h:596
void setOutputType(int32_t index, DataType dataType) noexcept
Set the output type of this layer in a weakly-typed network.
Definition: NvInfer.h:744
LayerType getType() const noexcept
Return the type of a layer.
Definition: NvInfer.h:551
virtual ~ILayer() noexcept=default
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeBase.h:676
This is a base class for Loop boundary layers.
Definition: NvInfer.h:4347
virtual ~ILoopBoundaryLayer() noexcept=default
ILoop * getLoop() const noexcept
Get a pointer to ILoop associated with this boundary layer.
Definition: NvInfer.h:4352
Helper for creating a recurrent subgraph.
Definition: NvInfer.h:4719
void setName(char const *name) noexcept
Set the name of the loop.
Definition: NvInfer.h:4789
ITripLimitLayer * addTripLimit(ITensor &tensor, TripLimit limit) noexcept
Add a trip-count limiter, based on the given tensor.
Definition: NvInfer.h:4748
IIteratorLayer * addIterator(ITensor &tensor, int32_t axis=0, bool reverse=false) noexcept
Return layer that subscripts tensor by loop iteration.
Definition: NvInfer.h:4761
ILoopOutputLayer * addLoopOutput(ITensor &tensor, LoopOutput outputKind, int32_t axis=0) noexcept
Make an output for this loop, based on the given tensor.
Definition: NvInfer.h:4774
virtual ~ILoop() noexcept=default
char const * getName() const noexcept
Return the name of the loop.
Definition: NvInfer.h:4799
IRecurrenceLayer * addRecurrence(ITensor &initialValue) noexcept
Create a recurrence layer for this loop with initialValue as its first input.
Definition: NvInfer.h:4727
An ILoopOutputLayer is the sole way to get output from a loop.
Definition: NvInfer.h:4577
virtual ~ILoopOutputLayer() noexcept=default
int32_t getAxis() const noexcept
Get axis being concatenated over.
Definition: NvInfer.h:4607
LoopOutput getLoopOutput() const noexcept
Get which kind a loop output has.
Definition: NvInfer.h:4582
void setAxis(int32_t axis) noexcept
Set where to insert the contenation axis. Ignored if getLoopOutput() is kLAST_VALUE.
Definition: NvInfer.h:4599
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:3618
apiv::VMatrixMultiplyLayer * mImpl
Definition: NvInfer.h:3646
virtual ~IMatrixMultiplyLayer() noexcept=default
MatrixOperation getOperation(int32_t index) const noexcept
Get the operation for an input tensor.
Definition: NvInfer.h:3640
void setOperation(int32_t index, MatrixOperation op) noexcept
Set the operation for an input tensor.
Definition: NvInfer.h:3628
A non-maximum suppression layer in a network definition.
Definition: NvInfer.h:5889
virtual ~INMSLayer() noexcept=default
void setTopKBoxLimit(int32_t limit) noexcept
Set the TopK box limit parameter for the layer.
Definition: NvInfer.h:5926
void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept
Set the bounding box format parameter for the layer.
Definition: NvInfer.h:5900
BoundingBoxFormat getBoundingBoxFormat() const noexcept
Get the bounding box format parameter for the layer.
Definition: NvInfer.h:5912
apiv::VNMSLayer * mImpl
Definition: NvInfer.h:5962
int32_t getTopKBoxLimit() const noexcept
Get the TopK box limit parameter for the layer.
Definition: NvInfer.h:5936
A network definition for input to the builder.
Definition: NvInfer.h:6184
IPluginV2Layer * addPluginV2(ITensor *const *inputs, int32_t nbInputs, IPluginV2 &plugin) noexcept
Add a plugin layer to the network using the IPluginV2 interface.
Definition: NvInfer.h:6786
IConcatenationLayer * addConcatenation(ITensor *const *inputs, int32_t nbInputs) noexcept
Add a concatenation layer to the network.
Definition: NvInfer.h:6377
IShuffleLayer * addShuffle(ITensor &input) noexcept
Add a shuffle layer to the network.
Definition: NvInfer.h:6440
INormalizationLayer * addNormalization(ITensor &input, ITensor &scale, ITensor &bias, uint32_t axesMask) noexcept
Add a normalization layer to the network.
Definition: NvInfer.h:7487
void setName(char const *name) noexcept
Sets the name of the network.
Definition: NvInfer.h:6848
bool markDebug(ITensor &tensor) noexcept
Mark a tensor as a debug tensor.
Definition: NvInfer.h:6256
ILRNLayer * addLRN(ITensor &input, int64_t window, float alpha, float beta, float k) noexcept
Add a LRN layer to the network.
Definition: NvInfer.h:6321
ITopKLayer * addTopK(ITensor &input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
Add a TopK layer to the network.
Definition: NvInfer.h:6600
IAssertionLayer * addAssertion(ITensor &condition, char const *message) noexcept
Add an assertion layer to the network.
Definition: NvInfer.h:7164
IConvolutionLayer * addConvolutionNd(ITensor &input, int64_t nbOutputMaps, Dims const &kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension convolution layer to the network.
Definition: NvInfer.h:6983
ICastLayer * addCast(ITensor &input, DataType toType) noexcept
Add a cast layer.
Definition: NvInfer.h:6740
IScaleLayer * addScaleNd(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
Add a multi-dimension scale layer to the network.
Definition: NvInfer.h:7062
char const * getName() const noexcept
Returns the name associated with the network.
Definition: NvInfer.h:6862
IParametricReLULayer * addParametricReLU(ITensor &input, ITensor &slope) noexcept
Add a parametric ReLU layer to the network.
Definition: NvInfer.h:6961
ITensor * getOutput(int32_t index) const noexcept
Get the output tensor specified by the given index.
Definition: NvInfer.h:6541
ITensor * getInput(int32_t index) const noexcept
Get the input tensor specified by the given index.
Definition: NvInfer.h:6511
IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale, DataType outputType) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7331
bool unmarkOutputForShapes(ITensor &tensor) noexcept
Undo markOutputForShapes.
Definition: NvInfer.h:6943
IFillLayer * addFill(Dims const &dimensions, FillOperation op, DataType outputType) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7215
ILoop * addLoop() noexcept
Add a loop to the network.
Definition: NvInfer.h:7093
IActivationLayer * addActivation(ITensor &input, ActivationType type) noexcept
Add an activation layer to the network.
Definition: NvInfer.h:6302
TRT_DEPRECATED IFillLayer * addFill(Dims const &dimensions, FillOperation op) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7189
ISliceLayer * addSlice(ITensor &input, Dims const &start, Dims const &size, Dims const &stride) noexcept
Add a slice layer to the network.
Definition: NvInfer.h:6824
virtual ~INetworkDefinition() noexcept=default
TRT_DEPRECATED IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7372
virtual IBuilder & getBuilder() const noexcept
Return the builder from which this INetworkDefinition was created.
Definition: NvInfer.h:7498
INMSLayer * addNMS(ITensor &boxes, ITensor &scores, ITensor &maxOutputBoxesPerClass) noexcept
Add a non-maximum suppression layer to the network.
Definition: NvInfer.h:7444
ILayer * getLayer(int32_t index) const noexcept
Get the layer specified by the given index.
Definition: NvInfer.h:6483
bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept
Returns true if the network definition creation flag is set.
Definition: NvInfer.h:6914
IIfConditional * addIfConditional() noexcept
Add an if-then-else to the network.
Definition: NvInfer.h:7108
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:7289
IReverseSequenceLayer * addReverseSequence(ITensor &input, ITensor &sequenceLens) noexcept
Add a ReverseSequence layer to the network.
Definition: NvInfer.h:7461
int32_t getNbInputs() const noexcept
Get the number of inputs in the network.
Definition: NvInfer.h:6495
NetworkDefinitionCreationFlags getFlags() const noexcept
Get the network definition creation flags for this network definition object. Defaults to 0.
Definition: NvInfer.h:6902
IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale, DataType outputType) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7393
IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
Add a reduce layer to the network.
Definition: NvInfer.h:6567
IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation) noexcept
Add a unary layer to the network.
Definition: NvInfer.h:6426
IGridSampleLayer * addGridSample(ITensor &input, ITensor &grid) noexcept
Add a GridSample layer to the network.
Definition: NvInfer.h:7426
void removeTensor(ITensor &tensor) noexcept
remove a tensor from the network definition.
Definition: NvInfer.h:6755
ISelectLayer * addSelect(ITensor &condition, ITensor &thenInput, ITensor &elseInput) noexcept
Add a select layer to the network.
Definition: NvInfer.h:7147
IScatterLayer * addScatter(ITensor &data, ITensor &indices, ITensor &updates, ScatterMode mode) noexcept
Add a Scatter layer to the network with specified mode and axis=0.
Definition: NvInfer.h:7351
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInfer.h:6469
TRT_DEPRECATED bool hasImplicitBatchDimension() const noexcept
Query whether the network was created with an implicit batch dimension.
Definition: NvInfer.h:6892
apiv::VNetworkDefinition * mImpl
Definition: NvInfer.h:7504
bool markOutputForShapes(ITensor &tensor) noexcept
Enable tensor's value to be computed by IExecutionContext::getShapeBinding.
Definition: NvInfer.h:6931
IOneHotLayer * addOneHot(ITensor &indices, ITensor &values, ITensor &depth, int32_t axis) noexcept
Add a OneHot layer to the network.
Definition: NvInfer.h:6457
IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
Add a Scale layer to the network.
Definition: NvInfer.h:6347
IPluginV3Layer * addPluginV3(ITensor *const *inputs, int32_t nbInputs, ITensor *const *shapeInputs, int32_t nbShapeInputs, IPluginV3 &plugin) noexcept
Add a plugin layer implementing the IPluginV3 interface to the network.
Definition: NvInfer.h:6804
void unmarkOutput(ITensor &tensor) noexcept
unmark a tensor as a network output.
Definition: NvInfer.h:6767
IIdentityLayer * addIdentity(ITensor &input) noexcept
Add an identity layer.
Definition: NvInfer.h:6725
IGatherLayer * addGatherV2(ITensor &data, ITensor &indices, GatherMode mode) noexcept
Add gather with specified mode, axis=0 and nbElementWiseDims=0.
Definition: NvInfer.h:6632
IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op) noexcept
Add an elementwise layer to the network.
Definition: NvInfer.h:6404
IConstantLayer * addConstant(Dims const &dimensions, Weights weights) noexcept
Add a constant layer to the network.
Definition: NvInfer.h:6711
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:7274
IPoolingLayer * addPoolingNd(ITensor &input, PoolingType type, Dims const &windowSize) noexcept
Add a multi-dimension pooling layer to the network.
Definition: NvInfer.h:7003
IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds) noexcept
Add a RaggedSoftMax layer to the network.
Definition: NvInfer.h:6651
IShapeLayer * addShape(ITensor &input) noexcept
Add a shape layer to the network.
Definition: NvInfer.h:6878
IGatherLayer * addGather(ITensor &data, ITensor &indices, int32_t axis) noexcept
Add gather with mode GatherMode::kDEFAULT and specified axis and nbElementWiseDims=0.
Definition: NvInfer.h:6616
IDeconvolutionLayer * addDeconvolutionNd(ITensor &input, int64_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension deconvolution layer to the network.
Definition: NvInfer.h:7025
IResizeLayer * addResize(ITensor &input) noexcept
Add a resize layer to the network.
Definition: NvInfer.h:7079
IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1) noexcept
Add a MatrixMultiply layer to the network.
Definition: NvInfer.h:6672
ISoftMaxLayer * addSoftMax(ITensor &input) noexcept
Add a SoftMax layer to the network.
Definition: NvInfer.h:6360
bool isDebugTensor(nvinfer1::ITensor const &tensor) const noexcept
Check if a tensor is marked as debug tensor.
Definition: NvInfer.h:6282
bool unmarkDebug(ITensor &tensor) noexcept
Unmark a tensor as a debug tensor.
Definition: NvInfer.h:6272
IEinsumLayer * addEinsum(ITensor *const *inputs, int32_t nbInputs, char const *equation) noexcept
Add an Einsum layer to the network.
Definition: NvInfer.h:7408
void markOutput(ITensor &tensor) noexcept
Mark a tensor as a network output.
Definition: NvInfer.h:6238
IPaddingLayer * addPaddingNd(ITensor &input, Dims const &prePadding, Dims const &postPadding) noexcept
Add a padding layer to the network. Only 2D padding is currently supported.
Definition: NvInfer.h:7231
INonZeroLayer * addNonZero(ITensor &input) noexcept
Add a nonzero layer to the network.
Definition: NvInfer.h:6687
TRT_DEPRECATED IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7310
int32_t getNbOutputs() const noexcept
Get the number of outputs in the network.
Definition: NvInfer.h:6525
bool setWeightsName(Weights weights, char const *name) noexcept
Associate a name with all current uses of the given weights.
Definition: NvInfer.h:7255
Forward declaration of IEngineInspector for use by other interfaces.
Definition: NvInferRuntime.h:48
Definition: NvInfer.h:3672
virtual ~INonZeroLayer() noexcept=default
A normalization layer in a network definition.
Definition: NvInfer.h:6051
float getEpsilon() const noexcept
Get the epsilon value used for the normalization calculation.
Definition: NvInfer.h:6070
uint32_t getAxes() const noexcept
Get the axes value used for the normalization calculation.
Definition: NvInfer.h:6090
virtual ~INormalizationLayer() noexcept=default
void setEpsilon(float eps) noexcept
Set the epsilon value used for the normalization calculation.
Definition: NvInfer.h:6060
DataType getComputePrecision() const noexcept
Get the compute precision of this layer.
Definition: NvInfer.h:6156
apiv::VNormalizationLayer * mImpl
Definition: NvInfer.h:6162
int64_t getNbGroups() const noexcept
Get the number of groups used to split the channels for the normalization calculation.
Definition: NvInfer.h:6121
void setAxes(uint32_t axesMask) noexcept
Set the reduction axes for the normalization calculation.
Definition: NvInfer.h:6080
void setComputePrecision(DataType type) noexcept
Set the compute precision of this layer.
Definition: NvInfer.h:6146
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups used to split the channels in the normalization calculation.
Definition: NvInfer.h:6111
A OneHot layer in a network definition.
Definition: NvInfer.h:5705
apiv::VOneHotLayer * mImpl
Definition: NvInfer.h:5726
void setAxis(int32_t axis) noexcept
Set the axis parameter.
Definition: NvInfer.h:5712
int32_t getAxis() const noexcept
Get the value of the axis parameter.
Definition: NvInfer.h:5720
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:2075
Layer that represents a padding operation.
Definition: NvInfer.h:2952
Dims getPostPaddingNd() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3001
void setPrePaddingNd(Dims const &padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:2963
virtual ~IPaddingLayer() noexcept=default
void setPostPaddingNd(Dims const &padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:2989
Dims getPrePaddingNd() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:2975
apiv::VPaddingLayer * mImpl
Definition: NvInfer.h:3007
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:3843
apiv::VParametricReLULayer * mImpl
Definition: NvInfer.h:3845
virtual ~IParametricReLULayer() noexcept=default
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:54
Plugin class for user-implemented layers.
Definition: NvInferRuntimePlugin.h:126
Layer type for pluginV2.
Definition: NvInfer.h:2692
virtual ~IPluginV2Layer() noexcept=default
apiv::VPluginV2Layer * mImpl
Definition: NvInfer.h:2705
IPluginV2 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:2699
Layer type for V3 plugins.
Definition: NvInfer.h:2719
virtual ~IPluginV3Layer() noexcept=default
IPluginV3 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:2726
apiv::VPluginV3Layer * mImpl
Definition: NvInfer.h:2732
A Pooling layer in a network definition.
Definition: NvInfer.h:1451
PoolingType getPoolingType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1470
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1603
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:1579
bool getAverageCountExcludesPadding() const noexcept
Get whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1523
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1551
void setPoolingType(PoolingType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1460
void setWindowSizeNd(Dims const &windowSize) noexcept
Set the multi-dimension window size for pooling.
Definition: NvInfer.h:1616
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1592
Dims getWindowSizeNd() const noexcept
Get the multi-dimension window size for pooling.
Definition: NvInfer.h:1626
void setAverageCountExcludesPadding(bool exclusive) noexcept
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1512
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding for pooling.
Definition: NvInfer.h:1670
float getBlendFactor() const noexcept
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1498
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride for pooling.
Definition: NvInfer.h:1641
Dims getStrideNd() const noexcept
Get the multi-dimension stride for pooling.
Definition: NvInfer.h:1651
virtual ~IPoolingLayer() noexcept=default
Dims getPaddingNd() const noexcept
Get the multi-dimension padding for pooling.
Definition: NvInfer.h:1682
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding for pooling.
Definition: NvInfer.h:1569
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding for pooling.
Definition: NvInfer.h:1541
void setBlendFactor(float blendFactor) noexcept
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1485
A Quantize layer in a network definition.
Definition: NvInfer.h:5295
void setToType(DataType toType) noexcept
Set the Quantize layer output type.
Definition: NvInfer.h:5332
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5316
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5305
virtual ~IQuantizeLayer() noexcept=default
DataType getToType() const noexcept
Return the Quantize layer output type.
Definition: NvInfer.h:5344
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:3693
apiv::VRaggedSoftMaxLayer * mImpl
Definition: NvInfer.h:3695
virtual ~IRaggedSoftMaxLayer() noexcept=default
A recurrence layer in a network definition.
Definition: NvInfer.h:4530
virtual ~IRecurrenceLayer() noexcept=default
Layer that represents a reduction across a non-bool tensor.
Definition: NvInfer.h:2874
void setKeepDimensions(bool keepDimensions) noexcept
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:2921
void setOperation(ReduceOperation op) noexcept
Set the reduce operation for the layer.
Definition: NvInfer.h:2881
ReduceOperation getOperation() const noexcept
Get the reduce operation for the layer.
Definition: NvInfer.h:2891
virtual ~IReduceLayer() noexcept=default
uint32_t getReduceAxes() const noexcept
Get the axes over which to reduce for the layer.
Definition: NvInfer.h:2911
void setReduceAxes(uint32_t reduceAxes) noexcept
Set the axes over which to reduce.
Definition: NvInfer.h:2901
apiv::VReduceLayer * mImpl
Definition: NvInfer.h:2937
bool getKeepDimensions() const noexcept
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:2931
A resize layer in a network definition.
Definition: NvInfer.h:4032
void setSelectorForSinglePixel(ResizeSelector selector) noexcept
Set coordinate selector function when resized to single pixel.
Definition: NvInfer.h:4193
void setNearestRounding(ResizeRoundMode value) noexcept
Set rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4217
virtual ~IResizeLayer() noexcept=default
int32_t getScales(int32_t size, float *scales) const noexcept
Copies resize scales to scales[0, ..., nbScales-1], where nbScales is the number of scales that were ...
Definition: NvInfer.h:4111
void setOutputDimensions(Dims const &dimensions) noexcept
Set the output dimensions.
Definition: NvInfer.h:4052
void setCubicCoeff(float A) noexcept
Set the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:4249
void setScales(float const *scales, int32_t nbScales) noexcept
Set the resize scales.
Definition: NvInfer.h:4092
float getCubicCoeff() const noexcept
Get the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:4259
ResizeSelector getSelectorForSinglePixel() const noexcept
Get the coordinate selector function when resized to single pixel.
Definition: NvInfer.h:4203
InterpolationMode getResizeMode() const noexcept
Get resize mode for an input tensor.
Definition: NvInfer.h:4133
void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept
Set coordinate transformation function.
Definition: NvInfer.h:4168
void setExcludeOutside(bool excludeFlag) noexcept
Set the state for excluding outside pixels.
Definition: NvInfer.h:4272
void setResizeMode(InterpolationMode interpolationMode) noexcept
Set resize mode for an input tensor.
Definition: NvInfer.h:4123
Dims getOutputDimensions() const noexcept
Get the output dimensions.
Definition: NvInfer.h:4062
ResizeRoundMode getNearestRounding() const noexcept
Get rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4227
bool getExcludeOutside() const noexcept
Get the state for excluding outside pixels.
Definition: NvInfer.h:4282
ResizeCoordinateTransformation getCoordinateTransformation() const noexcept
Get coordinate transformation function.
Definition: NvInfer.h:4178
A ReverseSequence layer in a network definition.
Definition: NvInfer.h:5979
void setSequenceAxis(int32_t sequenceAxis) noexcept
Set the sequence axis. Default is 0.
Definition: NvInfer.h:6012
int32_t getBatchAxis() const noexcept
Return the batch axis. Return 1 if no batch axis was set.
Definition: NvInfer.h:5999
apiv::VReverseSequenceLayer * mImpl
Definition: NvInfer.h:6028
int32_t getSequenceAxis() const noexcept
Return the sequence axis. Return 0 if no sequence axis was set.
Definition: NvInfer.h:6022
void setBatchAxis(int32_t batchAxis) noexcept
Set the batch axis. Default is 1.
Definition: NvInfer.h:5989
virtual ~IReverseSequenceLayer() noexcept=default
A Scale layer in a network definition.
Definition: NvInfer.h:1848
Weights getScale() const noexcept
Get the scale value.
Definition: NvInfer.h:1905
Weights getPower() const noexcept
Get the power value.
Definition: NvInfer.h:1925
void setScale(Weights scale) noexcept
Set the scale value.
Definition: NvInfer.h:1895
void setPower(Weights power) noexcept
Set the power value.
Definition: NvInfer.h:1915
ScaleMode getMode() const noexcept
Get the scale mode.
Definition: NvInfer.h:1865
void setShift(Weights shift) noexcept
Set the shift value.
Definition: NvInfer.h:1875
void setChannelAxis(int32_t channelAxis) noexcept
Set the channel axis.
Definition: NvInfer.h:1961
Weights getShift() const noexcept
Get the shift value.
Definition: NvInfer.h:1885
virtual ~IScaleLayer() noexcept=default
void setMode(ScaleMode mode) noexcept
Set the scale mode.
Definition: NvInfer.h:1855
int32_t getChannelAxis() const noexcept
Get the channel axis.
Definition: NvInfer.h:1940
A scatter layer in a network definition. Supports several kinds of scattering.
Definition: NvInfer.h:5633
void setMode(ScatterMode mode) noexcept
Set the scatter mode.
Definition: NvInfer.h:5640
apiv::VScatterLayer * mImpl
Definition: NvInfer.h:5674
void setAxis(int32_t axis) noexcept
Set the axis used by ScatterMode::kELEMENTS.
Definition: NvInfer.h:5660
int32_t getAxis() const noexcept
Get the axis.
Definition: NvInfer.h:5668
ScatterMode getMode() const noexcept
Get the scatter mode.
Definition: NvInfer.h:5650
virtual ~IScatterLayer() noexcept=default
A select layer in a network definition.
Definition: NvInfer.h:4817
virtual ~ISelectLayer() noexcept=default
Layer type for getting shape of a tensor.
Definition: NvInfer.h:3421
virtual ~IShapeLayer() noexcept=default
apiv::VShapeLayer * mImpl
Definition: NvInfer.h:3423
Layer type for shuffling data.
Definition: NvInfer.h:3040
apiv::VShuffleLayer * mImpl
Definition: NvInfer.h:3195
void setFirstTranspose(Permutation permutation) noexcept
Set the permutation applied by the first transpose operation.
Definition: NvInfer.h:3051
void setSecondTranspose(Permutation permutation) noexcept
Set the permutation applied by the second transpose operation.
Definition: NvInfer.h:3148
Dims getReshapeDimensions() const noexcept
Get the reshaped dimensions.
Definition: NvInfer.h:3101
void setReshapeDimensions(Dims const &dimensions) noexcept
Set the reshaped dimensions.
Definition: NvInfer.h:3088
Permutation getFirstTranspose() const noexcept
Get the permutation applied by the first transpose operation.
Definition: NvInfer.h:3063
virtual ~IShuffleLayer() noexcept=default
Permutation getSecondTranspose() const noexcept
Get the permutation applied by the second transpose operation.
Definition: NvInfer.h:3160
bool getZeroIsPlaceholder() const noexcept
Get meaning of 0 in reshape dimensions.
Definition: NvInfer.h:3189
void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
Set meaning of 0 in reshape dimensions.
Definition: NvInfer.h:3176
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:3269
void setStride(Dims const &stride) noexcept
Set the stride for computing the output slice data.
Definition: NvInfer.h:3338
apiv::VSliceLayer * mImpl
Definition: NvInfer.h:3404
virtual ~ISliceLayer() noexcept=default
void setSize(Dims const &size) noexcept
Set the dimensions of the output slice.
Definition: NvInfer.h:3309
void setStart(Dims const &start) noexcept
Set the start offset that the slice layer uses to create the output slice.
Definition: NvInfer.h:3280
Dims getStart() const noexcept
Get the start offset for the slice layer.
Definition: NvInfer.h:3295
void setMode(SampleMode mode) noexcept
Set the slice mode.
Definition: NvInfer.h:3363
Dims getSize() const noexcept
Get dimensions of the output slice.
Definition: NvInfer.h:3324
SampleMode getMode() const noexcept
Get the slice mode.
Definition: NvInfer.h:3373
Dims getStride() const noexcept
Get the stride for the output slice.
Definition: NvInfer.h:3353
A Softmax layer in a network definition.
Definition: NvInfer.h:1992
void setAxes(uint32_t axes) noexcept
Set the axis along which softmax is computed. Currently, only one axis can be set.
Definition: NvInfer.h:2014
uint32_t getAxes() const noexcept
Get the axis along which softmax occurs.
Definition: NvInfer.h:2024
virtual ~ISoftMaxLayer() noexcept=default
A tensor in a network definition.
Definition: NvInfer.h:181
bool setDynamicRange(float min, float max) noexcept
Set dynamic range for the tensor.
Definition: NvInfer.h:284
void setAllowedFormats(TensorFormats formats) noexcept
Set allowed formats for this tensor. By default all formats are allowed. Shape tensors (for which isS...
Definition: NvInfer.h:420
TensorLocation getLocation() const noexcept
Get the storage location of a tensor.
Definition: NvInfer.h:343
void setDimensions(Dims const &dimensions) noexcept
Set the dimensions of a tensor.
Definition: NvInfer.h:228
void resetDynamicRange() noexcept
Undo effect of setDynamicRange.
Definition: NvInfer.h:380
void setName(char const *name) noexcept
Set the tensor name.
Definition: NvInfer.h:197
bool isExecutionTensor() const noexcept
Whether the tensor is an execution tensor.
Definition: NvInfer.h:485
void setType(DataType type) noexcept
Set the data type of a tensor.
Definition: NvInfer.h:257
bool dynamicRangeIsSet() const noexcept
Query whether dynamic range is set.
Definition: NvInfer.h:372
char const * getName() const noexcept
Get the tensor name.
Definition: NvInfer.h:209
bool isShapeTensor() const noexcept
Whether the tensor is a shape tensor.
Definition: NvInfer.h:464
float getDynamicRangeMax() const noexcept
Get maximum of dynamic range.
Definition: NvInfer.h:400
bool isNetworkInput() const noexcept
Whether the tensor is a network input.
Definition: NvInfer.h:292
TRT_DEPRECATED void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
Set whether to enable broadcast of tensor across the implicit batch dimension.
Definition: NvInfer.h:317
TRT_DEPRECATED bool getBroadcastAcrossBatch() const noexcept
Check if tensor is broadcast across the implicit batch dimension.
Definition: NvInfer.h:331
bool isNetworkOutput() const noexcept
Whether the tensor is a network output.
Definition: NvInfer.h:300
DataType getType() const noexcept
Get the data type of a tensor.
Definition: NvInfer.h:269
apiv::VTensor * mImpl
Definition: NvInfer.h:532
float getDynamicRangeMin() const noexcept
Get minimum of dynamic range.
Definition: NvInfer.h:390
virtual ~ITensor() noexcept=default
void setDimensionName(int32_t index, char const *name) noexcept
Name a dimension of an input tensor.
Definition: NvInfer.h:511
char const * getDimensionName(int32_t index) const noexcept
Get the name of an input dimension.
Definition: NvInfer.h:526
TRT_DEPRECATED void setLocation(TensorLocation location) noexcept
Set the storage location of a tensor.
Definition: NvInfer.h:362
Dims getDimensions() const noexcept
Get the dimensions of a tensor.
Definition: NvInfer.h:242
TensorFormats getAllowedFormats() const noexcept
Get a bitmask of TensorFormat values that the tensor supports. For a shape tensor,...
Definition: NvInfer.h:433
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:8259
bool combine(ITimingCache const &inputCache, bool ignoreMismatch) noexcept
Combine input timing cache into local instance.
Definition: NvInfer.h:8296
virtual ~ITimingCache() noexcept=default
apiv::VTimingCache * mImpl
Definition: NvInfer.h:8312
bool reset() noexcept
Empty the timing cache.
Definition: NvInfer.h:8306
Layer that represents a TopK reduction.
Definition: NvInfer.h:3461
void setK(int32_t k) noexcept
Set the static k value for the layer.
Definition: NvInfer.h:3492
void setReduceAxes(uint32_t reduceAxes) noexcept
Set which axes to reduce for the layer.
Definition: NvInfer.h:3516
TopKOperation getOperation() const noexcept
Get the operation for the layer.
Definition: NvInfer.h:3478
apiv::VTopKLayer * mImpl
Definition: NvInfer.h:3548
void setOperation(TopKOperation op) noexcept
Set the operation for the layer.
Definition: NvInfer.h:3468
int32_t getK() const noexcept
Get the k value for the layer.
Definition: NvInfer.h:3506
uint32_t getReduceAxes() const noexcept
Get the axes to reduce for the layer.
Definition: NvInfer.h:3526
virtual ~ITopKLayer() noexcept=default
A layer that represents a trip-count limiter.
Definition: NvInfer.h:4645
TripLimit getTripLimit() const noexcept
Get a trip limiter type.
Definition: NvInfer.h:4650
virtual ~ITripLimitLayer() noexcept=default
Layer that represents an unary operation.
Definition: NvInfer.h:2799
void setOperation(UnaryOperation op) noexcept
Set the unary operation for the layer.
Definition: NvInfer.h:2808
apiv::VUnaryLayer * mImpl
Definition: NvInfer.h:2824
UnaryOperation getOperation() const noexcept
Get the unary operation for the layer.
Definition: NvInfer.h:2818
virtual ~IUnaryLayer() noexcept=default
An Interface class for version control.
Definition: NvInferRuntimeBase.h:393
Version information associated with a TRT interface.
Definition: NvInferRuntimeBase.h:358
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:121
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferSafeRuntime.h:828
Definition: NvInfer.h:8006
virtual int32_t selectAlgorithms(IAlgorithmContext const &context, IAlgorithm const *const *choices, int32_t nbChoices, int32_t *selection) noexcept=0
Select Algorithms for a layer from the given list of algorithm choices.
virtual void reportAlgorithms(IAlgorithmContext const *const *algoContexts, IAlgorithm const *const *algoChoices, int32_t nbAlgorithms) noexcept=0
Called by TensorRT to report choices it made.
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8011
virtual ~IAlgorithmSelector() noexcept=default
Definition: NvInferRuntimeBase.h:850
Definition: NvInferRuntimeBase.h:462
~IInt8EntropyCalibrator2() noexcept override=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7660
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7652
Definition: NvInfer.h:7609
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7622
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7614
~IInt8EntropyCalibrator() noexcept override=default
Definition: NvInfer.h:7722
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7735
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7727
virtual double getQuantile() const noexcept=0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
Definition: NvInfer.h:7685
~IInt8MinMaxCalibrator() noexcept override=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7698
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7690
Definition: NvInferRuntime.h:687
Definition: NvInfer.h:8457
virtual bool stepComplete(char const *phaseName, int32_t step) noexcept=0
Signal that a step of an optimizer phase has finished.
virtual ~IProgressMonitor() noexcept=default
virtual void phaseFinish(char const *phaseName) noexcept=0
Signal that a phase of the optimizer has finished.
virtual void phaseStart(char const *phaseName, char const *parentPhase, int32_t nbSteps) noexcept=0
Signal that a phase of the optimizer has started.
IBuilder * createInferBuilder(ILogger &logger) noexcept
Create an instance of an IBuilder class.
Definition: NvInfer.h:9661
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2292
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:3937
@ kFORMULA
Use formula to map the original index.
@ kUPPER
Select the upper left pixel.
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:73
nvinfer1::IPluginRegistry * getBuilderPluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for building a Standard engine, or nullptr if no registry exists.
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8323
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1805
@ kUNIFORM
Identical coefficients across all elements of the tensor.
@ kCHANNEL
Per-channel coefficients.
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:8067
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:8428
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:5824
@ kCENTER_SIZES
(x_center, y_center, width, height) where (x_center, y_center) is the center point of the box
@ kCORNER_PAIRS
(x1, y1, x2, y2) where (x1, y1) and (x2, y2) are any pair of diagonal corners
constexpr int32_t EnumMax< BuilderFlag >() noexcept
Definition: NvInfer.h:8243
constexpr int32_t EnumMax< LayerType >() noexcept
Definition: NvInfer.h:114
constexpr int32_t EnumMax< CalibrationAlgoType >() noexcept
Definition: NvInfer.h:7526
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2753
@ kISINF
Return true if input value equals +/- infinity for floating-point data type.
@ kCOSH
Hyperbolic cosine.
@ kACOSH
Inverse hyperbolic cosine.
@ kERF
Gauss error function.
@ kACOS
Inverse cosine.
@ kABS
Absolute value.
@ kSINH
Hyperbolic sine.
@ kROUND
Round to nearest even for floating-point data type.
@ kATANH
Inverse hyperbolic tangent.
@ kASINH
Inverse hyperbolic sine.
@ kSIGN
Sign, If input > 0, output 1; if input < 0, output -1; if input == 0, output 0.
@ kEXP
Exponentiation.
@ kATAN
Inverse tangent.
constexpr int32_t EnumMax< ReduceOperation >() noexcept
Definition: NvInfer.h:2861
constexpr int32_t EnumMax< TripLimit >() noexcept
Definition: NvInfer.h:4334
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:133
@ kSELU
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
@ kTANH
TanH activation.
@ kSCALED_TANH
Scaled tanh activation: alpha*tanh(beta*x)
@ kRELU
Rectified linear activation.
@ kELU
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
@ kLEAKY_RELU
LeakyRelu activation: x>=0 ? x : alpha * x.
@ kSOFTSIGN
Softsign activation: x / (1+|x|)
@ kHARD_SIGMOID
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
@ kTHRESHOLDED_RELU
Thresholded ReLU activation: x>alpha ? x : 0.
@ kSIGMOID
Sigmoid activation.
@ kCLIP
Clip activation: max(alpha, min(beta, x))
@ kGELU_TANH
GELU tanh activation: 0.5 * x * (1 + tanh(sqrt(2/pi) * (0.044715F * pow(x, 3) + x)))
@ kGELU_ERF
GELU erf activation: 0.5 * x * (1 + erf(sqrt(0.5) * x))
@ kSOFTPLUS
Parametric softplus activation: alpha*log(exp(beta*x)+1)
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:4878
@ kRANDOM_UNIFORM
Randomly draw values from a uniform distribution.
@ kRANDOM_NORMAL
Randomly draw values from a normal distribution.
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:3967
@ kHALF_UP
Round half up.
@ kHALF_DOWN
Round half down.
nvinfer1::safe::IPluginRegistry * getBuilderSafePluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for building a Safety engine, or nullptr if no registry exists.
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:983
@ kSAME_LOWER
Use SAME padding, with prePadding >= postPadding.
@ kEXPLICIT_ROUND_DOWN
Use explicit padding, rounding output size down.
@ kEXPLICIT_ROUND_UP
Use explicit padding, rounding output size up.
@ kSAME_UPPER
Use SAME padding, with prePadding <= postPadding.
TripLimit
Definition: NvInfer.h:4322
@ kWHILE
Tensor is a scalar of type kBOOL. Loop terminates when value is false.
@ kCOUNT
Tensor is a scalar of type kINT32 or kINT64 that contains the trip count.
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:9349
PreviewFeature
Define preview features.
Definition: NvInfer.h:8395
constexpr int32_t EnumMax< GatherMode >() noexcept
Definition: NvInfer.h:2513
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:129
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kFP16 ...
Definition: NvInfer.h:8101
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:1252
constexpr int32_t EnumMax< ScaleMode >() noexcept
Definition: NvInfer.h:1817
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7513
@ kENTROPY_CALIBRATION_2
Entropy calibration.
@ kLEGACY_CALIBRATION
Legacy calibration.
@ kENTROPY_CALIBRATION
Legacy entropy calibration.
@ kMINMAX_CALIBRATION
Minmax calibration.
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
@ kGRID_SAMPLE
Grid sample layer.
@ kRAGGED_SOFTMAX
Ragged softmax layer.
@ kDECONVOLUTION
Deconvolution layer.
@ kREDUCE
Reduce layer.
@ kASSERTION
Assertion layer.
@ kTOPK
TopK layer.
@ kRESIZE
Resize Layer.
@ kCAST
Cast layer.
@ kPADDING
Padding layer.
@ kMATRIX_MULTIPLY
Matrix multiply layer.
@ kCONDITION
Condition layer.
@ kCONDITIONAL_INPUT
Conditional Input layer.
@ kIDENTITY
Identity layer.
@ kNORMALIZATION
Normalization layer.
@ kQUANTIZE
Quantize layer.
@ kSCATTER
Scatter layer.
@ kCONVOLUTION
Convolution layer.
@ kPARAMETRIC_RELU
Parametric ReLU layer.
@ kCONCATENATION
Concatenation layer.
@ kONE_HOT
OneHot layer.
@ kREVERSE_SEQUENCE
Reverse sequence layer.
@ kSLICE
Slice layer.
@ kEINSUM
Einsum layer.
@ kSOFTMAX
SoftMax layer.
@ kSHAPE
Shape layer.
@ kRECURRENCE
Loop Recurrence layer.
@ kDEQUANTIZE
Dequantize layer.
@ kSHUFFLE
Shuffle layer.
@ kPLUGIN_V3
PluginV3 layer.
@ kITERATOR
Loop Iterator layer.
@ kPOOLING
Pooling layer.
@ kTRIP_LIMIT
Loop Trip limit layer.
@ kSCALE
Scale layer.
@ kGATHER
Gather layer.
@ kUNARY
UnaryOp operation Layer.
@ kACTIVATION
Activation layer.
@ kELEMENTWISE
Elementwise layer.
@ kSELECT
Select layer.
@ kPLUGIN_V2
PluginV2 layer.
@ kLOOP_OUTPUT
Loop output layer.
@ kCONDITIONAL_OUTPUT
Conditional Output layer.
@ kCONSTANT
Constant layer.
@ kNON_ZERO
NonZero layer.
@ kFILL
Fill layer.
@ kPLUGIN
Plugin layer.
constexpr int32_t EnumMax< QuantizationFlag >() noexcept
Definition: NvInfer.h:8090
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3205
@ kCLAMP
Out of bounds indices are clamped to bounds.
@ kSTRICT_BOUNDS
Fail with error when the coordinates are out of bounds.
@ kWRAP
Coordinates wrap around periodically.
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2501
@ kDEFAULT
Similar to ONNX Gather.
@ kELEMENT
Similar to ONNX GatherElements.
@ kND
Similar to ONNX GatherND.
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:125
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2304
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:9360
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2411
@ kSUB
Subtract the second element from the first.
@ kSUM
Sum of the two elements.
@ kPROD
Product of the two elements.
@ kFLOOR_DIV
Floor division of the first element by the second.
@ kEQUAL
Check if two elements are equal.
@ kAND
Logical AND of two elements.
@ kOR
Logical OR of two elements.
@ kMIN
Minimum of the two elements.
@ kPOW
The first element to the power of the second element.
@ kLESS
Check if element in first tensor is less than corresponding element in second tensor.
@ kGREATER
Check if element in first tensor is greater than corresponding element in second tensor.
@ kXOR
Logical XOR of two elements.
@ kDIV
Divide the first element by the second.
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:8077
constexpr int32_t EnumMax< SampleMode >() noexcept
Definition: NvInfer.h:3221
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:3855
@ kNEAREST
ND (0 < N <= 8) nearest neighbor resizing.
@ kCUBIC
Supports bicubic (2D) interpolation.
@ kLINEAR
Supports linear (1D), bilinear (2D), and trilinear (3D) interpolation.
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:8111
@ kWEIGHT_STREAMING
Enable weight streaming for the current engine.
@ kDEBUG
Enable debugging of layers via synchronizing after every layer.
@ kGPU_FALLBACK
Enable layers marked to execute on GPU if layer cannot execute on DLA.
@ kSPARSE_WEIGHTS
Allow the builder to examine weights and use optimized functions when weights have suitable sparsity.
@ kFP16
Enable FP16 layer selection, with FP32 fallback.
@ kINT8
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
@ kSTRIP_PLAN
Strip the refittable weights from the engine plan file.
@ kDISABLE_TIMING_CACHE
Disable reuse of timing information across identical layers.
@ kREFIT
Enable building a refittable engine.
@ kOBEY_PRECISION_CONSTRAINTS
Require that layers execute in specified precisions. Build fails otherwise.
@ kREJECT_EMPTY_ALGORITHMS
Fail if IAlgorithmSelector::selectAlgorithms returns an empty set of algorithms.
constexpr int32_t EnumMax< TopKOperation >() noexcept
Definition: NvInfer.h:3444
constexpr int32_t EnumMax< MemoryPoolType >() noexcept
Definition: NvInfer.h:8381
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3433
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:2847
constexpr int32_t EnumMax< LoopOutput >() noexcept
Definition: NvInfer.h:4313
constexpr int32_t EnumMax< NetworkDefinitionCreationFlag >() noexcept
Definition: NvInfer.h:9379
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5560
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3559
@ kTRANSPOSE
Like kNONE, but transpose the matrix dimensions.
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:3883
constexpr int32_t EnumMax< UnaryOperation >() noexcept
Definition: NvInfer.h:2786
LoopOutput
Definition: NvInfer.h:4296
@ kLAST_VALUE
Output value is value of tensor for last iteration.
@ kCONCATENATE
Output value is concatenation of values of tensor for each iteration, in forward order.
@ kREVERSE
Output value is concatenation of values of tensor for each iteration, in reverse order.
constexpr int32_t EnumMax< BoundingBoxFormat >() noexcept
Definition: NvInfer.h:5837
constexpr int32_t EnumMax< MatrixOperation >() noexcept
Definition: NvInfer.h:3587