TensorRT 10.6.0
NvInfer.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_H
19#define NV_INFER_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntime.h"
23
32//
35
41
47namespace nvinfer1
48{
49
57enum class LayerType : int32_t
58{
59 kCONVOLUTION = 0,
60 kCAST = 1,
61 kACTIVATION = 2,
62 kPOOLING = 3,
63 kLRN = 4,
64 kSCALE = 5,
65 kSOFTMAX = 6,
66 kDECONVOLUTION = 7,
67 kCONCATENATION = 8,
68 kELEMENTWISE = 9,
69 kPLUGIN = 10,
70 kUNARY = 11,
71 kPADDING = 12,
72 kSHUFFLE = 13,
73 kREDUCE = 14,
74 kTOPK = 15,
75 kGATHER = 16,
76 kMATRIX_MULTIPLY = 17,
77 kRAGGED_SOFTMAX = 18,
78 kCONSTANT = 19,
79 kIDENTITY = 20,
80 kPLUGIN_V2 = 21,
81 kSLICE = 22,
82 kSHAPE = 23,
83 kPARAMETRIC_RELU = 24,
84 kRESIZE = 25,
85 kTRIP_LIMIT = 26,
86 kRECURRENCE = 27,
87 kITERATOR = 28,
88 kLOOP_OUTPUT = 29,
89 kSELECT = 30,
90 kFILL = 31,
91 kQUANTIZE = 32,
92 kDEQUANTIZE = 33,
93 kCONDITION = 34,
96 kSCATTER = 37,
97 kEINSUM = 38,
98 kASSERTION = 39,
99 kONE_HOT = 40,
100 kNON_ZERO = 41,
101 kGRID_SAMPLE = 42,
102 kNMS = 43,
103 kREVERSE_SEQUENCE = 44,
104 kNORMALIZATION = 45,
105 kPLUGIN_V3 = 46,
106};
107
113template <>
114constexpr inline int32_t EnumMax<LayerType>() noexcept
115{
116 return 47;
117}
118
125using TensorFormats = uint32_t;
126
132enum class ActivationType : int32_t
133{
134 kRELU = 0,
135 kSIGMOID = 1,
136 kTANH = 2,
137 kLEAKY_RELU = 3,
138 kELU = 4,
139 kSELU = 5,
140 kSOFTSIGN = 6,
141 kSOFTPLUS = 7,
142 kCLIP = 8,
143 kHARD_SIGMOID = 9,
144 kSCALED_TANH = 10,
145 kTHRESHOLDED_RELU = 11,
146 kGELU_ERF = 12,
147 kGELU_TANH = 13
148};
149
150namespace impl
151{
157template <>
159{
160 static constexpr int32_t kVALUE = 14;
161};
162} // namespace impl
163
180class ITensor : public INoCopy
181{
182public:
197 void setName(char const* name) noexcept
198 {
199 mImpl->setName(name);
200 }
201
209 char const* getName() const noexcept
210 {
211 return mImpl->getName();
212 }
213
228 void setDimensions(Dims const& dimensions) noexcept
229 {
230 mImpl->setDimensions(dimensions);
231 }
232
242 Dims getDimensions() const noexcept
243 {
244 return mImpl->getDimensions();
245 }
246
257 void setType(DataType type) noexcept
258 {
259 mImpl->setType(type);
260 }
261
269 DataType getType() const noexcept
270 {
271 return mImpl->getType();
272 }
273
286 TRT_DEPRECATED bool setDynamicRange(float min, float max) noexcept
287 {
288 return mImpl->setDynamicRange(min, max);
289 }
290
294 bool isNetworkInput() const noexcept
295 {
296 return mImpl->isNetworkInput();
297 }
298
302 bool isNetworkOutput() const noexcept
303 {
304 return mImpl->isNetworkOutput();
305 }
306
319 TRT_DEPRECATED void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
320 {
321 mImpl->setBroadcastAcrossBatch(broadcastAcrossBatch);
322 }
323
334 {
335 return mImpl->getBroadcastAcrossBatch();
336 }
337
346 {
347 return mImpl->getLocation();
348 }
349
365 {
366 mImpl->setLocation(location);
367 }
368
376 bool dynamicRangeIsSet() const noexcept
377 {
378 return mImpl->dynamicRangeIsSet();
379 }
380
384 void resetDynamicRange() noexcept
385 {
386 mImpl->resetDynamicRange();
387 }
388
394 float getDynamicRangeMin() const noexcept
395 {
396 return mImpl->getDynamicRangeMin();
397 }
398
404 float getDynamicRangeMax() const noexcept
405 {
406 return mImpl->getDynamicRangeMax();
407 }
408
426 void setAllowedFormats(TensorFormats formats) noexcept
427 {
428 mImpl->setAllowedFormats(formats);
429 }
430
440 {
441 return mImpl->getAllowedFormats();
442 }
443
470 bool isShapeTensor() const noexcept
471 {
472 return mImpl->isShapeTensor();
473 }
474
491 bool isExecutionTensor() const noexcept
492 {
493 return mImpl->isExecutionTensor();
494 }
495
517 void setDimensionName(int32_t index, char const* name) noexcept
518 {
519 mImpl->setDimensionName(index, name);
520 }
521
532 char const* getDimensionName(int32_t index) const noexcept
533 {
534 return mImpl->getDimensionName(index);
535 }
536
537protected:
538 apiv::VTensor* mImpl;
539 virtual ~ITensor() noexcept = default;
540};
541
549class ILayer : public INoCopy
550{
551public:
557 LayerType getType() const noexcept
558 {
559 return mLayer->getType();
560 }
561
571 void setName(char const* name) noexcept
572 {
573 mLayer->setName(name);
574 }
575
581 char const* getName() const noexcept
582 {
583 return mLayer->getName();
584 }
585
589 int32_t getNbInputs() const noexcept
590 {
591 return mLayer->getNbInputs();
592 }
593
602 ITensor* getInput(int32_t index) const noexcept
603 {
604 return mLayer->getInput(index);
605 }
606
610 int32_t getNbOutputs() const noexcept
611 {
612 return mLayer->getNbOutputs();
613 }
614
620 ITensor* getOutput(int32_t index) const noexcept
621 {
622 return mLayer->getOutput(index);
623 }
624
637 void setInput(int32_t index, ITensor& tensor) noexcept
638 {
639 return mLayer->setInput(index, tensor);
640 }
641
668 void setPrecision(DataType dataType) noexcept
669 {
670 mLayer->setPrecision(dataType);
671 }
672
680 DataType getPrecision() const noexcept
681 {
682 return mLayer->getPrecision();
683 }
684
692 bool precisionIsSet() const noexcept
693 {
694 return mLayer->precisionIsSet();
695 }
696
702 void resetPrecision() noexcept
703 {
704 mLayer->resetPrecision();
705 }
706
749 void setOutputType(int32_t index, DataType dataType) noexcept
750 {
751 mLayer->setOutputType(index, dataType);
752 }
753
764 DataType getOutputType(int32_t index) const noexcept
765 {
766 return mLayer->getOutputType(index);
767 }
768
778 bool outputTypeIsSet(int32_t index) const noexcept
779 {
780 return mLayer->outputTypeIsSet(index);
781 }
782
790 void resetOutputType(int32_t index) noexcept
791 {
792 return mLayer->resetOutputType(index);
793 }
794
808 void setMetadata(char const* metadata) noexcept
809 {
810 mLayer->setMetadata(metadata);
811 }
812
821 char const* getMetadata() const noexcept
822 {
823 return mLayer->getMetadata();
824 }
825
826protected:
827 virtual ~ILayer() noexcept = default;
828 apiv::VLayer* mLayer;
829};
830
987enum class PaddingMode : int32_t
988{
991 kSAME_UPPER = 2,
992 kSAME_LOWER = 3,
993};
994
995namespace impl
996{
1002template <>
1004{
1005 static constexpr int32_t kVALUE = 4;
1006};
1007} // namespace impl
1008
1022{
1023public:
1031 void setNbOutputMaps(int64_t nbOutputMaps) noexcept
1032 {
1033 mImpl->setNbOutputMaps(nbOutputMaps);
1034 }
1035
1041 int64_t getNbOutputMaps() const noexcept
1042 {
1043 return mImpl->getNbOutputMaps();
1044 }
1045
1061 void setNbGroups(int64_t nbGroups) noexcept
1062 {
1063 mImpl->setNbGroups(nbGroups);
1064 }
1065
1071 int64_t getNbGroups() const noexcept
1072 {
1073 return mImpl->getNbGroups();
1074 }
1075
1085 void setKernelWeights(Weights weights) noexcept
1086 {
1087 mImpl->setKernelWeights(weights);
1088 }
1089
1095 Weights getKernelWeights() const noexcept
1096 {
1097 return mImpl->getKernelWeights();
1098 }
1099
1110 void setBiasWeights(Weights weights) noexcept
1111 {
1112 mImpl->setBiasWeights(weights);
1113 }
1114
1120 Weights getBiasWeights() const noexcept
1121 {
1122 return mImpl->getBiasWeights();
1123 }
1124
1137 void setPrePadding(Dims const& padding) noexcept
1138 {
1139 mImpl->setPrePadding(padding);
1140 }
1141
1147 Dims getPrePadding() const noexcept
1148 {
1149 return mImpl->getPrePadding();
1150 }
1151
1164 void setPostPadding(Dims const& padding) noexcept
1165 {
1166 mImpl->setPostPadding(padding);
1167 }
1168
1174 Dims getPostPadding() const noexcept
1175 {
1176 return mImpl->getPostPadding();
1177 }
1178
1188 void setPaddingMode(PaddingMode paddingMode) noexcept
1189 {
1190 mImpl->setPaddingMode(paddingMode);
1191 }
1192
1201 {
1202 return mImpl->getPaddingMode();
1203 }
1204
1213 void setKernelSizeNd(Dims const& kernelSize) noexcept
1214 {
1215 mImpl->setKernelSizeNd(kernelSize);
1216 }
1217
1223 Dims getKernelSizeNd() const noexcept
1224 {
1225 return mImpl->getKernelSizeNd();
1226 }
1227
1238 void setStrideNd(Dims const& stride) noexcept
1239 {
1240 mImpl->setStrideNd(stride);
1241 }
1242
1248 Dims getStrideNd() const noexcept
1249 {
1250 return mImpl->getStrideNd();
1251 }
1252
1266 void setPaddingNd(Dims const& padding) noexcept
1267 {
1268 mImpl->setPaddingNd(padding);
1269 }
1270
1278 Dims getPaddingNd() const noexcept
1279 {
1280 return mImpl->getPaddingNd();
1281 }
1282
1292 void setDilationNd(Dims const& dilation) noexcept
1293 {
1294 mImpl->setDilationNd(dilation);
1295 }
1296
1302 Dims getDilationNd() const noexcept
1303 {
1304 return mImpl->getDilationNd();
1305 }
1306
1321 using ILayer::setInput;
1322
1323protected:
1324 virtual ~IConvolutionLayer() noexcept = default;
1325 apiv::VConvolutionLayer* mImpl;
1326};
1327
1342{
1343public:
1352 {
1353 mImpl->setActivationType(type);
1354 }
1355
1362 {
1363 return mImpl->getActivationType();
1364 }
1365
1376 void setAlpha(float alpha) noexcept
1377 {
1378 mImpl->setAlpha(alpha);
1379 }
1380
1390 void setBeta(float beta) noexcept
1391 {
1392 mImpl->setBeta(beta);
1393 }
1394
1399 float getAlpha() const noexcept
1400 {
1401 return mImpl->getAlpha();
1402 }
1403
1408 float getBeta() const noexcept
1409 {
1410 return mImpl->getBeta();
1411 }
1412
1413protected:
1414 virtual ~IActivationLayer() noexcept = default;
1415 apiv::VActivationLayer* mImpl;
1416};
1417
1423enum class PoolingType : int32_t
1424{
1425 kMAX = 0,
1426 kAVERAGE = 1,
1428};
1429
1430namespace impl
1431{
1437template <>
1439{
1440 static constexpr int32_t kVALUE = 3;
1441};
1442} // namespace impl
1443
1455class IPoolingLayer : public ILayer
1456{
1457public:
1465 void setPoolingType(PoolingType type) noexcept
1466 {
1467 mImpl->setPoolingType(type);
1468 }
1469
1476 {
1477 return mImpl->getPoolingType();
1478 }
1479
1490 void setBlendFactor(float blendFactor) noexcept
1491 {
1492 mImpl->setBlendFactor(blendFactor);
1493 }
1494
1503 float getBlendFactor() const noexcept
1504 {
1505 return mImpl->getBlendFactor();
1506 }
1507
1517 void setAverageCountExcludesPadding(bool exclusive) noexcept
1518 {
1519 mImpl->setAverageCountExcludesPadding(exclusive);
1520 }
1521
1529 {
1530 return mImpl->getAverageCountExcludesPadding();
1531 }
1532
1546 void setPrePadding(Dims const& padding) noexcept
1547 {
1548 mImpl->setPrePadding(padding);
1549 }
1550
1556 Dims getPrePadding() const noexcept
1557 {
1558 return mImpl->getPrePadding();
1559 }
1560
1574 void setPostPadding(Dims const& padding) noexcept
1575 {
1576 mImpl->setPostPadding(padding);
1577 }
1578
1584 Dims getPostPadding() const noexcept
1585 {
1586 return mImpl->getPostPadding();
1587 }
1588
1597 void setPaddingMode(PaddingMode paddingMode) noexcept
1598 {
1599 mImpl->setPaddingMode(paddingMode);
1600 }
1601
1609 {
1610 return mImpl->getPaddingMode();
1611 }
1612
1621 void setWindowSizeNd(Dims const& windowSize) noexcept
1622 {
1623 mImpl->setWindowSizeNd(windowSize);
1624 }
1625
1631 Dims getWindowSizeNd() const noexcept
1632 {
1633 return mImpl->getWindowSizeNd();
1634 }
1635
1646 void setStrideNd(Dims const& stride) noexcept
1647 {
1648 mImpl->setStrideNd(stride);
1649 }
1650
1656 Dims getStrideNd() const noexcept
1657 {
1658 return mImpl->getStrideNd();
1659 }
1660
1675 void setPaddingNd(Dims const& padding) noexcept
1676 {
1677 mImpl->setPaddingNd(padding);
1678 }
1679
1687 Dims getPaddingNd() const noexcept
1688 {
1689 return mImpl->getPaddingNd();
1690 }
1691
1692protected:
1693 virtual ~IPoolingLayer() noexcept = default;
1694 apiv::VPoolingLayer* mImpl;
1695};
1696
1706class ILRNLayer : public ILayer
1707{
1708public:
1718 void setWindowSize(int64_t windowSize) noexcept
1719 {
1720 mImpl->setWindowSize(windowSize);
1721 }
1722
1728 int64_t getWindowSize() const noexcept
1729 {
1730 return mImpl->getWindowSize();
1731 }
1732
1740 void setAlpha(float alpha) noexcept
1741 {
1742 mImpl->setAlpha(alpha);
1743 }
1744
1750 float getAlpha() const noexcept
1751 {
1752 return mImpl->getAlpha();
1753 }
1754
1762 void setBeta(float beta) noexcept
1763 {
1764 mImpl->setBeta(beta);
1765 }
1766
1772 float getBeta() const noexcept
1773 {
1774 return mImpl->getBeta();
1775 }
1776
1784 void setK(float k) noexcept
1785 {
1786 mImpl->setK(k);
1787 }
1788
1794 float getK() const noexcept
1795 {
1796 return mImpl->getK();
1797 }
1798
1799protected:
1800 virtual ~ILRNLayer() noexcept = default;
1801 apiv::VLRNLayer* mImpl;
1802};
1803
1809enum class ScaleMode : int32_t
1810{
1811 kUNIFORM = 0,
1812 kCHANNEL = 1,
1813 kELEMENTWISE = 2
1814};
1815
1821template <>
1822constexpr inline int32_t EnumMax<ScaleMode>() noexcept
1823{
1824 return 3;
1825}
1826
1852class IScaleLayer : public ILayer
1853{
1854public:
1860 void setMode(ScaleMode mode) noexcept
1861 {
1862 mImpl->setMode(mode);
1863 }
1864
1870 ScaleMode getMode() const noexcept
1871 {
1872 return mImpl->getMode();
1873 }
1874
1880 void setShift(Weights shift) noexcept
1881 {
1882 mImpl->setShift(shift);
1883 }
1884
1890 Weights getShift() const noexcept
1891 {
1892 return mImpl->getShift();
1893 }
1894
1900 void setScale(Weights scale) noexcept
1901 {
1902 mImpl->setScale(scale);
1903 }
1904
1910 Weights getScale() const noexcept
1911 {
1912 return mImpl->getScale();
1913 }
1914
1920 void setPower(Weights power) noexcept
1921 {
1922 mImpl->setPower(power);
1923 }
1924
1930 Weights getPower() const noexcept
1931 {
1932 return mImpl->getPower();
1933 }
1934
1945 int32_t getChannelAxis() const noexcept
1946 {
1947 return mImpl->getChannelAxis();
1948 }
1949
1966 void setChannelAxis(int32_t channelAxis) noexcept
1967 {
1968 mImpl->setChannelAxis(channelAxis);
1969 }
1970
1971protected:
1972 virtual ~IScaleLayer() noexcept = default;
1973 apiv::VScaleLayer* mImpl;
1974};
1975
1996class ISoftMaxLayer : public ILayer
1997{
1998public:
2019 void setAxes(uint32_t axes) noexcept
2020 {
2021 mImpl->setAxes(axes);
2022 }
2023
2029 uint32_t getAxes() const noexcept
2030 {
2031 return mImpl->getAxes();
2032 }
2033
2034protected:
2035 virtual ~ISoftMaxLayer() noexcept = default;
2036 apiv::VSoftMaxLayer* mImpl;
2037};
2038
2052{
2053public:
2065 void setAxis(int32_t axis) noexcept
2066 {
2067 mImpl->setAxis(axis);
2068 }
2069
2075 int32_t getAxis() const noexcept
2076 {
2077 return mImpl->getAxis();
2078 }
2079
2080protected:
2081 virtual ~IConcatenationLayer() noexcept = default;
2082 apiv::VConcatenationLayer* mImpl;
2083};
2084
2093{
2094public:
2102 void setNbOutputMaps(int64_t nbOutputMaps) noexcept
2103 {
2104 mImpl->setNbOutputMaps(nbOutputMaps);
2105 }
2106
2112 int64_t getNbOutputMaps() const noexcept
2113 {
2114 return mImpl->getNbOutputMaps();
2115 }
2116
2132 void setNbGroups(int64_t nbGroups) noexcept
2133 {
2134 mImpl->setNbGroups(nbGroups);
2135 }
2136
2142 int64_t getNbGroups() const noexcept
2143 {
2144 return mImpl->getNbGroups();
2145 }
2146
2156 void setKernelWeights(Weights weights) noexcept
2157 {
2158 mImpl->setKernelWeights(weights);
2159 }
2160
2166 Weights getKernelWeights() const noexcept
2167 {
2168 return mImpl->getKernelWeights();
2169 }
2170
2181 void setBiasWeights(Weights weights) noexcept
2182 {
2183 mImpl->setBiasWeights(weights);
2184 }
2185
2191 Weights getBiasWeights() const noexcept
2192 {
2193 return mImpl->getBiasWeights();
2194 }
2195
2208 void setPrePadding(Dims const& padding) noexcept
2209 {
2210 mImpl->setPrePadding(padding);
2211 }
2212
2218 Dims getPrePadding() const noexcept
2219 {
2220 return mImpl->getPrePadding();
2221 }
2222
2235 void setPostPadding(Dims const& padding) noexcept
2236 {
2237 mImpl->setPostPadding(padding);
2238 }
2239
2245 Dims getPostPadding() const noexcept
2246 {
2247 return mImpl->getPostPadding();
2248 }
2249
2259 void setPaddingMode(PaddingMode paddingMode) noexcept
2260 {
2261 mImpl->setPaddingMode(paddingMode);
2262 }
2263
2272 {
2273 return mImpl->getPaddingMode();
2274 }
2275
2286 void setKernelSizeNd(Dims const& kernelSize) noexcept
2287 {
2288 mImpl->setKernelSizeNd(kernelSize);
2289 }
2290
2296 Dims getKernelSizeNd() const noexcept
2297 {
2298 return mImpl->getKernelSizeNd();
2299 }
2300
2313 void setStrideNd(Dims const& stride) noexcept
2314 {
2315 mImpl->setStrideNd(stride);
2316 }
2317
2323 Dims getStrideNd() const noexcept
2324 {
2325 return mImpl->getStrideNd();
2326 }
2327
2341 void setPaddingNd(Dims const& padding) noexcept
2342 {
2343 mImpl->setPaddingNd(padding);
2344 }
2345
2353 Dims getPaddingNd() const noexcept
2354 {
2355 return mImpl->getPaddingNd();
2356 }
2357
2370 using ILayer::setInput;
2371
2379 void setDilationNd(Dims const& dilation) noexcept
2380 {
2381 mImpl->setDilationNd(dilation);
2382 }
2383
2389 Dims getDilationNd() const noexcept
2390 {
2391 return mImpl->getDilationNd();
2392 }
2393
2394protected:
2395 virtual ~IDeconvolutionLayer() noexcept = default;
2396 apiv::VDeconvolutionLayer* mImpl;
2397};
2398
2413enum class ElementWiseOperation : int32_t
2414{
2415 kSUM = 0,
2416 kPROD = 1,
2417 kMAX = 2,
2418 kMIN = 3,
2419 kSUB = 4,
2420 kDIV = 5,
2421 kPOW = 6,
2422 kFLOOR_DIV = 7,
2423 kAND = 8,
2424 kOR = 9,
2425 kXOR = 10,
2426 kEQUAL = 11,
2427 kGREATER = 12,
2428 kLESS = 13
2429};
2430
2431namespace impl
2432{
2438template <>
2440{
2441 static constexpr int32_t kVALUE = 14;
2442};
2443} // namespace impl
2444
2465{
2466public:
2477 {
2478 return mImpl->setOperation(op);
2479 }
2480
2489 {
2490 return mImpl->getOperation();
2491 }
2492
2493protected:
2494 apiv::VElementWiseLayer* mImpl;
2495 virtual ~IElementWiseLayer() noexcept = default;
2496};
2497
2503enum class GatherMode : int32_t
2504{
2505 kDEFAULT = 0,
2506 kELEMENT = 1,
2507 kND = 2
2508};
2509
2515template <>
2516constexpr inline int32_t EnumMax<GatherMode>() noexcept
2517{
2518 return 3;
2519}
2520
2597class IGatherLayer : public ILayer
2598{
2599public:
2609 void setGatherAxis(int32_t axis) noexcept
2610 {
2611 mImpl->setGatherAxis(axis);
2612 }
2613
2621 int32_t getGatherAxis() const noexcept
2622 {
2623 return mImpl->getGatherAxis();
2624 }
2625
2644 void setNbElementWiseDims(int32_t elementWiseDims) noexcept
2645 {
2646 mImpl->setNbElementWiseDims(elementWiseDims);
2647 }
2648
2654 int32_t getNbElementWiseDims() const noexcept
2655 {
2656 return mImpl->getNbElementWiseDims();
2657 }
2658
2664 void setMode(GatherMode mode) noexcept
2665 {
2666 mImpl->setMode(mode);
2667 }
2668
2674 GatherMode getMode() const noexcept
2675 {
2676 return mImpl->getMode();
2677 }
2678
2679protected:
2680 apiv::VGatherLayer* mImpl;
2681 virtual ~IGatherLayer() noexcept = default;
2682};
2683
2694{
2695public:
2702 {
2703 return mImpl->getPlugin();
2704 }
2705
2706protected:
2707 apiv::VPluginV2Layer* mImpl;
2708 virtual ~IPluginV2Layer() noexcept = default;
2709};
2710
2721{
2722public:
2729 {
2730 return mImpl->getPlugin();
2731 }
2732
2733protected:
2734 apiv::VPluginV3Layer* mImpl;
2735 virtual ~IPluginV3Layer() noexcept = default;
2736};
2737
2754enum class UnaryOperation : int32_t
2755{
2756 kEXP = 0,
2757 kLOG = 1,
2758 kSQRT = 2,
2759 kRECIP = 3,
2760 kABS = 4,
2761 kNEG = 5,
2762 kSIN = 6,
2763 kCOS = 7,
2764 kTAN = 8,
2765 kSINH = 9,
2766 kCOSH = 10,
2767 kASIN = 11,
2768 kACOS = 12,
2769 kATAN = 13,
2770 kASINH = 14,
2771 kACOSH = 15,
2772 kATANH = 16,
2773 kCEIL = 17,
2774 kFLOOR = 18,
2775 kERF = 19,
2776 kNOT = 20,
2777 kSIGN = 21,
2778 kROUND = 22,
2779 kISINF = 23,
2780 kISNAN = 24,
2781};
2782
2788template <>
2789constexpr inline int32_t EnumMax<UnaryOperation>() noexcept
2790{
2791 return 25;
2792}
2793
2801class IUnaryLayer : public ILayer
2802{
2803public:
2812 {
2813 mImpl->setOperation(op);
2814 }
2815
2822 {
2823 return mImpl->getOperation();
2824 }
2825
2826protected:
2827 apiv::VUnaryLayer* mImpl;
2828 virtual ~IUnaryLayer() noexcept = default;
2829};
2830
2849enum class ReduceOperation : int32_t
2850{
2851 kSUM = 0,
2852 kPROD = 1,
2853 kMAX = 2,
2854 kMIN = 3,
2855 kAVG = 4
2856};
2857
2863template <>
2864constexpr inline int32_t EnumMax<ReduceOperation>() noexcept
2865{
2866 return 5;
2867}
2868
2876class IReduceLayer : public ILayer
2877{
2878public:
2885 {
2886 mImpl->setOperation(op);
2887 }
2888
2895 {
2896 return mImpl->getOperation();
2897 }
2898
2904 void setReduceAxes(uint32_t reduceAxes) noexcept
2905 {
2906 mImpl->setReduceAxes(reduceAxes);
2907 }
2908
2914 uint32_t getReduceAxes() const noexcept
2915 {
2916 return mImpl->getReduceAxes();
2917 }
2918
2924 void setKeepDimensions(bool keepDimensions) noexcept
2925 {
2926 mImpl->setKeepDimensions(keepDimensions);
2927 }
2928
2934 bool getKeepDimensions() const noexcept
2935 {
2936 return mImpl->getKeepDimensions();
2937 }
2938
2939protected:
2940 apiv::VReduceLayer* mImpl;
2941 virtual ~IReduceLayer() noexcept = default;
2942};
2943
2956class IPaddingLayer : public ILayer
2957{
2958public:
2968 void setPrePaddingNd(Dims const& padding) noexcept
2969 {
2970 mImpl->setPrePaddingNd(padding);
2971 }
2972
2980 Dims getPrePaddingNd() const noexcept
2981 {
2982 return mImpl->getPrePaddingNd();
2983 }
2984
2994 void setPostPaddingNd(Dims const& padding) noexcept
2995 {
2996 mImpl->setPostPaddingNd(padding);
2997 }
2998
3006 Dims getPostPaddingNd() const noexcept
3007 {
3008 return mImpl->getPostPaddingNd();
3009 }
3010
3011protected:
3012 apiv::VPaddingLayer* mImpl;
3013 virtual ~IPaddingLayer() noexcept = default;
3014};
3015
3022{
3029 int32_t order[Dims::MAX_DIMS];
3030};
3031
3044class IShuffleLayer : public ILayer
3045{
3046public:
3056 void setFirstTranspose(Permutation permutation) noexcept
3057 {
3058 mImpl->setFirstTranspose(permutation);
3059 }
3060
3069 {
3070 return mImpl->getFirstTranspose();
3071 }
3072
3096 void setReshapeDimensions(Dims const& dimensions) noexcept
3097 {
3098 mImpl->setReshapeDimensions(dimensions);
3099 }
3100
3110 {
3111 return mImpl->getReshapeDimensions();
3112 }
3113
3119 //
3142 using ILayer::setInput;
3143
3156 void setSecondTranspose(Permutation permutation) noexcept
3157 {
3158 mImpl->setSecondTranspose(permutation);
3159 }
3160
3169 {
3170 return mImpl->getSecondTranspose();
3171 }
3172
3184 void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
3185 {
3186 return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);
3187 }
3188
3197 bool getZeroIsPlaceholder() const noexcept
3198 {
3199 return mImpl->getZeroIsPlaceholder();
3200 }
3201
3202protected:
3203 apiv::VShuffleLayer* mImpl;
3204 virtual ~IShuffleLayer() noexcept = default;
3205};
3206
3212enum class SampleMode : int32_t
3213{
3214 kSTRICT_BOUNDS = 0,
3215 kWRAP = 1,
3216 kCLAMP = 2,
3217 kFILL = 3,
3218 kREFLECT = 4,
3221};
3222
3228template <>
3229constexpr inline int32_t EnumMax<SampleMode>() noexcept
3230{
3231 return 5;
3232}
3233
3296class ISliceLayer : public ILayer
3297{
3298public:
3308 void setStart(Dims const& start) noexcept
3309 {
3310 mImpl->setStart(start);
3311 }
3312
3323 Dims getStart() const noexcept
3324 {
3325 return mImpl->getStart();
3326 }
3327
3337 void setSize(Dims const& size) noexcept
3338 {
3339 return mImpl->setSize(size);
3340 }
3341
3352 Dims getSize() const noexcept
3353 {
3354 return mImpl->getSize();
3355 }
3356
3366 void setStride(Dims const& stride) noexcept
3367 {
3368 mImpl->setStride(stride);
3369 }
3370
3381 Dims getStride() const noexcept
3382 {
3383 return mImpl->getStride();
3384 }
3385
3391 void setMode(SampleMode mode) noexcept
3392 {
3393 mImpl->setMode(mode);
3394 }
3395
3401 SampleMode getMode() const noexcept
3402 {
3403 return mImpl->getMode();
3404 }
3405
3433 using ILayer::setInput;
3434
3444 void setAxes(Dims const& axes) noexcept
3445 {
3446 mImpl->setAxes(axes);
3447 }
3448
3459 Dims getAxes() const noexcept
3460 {
3461 return mImpl->getAxes();
3462 }
3463
3464protected:
3465 apiv::VSliceLayer* mImpl;
3466 virtual ~ISliceLayer() noexcept = default;
3467};
3468
3481class IShapeLayer : public ILayer
3482{
3483protected:
3484 apiv::VShapeLayer* mImpl;
3485 virtual ~IShapeLayer() noexcept = default;
3486};
3487
3493enum class TopKOperation : int32_t
3494{
3495 kMAX = 0,
3496 kMIN = 1,
3497};
3498
3504template <>
3505constexpr inline int32_t EnumMax<TopKOperation>() noexcept
3506{
3507 return 2;
3508}
3509
3521class ITopKLayer : public ILayer
3522{
3523public:
3529 void setOperation(TopKOperation op) noexcept
3530 {
3531 mImpl->setOperation(op);
3532 }
3533
3540 {
3541 return mImpl->getOperation();
3542 }
3543
3553 void setK(int32_t k) noexcept
3554 {
3555 mImpl->setK(k);
3556 }
3557
3567 int32_t getK() const noexcept
3568 {
3569 return mImpl->getK();
3570 }
3571
3577 void setReduceAxes(uint32_t reduceAxes) noexcept
3578 {
3579 mImpl->setReduceAxes(reduceAxes);
3580 }
3581
3587 uint32_t getReduceAxes() const noexcept
3588 {
3589 return mImpl->getReduceAxes();
3590 }
3591
3606 using ILayer::setInput;
3607
3608protected:
3609 apiv::VTopKLayer* mImpl;
3610 virtual ~ITopKLayer() noexcept = default;
3611};
3612
3619enum class MatrixOperation : int32_t
3620{
3624 kNONE = 0,
3625
3627 kTRANSPOSE = 1,
3628
3639 kVECTOR = 2,
3640};
3641
3647template <>
3648constexpr inline int32_t EnumMax<MatrixOperation>() noexcept
3649{
3650 return 3;
3651}
3652
3679{
3680public:
3689 void setOperation(int32_t index, MatrixOperation op) noexcept
3690 {
3691 mImpl->setOperation(index, op);
3692 }
3693
3701 MatrixOperation getOperation(int32_t index) const noexcept
3702 {
3703 return mImpl->getOperation(index);
3704 }
3705
3706protected:
3707 apiv::VMatrixMultiplyLayer* mImpl;
3708 virtual ~IMatrixMultiplyLayer() noexcept = default;
3709};
3710
3732class INonZeroLayer : public ILayer
3733{
3734protected:
3735 virtual ~INonZeroLayer() noexcept = default;
3736 apiv::VNonZeroLayer* mImpl;
3737};
3738
3754{
3755protected:
3756 apiv::VRaggedSoftMaxLayer* mImpl;
3757 virtual ~IRaggedSoftMaxLayer() noexcept = default;
3758};
3759
3787{
3788protected:
3789 apiv::VIdentityLayer* mImpl;
3790 virtual ~IIdentityLayer() noexcept = default;
3791};
3792
3799class ICastLayer : public ILayer
3800{
3801public:
3809 void setToType(DataType toType) noexcept
3810 {
3811 mImpl->setToType(toType);
3812 }
3813
3820 DataType getToType() const noexcept
3821 {
3822 return mImpl->getToType();
3823 }
3824
3825protected:
3826 apiv::VCastLayer* mImpl;
3827 virtual ~ICastLayer() noexcept = default;
3828};
3829
3839{
3840public:
3849 void setWeights(Weights weights) noexcept
3850 {
3851 mImpl->setWeights(weights);
3852 }
3853
3859 Weights getWeights() const noexcept
3860 {
3861 return mImpl->getWeights();
3862 }
3863
3871 void setDimensions(Dims const& dimensions) noexcept
3872 {
3873 mImpl->setDimensions(dimensions);
3874 }
3875
3883 Dims getDimensions() const noexcept
3884 {
3885 return mImpl->getDimensions();
3886 }
3887
3888protected:
3889 apiv::VConstantLayer* mImpl;
3890 virtual ~IConstantLayer() noexcept = default;
3891};
3892
3903{
3904protected:
3905 apiv::VParametricReLULayer* mImpl;
3906 virtual ~IParametricReLULayer() noexcept = default;
3907};
3908
3914enum class InterpolationMode : int32_t
3915{
3916 kNEAREST = 0,
3917 kLINEAR = 1,
3918 kCUBIC = 2
3919};
3920
3921namespace impl
3922{
3928template <>
3930{
3931 static constexpr int32_t kVALUE = 3;
3932};
3933} // namespace impl
3934
3943{
3956 kALIGN_CORNERS = 0,
3957
3964 kASYMMETRIC = 1,
3965
3972 kHALF_PIXEL = 2,
3973};
3974
3975namespace impl
3976{
3982template <>
3984{
3985 static constexpr int32_t kVALUE = 3;
3986};
3987} // namespace impl
3988
3996enum class ResizeSelector : int32_t
3997{
3999 kFORMULA = 0,
4000
4002 kUPPER = 1,
4003};
4004
4005namespace impl
4006{
4012template <>
4014{
4015 static constexpr int32_t kVALUE = 2;
4016};
4017} // namespace impl
4018
4026enum class ResizeRoundMode : int32_t
4027{
4029 kHALF_UP = 0,
4030
4032 kHALF_DOWN = 1,
4033
4035 kFLOOR = 2,
4036
4038 kCEIL = 3,
4039};
4040
4041namespace impl
4042{
4048template <>
4050{
4051 static constexpr int32_t kVALUE = 4;
4052};
4053} // namespace impl
4054
4091class IResizeLayer : public ILayer
4092{
4093public:
4112 void setOutputDimensions(Dims const& dimensions) noexcept
4113 {
4114 return mImpl->setOutputDimensions(dimensions);
4115 }
4116
4122 Dims getOutputDimensions() const noexcept
4123 {
4124 return mImpl->getOutputDimensions();
4125 }
4126
4152 void setScales(float const* scales, int32_t nbScales) noexcept
4153 {
4154 mImpl->setScales(scales, nbScales);
4155 }
4156
4171 int32_t getScales(int32_t size, float* scales) const noexcept
4172 {
4173 return mImpl->getScales(size, scales);
4174 }
4175
4183 void setResizeMode(InterpolationMode interpolationMode) noexcept
4184 {
4185 mImpl->setResizeMode(interpolationMode);
4186 }
4187
4194 {
4195 return mImpl->getResizeMode();
4196 }
4197
4217 using ILayer::setInput;
4218
4229 {
4230 mImpl->setCoordinateTransformation(coordTransform);
4231 }
4232
4239 {
4240 return mImpl->getCoordinateTransformation();
4241 }
4242
4254 {
4255 mImpl->setSelectorForSinglePixel(selector);
4256 }
4257
4264 {
4265 return mImpl->getSelectorForSinglePixel();
4266 }
4267
4278 {
4279 mImpl->setNearestRounding(value);
4280 }
4281
4288 {
4289 return mImpl->getNearestRounding();
4290 }
4291
4309 void setCubicCoeff(float A) noexcept
4310 {
4311 mImpl->setCubicCoeff(A);
4312 }
4313
4319 float getCubicCoeff() const noexcept
4320 {
4321 return mImpl->getCubicCoeff();
4322 }
4323
4332 void setExcludeOutside(bool excludeFlag) noexcept
4333 {
4334 mImpl->setExcludeOutside(excludeFlag);
4335 }
4336
4342 bool getExcludeOutside() const noexcept
4343 {
4344 return mImpl->getExcludeOutside();
4345 }
4346
4347protected:
4348 virtual ~IResizeLayer() noexcept = default;
4349 apiv::VResizeLayer* mImpl;
4350};
4351
4355enum class LoopOutput : int32_t
4356{
4358 kLAST_VALUE = 0,
4359
4361 kCONCATENATE = 1,
4362
4364 kREVERSE = 2
4365};
4366
4372template <>
4373constexpr inline int32_t EnumMax<LoopOutput>() noexcept
4374{
4375 return 3;
4376}
4377
4381enum class TripLimit : int32_t
4382{
4383
4384 kCOUNT = 0,
4385 kWHILE = 1
4386};
4387
4393template <>
4394constexpr inline int32_t EnumMax<TripLimit>() noexcept
4395{
4396 return 2;
4397}
4398
4399class ILoop;
4400
4415{
4416public:
4420 ILoop* getLoop() const noexcept
4421 {
4422 return mBoundary->getLoop();
4423 }
4424
4425protected:
4426 virtual ~ILoopBoundaryLayer() noexcept = default;
4427 apiv::VLoopBoundaryLayer* mBoundary;
4428};
4429
4438{
4439public:
4444 {
4445 return mBoundary->getConditional();
4446 }
4447
4448protected:
4449 virtual ~IIfConditionalBoundaryLayer() noexcept = default;
4450 apiv::VConditionalBoundaryLayer* mBoundary;
4451};
4452
4459{
4460public:
4461protected:
4462 virtual ~IConditionLayer() noexcept = default;
4463 apiv::VConditionLayer* mImpl;
4464};
4465
4476{
4477public:
4478protected:
4479 virtual ~IIfConditionalOutputLayer() noexcept = default;
4480 apiv::VConditionalOutputLayer* mImpl;
4481};
4482
4489{
4490public:
4491protected:
4492 virtual ~IIfConditionalInputLayer() noexcept = default;
4493 apiv::VConditionalInputLayer* mImpl;
4494};
4495
4520{
4521public:
4532 {
4533 return mImpl->setCondition(condition);
4534 }
4535
4549 IIfConditionalOutputLayer* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput) noexcept
4550 {
4551 return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);
4552 }
4553
4562 {
4563 return mImpl->addInput(input);
4564 }
4565
4576 void setName(char const* name) noexcept
4577 {
4578 mImpl->setName(name);
4579 }
4580
4586 char const* getName() const noexcept
4587 {
4588 return mImpl->getName();
4589 }
4590
4591protected:
4592 virtual ~IIfConditional() noexcept = default;
4593 apiv::VIfConditional* mImpl;
4594};
4595
4604{
4605public:
4611 //
4624 using ILayer::setInput;
4625
4626protected:
4627 virtual ~IRecurrenceLayer() noexcept = default;
4628 apiv::VRecurrenceLayer* mImpl;
4629};
4630
4651{
4652public:
4656 LoopOutput getLoopOutput() const noexcept
4657 {
4658 return mImpl->getLoopOutput();
4659 }
4660
4673 void setAxis(int32_t axis) noexcept
4674 {
4675 mImpl->setAxis(axis);
4676 }
4677
4681 int32_t getAxis() const noexcept
4682 {
4683 return mImpl->getAxis();
4684 }
4685
4691 //
4706 using ILayer::setInput;
4707
4708protected:
4709 virtual ~ILoopOutputLayer() noexcept = default;
4710 apiv::VLoopOutputLayer* mImpl;
4711};
4712
4725{
4726public:
4730 TripLimit getTripLimit() const noexcept
4731 {
4732 return mImpl->getTripLimit();
4733 }
4734
4735protected:
4736 virtual ~ITripLimitLayer() noexcept = default;
4737 apiv::VTripLimitLayer* mImpl;
4738};
4739
4751{
4752public:
4756 void setAxis(int32_t axis) noexcept
4757 {
4758 mImpl->setAxis(axis);
4759 }
4760
4764 int32_t getAxis() const noexcept
4765 {
4766 return mImpl->getAxis();
4767 }
4768
4778 void setReverse(bool reverse) noexcept
4779 {
4780 mImpl->setReverse(reverse);
4781 }
4782
4788 bool getReverse() const noexcept
4789 {
4790 return mImpl->getReverse();
4791 }
4792
4793protected:
4794 virtual ~IIteratorLayer() noexcept = default;
4795 apiv::VIteratorLayer* mImpl;
4796};
4797
4807class ILoop : public INoCopy
4808{
4809public:
4816 IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept
4817 {
4818 return mImpl->addRecurrence(initialValue);
4819 }
4820
4838 {
4839 return mImpl->addTripLimit(tensor, limit);
4840 }
4841
4850 IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept
4851 {
4852 return mImpl->addIterator(tensor, axis, reverse);
4853 }
4854
4863 ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept
4864 {
4865 return mImpl->addLoopOutput(tensor, outputKind, axis);
4866 }
4867
4878 void setName(char const* name) noexcept
4879 {
4880 mImpl->setName(name);
4881 }
4882
4888 char const* getName() const noexcept
4889 {
4890 return mImpl->getName();
4891 }
4892
4893protected:
4894 virtual ~ILoop() noexcept = default;
4895 apiv::VLoop* mImpl;
4896};
4897
4910class ISelectLayer : public ILayer
4911{
4912protected:
4913 virtual ~ISelectLayer() noexcept = default;
4914 apiv::VSelectLayer* mImpl;
4915};
4916
4933{
4934public:
4943 void setMessage(char const* message) noexcept
4944 {
4945 mImpl->setMessage(message);
4946 }
4947
4953 char const* getMessage() const noexcept
4954 {
4955 return mImpl->getMessage();
4956 }
4957
4958protected:
4959 virtual ~IAssertionLayer() noexcept = default;
4960
4961 apiv::VAssertionLayer* mImpl;
4962};
4963
4971enum class FillOperation : int32_t
4972{
4988 kLINSPACE = 0,
4989
4991 kRANDOM_UNIFORM = 1,
4992
4994 kRANDOM_NORMAL = 2
4995};
4996
5002template <>
5003constexpr inline int32_t EnumMax<FillOperation>() noexcept
5004{
5005 return 3;
5006}
5007
5043class IFillLayer : public ILayer
5044{
5045public:
5054 //
5055 void setDimensions(Dims const& dimensions) noexcept
5056 {
5057 mImpl->setDimensions(dimensions);
5058 }
5059
5070 Dims getDimensions() const noexcept
5071 {
5072 return mImpl->getDimensions();
5073 }
5074
5080 void setOperation(FillOperation op) noexcept
5081 {
5082 mImpl->setOperation(op);
5083 }
5084
5091 {
5092 return mImpl->getOperation();
5093 }
5094
5108 //
5109 void setAlpha(double alpha) noexcept
5110 {
5111 mImpl->setAlpha(alpha);
5112 }
5113
5124 double getAlpha() const noexcept
5125 {
5126 return mImpl->getAlpha();
5127 }
5128
5143 void setBeta(double beta) noexcept
5144 {
5145 mImpl->setBeta(beta);
5146 }
5147
5158 double getBeta() const noexcept
5159 {
5160 return mImpl->getBeta();
5161 }
5162
5203 using ILayer::setInput;
5204
5218 //
5219 void setAlphaInt64(int64_t alpha) noexcept
5220 {
5221 mImpl->setAlphaInt64(alpha);
5222 }
5223
5234 int64_t getAlphaInt64() const noexcept
5235 {
5236 return mImpl->getAlphaInt64();
5237 }
5238
5253 void setBetaInt64(int64_t beta) noexcept
5254 {
5255 mImpl->setBetaInt64(beta);
5256 }
5257
5268 int64_t getBetaInt64() const noexcept
5269 {
5270 return mImpl->getBetaInt64();
5271 }
5272
5276 bool isAlphaBetaInt64() const noexcept
5277 {
5278 return mImpl->isAlphaBetaInt64();
5279 }
5280
5293 void setToType(DataType toType) noexcept
5294 {
5295 mImpl->setToType(toType);
5296 }
5297
5305 DataType getToType() const noexcept
5306 {
5307 return mImpl->getToType();
5308 }
5309
5310protected:
5311 virtual ~IFillLayer() noexcept = default;
5312 apiv::VFillLayer* mImpl;
5313};
5314
5389{
5390public:
5399 int32_t getAxis() const noexcept
5400 {
5401 return mImpl->getAxis();
5402 }
5410 void setAxis(int32_t axis) noexcept
5411 {
5412 mImpl->setAxis(axis);
5413 }
5414
5426 void setToType(DataType toType) noexcept
5427 {
5428 mImpl->setToType(toType);
5429 }
5430
5438 DataType getToType() const noexcept
5439 {
5440 return mImpl->getToType();
5441 }
5442
5443protected:
5444 virtual ~IQuantizeLayer() noexcept = default;
5445 apiv::VQuantizeLayer* mImpl;
5446};
5447
5519{
5520public:
5529 int32_t getAxis() const noexcept
5530 {
5531 return mImpl->getAxis();
5532 }
5540 void setAxis(int32_t axis) noexcept
5541 {
5542 mImpl->setAxis(axis);
5543 }
5544
5556 void setToType(DataType toType) noexcept
5557 {
5558 mImpl->setToType(toType);
5559 }
5560
5568 DataType getToType() const noexcept
5569 {
5570 return mImpl->getToType();
5571 }
5572
5573protected:
5574 virtual ~IDequantizeLayer() noexcept = default;
5575 apiv::VDequantizeLayer* mImpl;
5576};
5577
5578
5615class IEinsumLayer : public ILayer
5616{
5617public:
5627 bool setEquation(char const* equation) noexcept
5628 {
5629 return mImpl->setEquation(equation);
5630 }
5631
5637 char const* getEquation() const noexcept
5638 {
5639 return mImpl->getEquation();
5640 }
5641
5642protected:
5643 virtual ~IEinsumLayer() noexcept = default;
5644 apiv::VEinsumLayer* mImpl;
5645};
5646
5654enum class ScatterMode : int32_t
5655{
5656 kELEMENT = 0,
5657 kND = 1,
5658};
5659
5665template <>
5666constexpr inline int32_t EnumMax<ScatterMode>() noexcept
5667{
5668 return 2;
5669}
5670
5728class IScatterLayer : public ILayer
5729{
5730public:
5736 void setMode(ScatterMode mode) noexcept
5737 {
5738 mImpl->setMode(mode);
5739 }
5740
5746 ScatterMode getMode() const noexcept
5747 {
5748 return mImpl->getMode();
5749 }
5750
5756 void setAxis(int32_t axis) noexcept
5757 {
5758 mImpl->setAxis(axis);
5759 }
5760
5764 int32_t getAxis() const noexcept
5765 {
5766 return mImpl->getAxis();
5767 }
5768
5769protected:
5770 apiv::VScatterLayer* mImpl;
5771 virtual ~IScatterLayer() noexcept = default;
5772}; // class IScatterLayer
5773
5800class IOneHotLayer : public ILayer
5801{
5802public:
5808 void setAxis(int32_t axis) noexcept
5809 {
5810 mImpl->setAxis(axis);
5811 }
5812
5816 int32_t getAxis() const noexcept
5817 {
5818 return mImpl->getAxis();
5819 }
5820
5821protected:
5822 apiv::VOneHotLayer* mImpl;
5823};
5824
5837{
5838public:
5845 {
5846 mImpl->setInterpolationMode(mode);
5847 }
5848
5857 {
5858 return mImpl->getInterpolationMode();
5859 }
5860
5866 void setAlignCorners(bool alignCorners) noexcept
5867 {
5868 mImpl->setAlignCorners(alignCorners);
5869 }
5870
5878 bool getAlignCorners() const noexcept
5879 {
5880 return mImpl->getAlignCorners();
5881 }
5882
5890 bool setSampleMode(SampleMode mode) noexcept
5891 {
5892 return mImpl->setSampleMode(mode);
5893 }
5894
5902 SampleMode getSampleMode() const noexcept
5903 {
5904 return mImpl->getSampleMode();
5905 }
5906
5907protected:
5908 apiv::VGridSampleLayer* mImpl;
5909 virtual ~IGridSampleLayer() noexcept = default;
5910}; // class IGridSampleLayer
5911
5919enum class BoundingBoxFormat : int32_t
5920{
5922 kCORNER_PAIRS = 0,
5924 kCENTER_SIZES = 1
5925};
5926
5932template <>
5933constexpr inline int32_t EnumMax<BoundingBoxFormat>() noexcept
5934{
5935 return 2;
5936}
5937
5984class INMSLayer : public ILayer
5985{
5986public:
5997 {
5998 mImpl->setBoundingBoxFormat(fmt);
5999 }
6000
6009 {
6010 return mImpl->getBoundingBoxFormat();
6011 }
6012
6022 void setTopKBoxLimit(int32_t limit) noexcept
6023 {
6024 mImpl->setTopKBoxLimit(limit);
6025 }
6026
6032 int32_t getTopKBoxLimit() const noexcept
6033 {
6034 return mImpl->getTopKBoxLimit();
6035 }
6036
6055 using ILayer::setInput;
6056
6057protected:
6058 apiv::VNMSLayer* mImpl;
6059 virtual ~INMSLayer() noexcept = default;
6060}; // class INMSLayer
6061
6075{
6076public:
6085 void setBatchAxis(int32_t batchAxis) noexcept
6086 {
6087 mImpl->setBatchAxis(batchAxis);
6088 }
6089
6095 int32_t getBatchAxis() const noexcept
6096 {
6097 return mImpl->getBatchAxis();
6098 }
6099
6108 void setSequenceAxis(int32_t sequenceAxis) noexcept
6109 {
6110 mImpl->setSequenceAxis(sequenceAxis);
6111 }
6112
6118 int32_t getSequenceAxis() const noexcept
6119 {
6120 return mImpl->getSequenceAxis();
6121 }
6122
6123protected:
6124 apiv::VReverseSequenceLayer* mImpl;
6125 virtual ~IReverseSequenceLayer() noexcept = default;
6126}; // class IReverseSequenceLayer
6127
6147{
6148public:
6156 void setEpsilon(float eps) noexcept
6157 {
6158 return mImpl->setEpsilon(eps);
6159 }
6160
6166 float getEpsilon() const noexcept
6167 {
6168 return mImpl->getEpsilon();
6169 }
6170
6176 void setAxes(uint32_t axesMask) noexcept
6177 {
6178 return mImpl->setAxes(axesMask);
6179 }
6180
6186 uint32_t getAxes() const noexcept
6187 {
6188 return mImpl->getAxes();
6189 }
6190
6207 void setNbGroups(int64_t nbGroups) noexcept
6208 {
6209 return mImpl->setNbGroups(nbGroups);
6210 }
6211
6217 int64_t getNbGroups() const noexcept
6218 {
6219 return mImpl->getNbGroups();
6220 }
6221
6243 void setComputePrecision(DataType type) noexcept
6244 {
6245 return mImpl->setComputePrecision(type);
6246 }
6247
6254 {
6255 return mImpl->getComputePrecision();
6256 }
6257
6258protected:
6259 apiv::VNormalizationLayer* mImpl;
6260 virtual ~INormalizationLayer() noexcept = default;
6261};
6262
6281{
6282public:
6283 virtual ~INetworkDefinition() noexcept = default;
6284
6321 ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept
6322 {
6323 return mImpl->addInput(name, type, dimensions);
6324 }
6325
6335 void markOutput(ITensor& tensor) noexcept
6336 {
6337 mImpl->markOutput(tensor);
6338 }
6339
6353 bool markDebug(ITensor& tensor) noexcept
6354 {
6355 return mImpl->markDebug(tensor);
6356 }
6357
6369 bool unmarkDebug(ITensor& tensor) noexcept
6370 {
6371 return mImpl->unmarkDebug(tensor);
6372 }
6373
6379 bool isDebugTensor(nvinfer1::ITensor const& tensor) const noexcept
6380 {
6381 return mImpl->isDebugTensor(tensor);
6382 }
6383
6400 {
6401 return mImpl->addActivation(input, type);
6402 }
6403
6418 ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept
6419 {
6420 return mImpl->addLRN(input, window, alpha, beta, k);
6421 }
6422
6444 IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
6445 {
6446 return mImpl->addScale(input, mode, shift, scale, power);
6447 }
6448
6458 {
6459 return mImpl->addSoftMax(input);
6460 }
6461
6474 IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept
6475 {
6476 return mImpl->addConcatenation(inputs, nbInputs);
6477 }
6478
6502 {
6503 return mImpl->addElementWise(input1, input2, op);
6504 }
6505
6523 IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept
6524 {
6525 return mImpl->addUnary(input, operation);
6526 }
6527
6538 {
6539 return mImpl->addShuffle(input);
6540 }
6541
6554 IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept
6555 {
6556 return mImpl->addOneHot(indices, values, depth, axis);
6557 }
6558
6566 int32_t getNbLayers() const noexcept
6567 {
6568 return mImpl->getNbLayers();
6569 }
6570
6580 ILayer* getLayer(int32_t index) const noexcept
6581 {
6582 return mImpl->getLayer(index);
6583 }
6584
6592 int32_t getNbInputs() const noexcept
6593 {
6594 return mImpl->getNbInputs();
6595 }
6596
6608 ITensor* getInput(int32_t index) const noexcept
6609 {
6610 return mImpl->getInput(index);
6611 }
6612
6622 int32_t getNbOutputs() const noexcept
6623 {
6624 return mImpl->getNbOutputs();
6625 }
6626
6638 ITensor* getOutput(int32_t index) const noexcept
6639 {
6640 return mImpl->getOutput(index);
6641 }
6642
6665 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
6666 {
6667 return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);
6668 }
6669
6697 ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
6698 {
6699 return mImpl->addTopK(input, op, k, reduceAxes);
6700 }
6701
6713 IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept
6714 {
6715 return mImpl->addGather(data, indices, axis);
6716 }
6717
6729 IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept
6730 {
6731 return mImpl->addGatherV2(data, indices, mode);
6732 }
6733
6749 {
6750 return mImpl->addRaggedSoftMax(input, bounds);
6751 }
6752
6770 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
6771 {
6772 return mImpl->addMatrixMultiply(input0, op0, input1, op1);
6773 }
6774
6785 {
6786 return mImpl->addNonZero(input);
6787 }
6788
6808 IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept
6809 {
6810 return mImpl->addConstant(dimensions, weights);
6811 }
6812
6823 {
6824 return mImpl->addIdentity(input);
6825 }
6826
6837 ICastLayer* addCast(ITensor& input, DataType toType) noexcept
6838 {
6839 return mImpl->addCast(input, toType);
6840 }
6841
6852 void removeTensor(ITensor& tensor) noexcept
6853 {
6854 mImpl->removeTensor(tensor);
6855 }
6856
6864 void unmarkOutput(ITensor& tensor) noexcept
6865 {
6866 mImpl->unmarkOutput(tensor);
6867 }
6868
6883 IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept
6884 {
6885 return mImpl->addPluginV2(inputs, nbInputs, plugin);
6886 }
6887
6901 IPluginV3Layer* addPluginV3(ITensor* const* inputs, int32_t nbInputs, ITensor* const* shapeInputs,
6902 int32_t nbShapeInputs, IPluginV3& plugin) noexcept
6903 {
6904 return mImpl->addPluginV3(inputs, nbInputs, shapeInputs, nbShapeInputs, plugin);
6905 }
6906
6921 ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept
6922 {
6923 return mImpl->addSlice(input, start, size, stride);
6924 }
6925
6945 void setName(char const* name) noexcept
6946 {
6947 mImpl->setName(name);
6948 }
6949
6959 char const* getName() const noexcept
6960 {
6961 return mImpl->getName();
6962 }
6963
6975 IShapeLayer* addShape(ITensor& input) noexcept
6976 {
6977 return mImpl->addShape(input);
6978 }
6979
6990 {
6991 return mImpl->hasImplicitBatchDimension();
6992 }
6993
7000 {
7001 return mImpl->getFlags();
7002 }
7003
7011 bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept
7012 {
7013 return mImpl->getFlag(networkDefinitionCreationFlag);
7014 }
7015
7028 bool markOutputForShapes(ITensor& tensor) noexcept
7029 {
7030 return mImpl->markOutputForShapes(tensor);
7031 }
7032
7040 bool unmarkOutputForShapes(ITensor& tensor) noexcept
7041 {
7042 return mImpl->unmarkOutputForShapes(tensor);
7043 }
7044
7059 {
7060 return mImpl->addParametricReLU(input, slope);
7061 }
7062
7081 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7082 {
7083 return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7084 }
7085
7100 IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept
7101 {
7102 return mImpl->addPoolingNd(input, type, windowSize);
7103 }
7104
7119 //
7123 ITensor& input, int64_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7124 {
7125 return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7126 }
7127
7160 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
7161 {
7162 return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);
7163 }
7164
7177 {
7178 return mImpl->addResize(input);
7179 }
7180
7190 ILoop* addLoop() noexcept
7191 {
7192 return mImpl->addLoop();
7193 }
7194
7206 {
7207 return mImpl->addIfConditional();
7208 }
7209
7244 ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept
7245 {
7246 return mImpl->addSelect(condition, thenInput, elseInput);
7247 }
7248
7261 IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept
7262 {
7263 return mImpl->addAssertion(condition, message);
7264 }
7265
7286 TRT_DEPRECATED IFillLayer* addFill(Dims const& dimensions, FillOperation op) noexcept
7287 {
7288 return mImpl->addFill(dimensions, op);
7289 }
7290
7312 IFillLayer* addFill(Dims const& dimensions, FillOperation op, DataType outputType) noexcept
7313 {
7314 return mImpl->addFillV2(dimensions, op, outputType);
7315 }
7316
7328 IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept
7329 {
7330 return mImpl->addPaddingNd(input, prePadding, postPadding);
7331 }
7332
7352 bool setWeightsName(Weights weights, char const* name) noexcept
7353 {
7354 return mImpl->setWeightsName(weights, name);
7355 }
7356
7368 //
7371 void setErrorRecorder(IErrorRecorder* recorder) noexcept
7372 {
7373 mImpl->setErrorRecorder(recorder);
7374 }
7375
7387 {
7388 return mImpl->getErrorRecorder();
7389 }
7390
7408 {
7409 return mImpl->addDequantize(input, scale);
7410 }
7411
7428 IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale, DataType outputType) noexcept
7429 {
7430 return mImpl->addDequantizeV2(input, scale, outputType);
7431 }
7432
7448 IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept
7449 {
7450 return mImpl->addScatter(data, indices, updates, mode);
7451 }
7452
7470 {
7471 return mImpl->addQuantize(input, scale);
7472 }
7473
7490 IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale, DataType outputType) noexcept
7491 {
7492 return mImpl->addQuantizeV2(input, scale, outputType);
7493 }
7494
7495
7506 IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept
7507 {
7508 return mImpl->addEinsum(inputs, nbInputs, equation);
7509 }
7510
7525 {
7526 return mImpl->addGridSample(input, grid);
7527 }
7528
7542 INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept
7543 {
7544 return mImpl->addNMS(boxes, scores, maxOutputBoxesPerClass);
7545 }
7546
7560 {
7561 return mImpl->addReverseSequence(input, sequenceLens);
7562 }
7563
7585 INormalizationLayer* addNormalization(ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept
7586 {
7587 return mImpl->addNormalization(input, scale, bias, axesMask);
7588 }
7589
7596 virtual IBuilder& getBuilder() const noexcept
7597 {
7598 return mImpl->getBuilder();
7599 }
7600
7609 bool markWeightsRefittable(char const* name) noexcept
7610 {
7611 return mImpl->markWeightsRefittable(name);
7612 }
7613
7621 bool unmarkWeightsRefittable(char const* name) noexcept
7622 {
7623 return mImpl->unmarkWeightsRefittable(name);
7624 }
7625
7634 bool areWeightsMarkedRefittable(char const* name) const noexcept
7635 {
7636 return mImpl->areWeightsMarkedRefittable(name);
7637 }
7638
7639protected:
7640 apiv::VNetworkDefinition* mImpl;
7641};
7642
7650enum class CalibrationAlgoType : int32_t
7651{
7656};
7657
7663template <>
7664constexpr inline int32_t EnumMax<CalibrationAlgoType>() noexcept
7665{
7666 return 4;
7667}
7668
7683{
7684public:
7692 TRT_DEPRECATED virtual int32_t getBatchSize() const noexcept = 0;
7693
7708 virtual bool getBatch(void* bindings[], char const* names[], int32_t nbBindings) noexcept = 0;
7709
7724 virtual void const* readCalibrationCache(std::size_t& length) noexcept = 0;
7725
7734 virtual void writeCalibrationCache(void const* ptr, std::size_t length) noexcept = 0;
7735
7741 virtual CalibrationAlgoType getAlgorithm() noexcept = 0;
7742
7743 ~IInt8Calibrator() noexcept override = default;
7744};
7745
7746namespace v_1_0
7747{
7749{
7750public:
7754 InterfaceInfo getInterfaceInfo() const noexcept override
7755 {
7756 return InterfaceInfo{"IInt8EntropyCalibrator", 1, 0};
7757 }
7758
7763 {
7765 }
7766
7767 ~IInt8EntropyCalibrator() noexcept override = default;
7768};
7769} // namespace v_1_0
7770
7785
7786namespace v_1_0
7787{
7789{
7790public:
7794 InterfaceInfo getInterfaceInfo() const noexcept override
7795 {
7796 return InterfaceInfo{"IInt8EntropyCalibrator2", 1, 0};
7797 }
7798
7803 {
7805 }
7806
7807 ~IInt8EntropyCalibrator2() noexcept override = default;
7808};
7809} // namespace v_1_0
7810
7825
7826namespace v_1_0
7827{
7829{
7830public:
7834 InterfaceInfo getInterfaceInfo() const noexcept override
7835 {
7836 return InterfaceInfo{"IInt8MinMaxCalibrator", 1, 0};
7837 }
7838
7843 {
7845 }
7846
7847 ~IInt8MinMaxCalibrator() noexcept override = default;
7848};
7849} // namespace v_1_0
7850
7864
7865namespace v_1_0
7866{
7868{
7869public:
7873 InterfaceInfo getInterfaceInfo() const noexcept override
7874 {
7875 return InterfaceInfo{"IInt8Calibrator", 1, 0};
7876 }
7877
7882 {
7884 }
7885
7892 virtual double getQuantile() const noexcept = 0;
7893
7900 virtual double getRegressionCutoff() const noexcept = 0;
7901
7914 virtual void const* readHistogramCache(std::size_t& length) noexcept = 0;
7915
7924 virtual void writeHistogramCache(void const* ptr, std::size_t length) noexcept = 0;
7925
7926 ~IInt8LegacyCalibrator() noexcept override = default;
7927};
7928} // namespace v_1_0
7929
7944
7956{
7957public:
7963 DataType getDataType() const noexcept
7964 {
7965 return mImpl->getDataType();
7966 }
7967
7974 Dims getStrides() const noexcept
7975 {
7976 return mImpl->getStrides();
7977 }
7978
7984 int64_t getVectorizedDim() const noexcept
7985 {
7986 return mImpl->getVectorizedDim();
7987 }
7988
7995 int64_t getComponentsPerElement() const noexcept
7996 {
7997 return mImpl->getComponentsPerElement();
7998 }
7999
8000protected:
8001 virtual ~IAlgorithmIOInfo() noexcept = default;
8002 apiv::VAlgorithmIOInfo* mImpl;
8003};
8004
8017{
8018public:
8022 int64_t getImplementation() const noexcept
8023 {
8024 return mImpl->getImplementation();
8025 }
8026
8030 int64_t getTactic() const noexcept
8031 {
8032 return mImpl->getTactic();
8033 }
8034
8035protected:
8036 virtual ~IAlgorithmVariant() noexcept = default;
8037 apiv::VAlgorithmVariant* mImpl;
8038};
8039
8049{
8050public:
8056 char const* getName() const noexcept
8057 {
8058 return mImpl->getName();
8059 }
8060
8068 Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
8069 {
8070 return mImpl->getDimensions(index, select);
8071 }
8072
8076 int32_t getNbInputs() const noexcept
8077 {
8078 return mImpl->getNbInputs();
8079 }
8080
8084 int32_t getNbOutputs() const noexcept
8085 {
8086 return mImpl->getNbOutputs();
8087 }
8088
8089protected:
8090 virtual ~IAlgorithmContext() noexcept = default;
8091 apiv::VAlgorithmContext* mImpl;
8092};
8093
8105class IAlgorithm : public INoCopy
8106{
8107public:
8112 {
8113 return mImpl->getAlgorithmVariant();
8114 }
8115
8119 float getTimingMSec() const noexcept
8120 {
8121 return mImpl->getTimingMSec();
8122 }
8123
8127 std::size_t getWorkspaceSize() const noexcept
8128 {
8129 return mImpl->getWorkspaceSize();
8130 }
8131
8141 IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept
8142 {
8143 return mImpl->getAlgorithmIOInfoByIndex(index);
8144 }
8145
8146protected:
8147 virtual ~IAlgorithm() noexcept = default;
8148 apiv::VAlgorithm* mImpl;
8149}; // IAlgorithm
8150
8151namespace v_1_0
8152{
8154{
8155public:
8159 InterfaceInfo getInterfaceInfo() const noexcept override
8160 {
8161 return InterfaceInfo{"IAlgorithmSelector", 1, 0};
8162 }
8177 virtual int32_t selectAlgorithms(IAlgorithmContext const& context, IAlgorithm const* const* choices,
8178 int32_t nbChoices, int32_t* selection) noexcept = 0;
8179
8190 virtual void reportAlgorithms(IAlgorithmContext const* const* algoContexts, IAlgorithm const* const* algoChoices,
8191 int32_t nbAlgorithms) noexcept = 0;
8192
8193 virtual ~IAlgorithmSelector() noexcept = default;
8194};
8195} // namespace v_1_0
8196
8208
8215using QuantizationFlags = uint32_t;
8216
8226enum class QuantizationFlag : int32_t
8227{
8232};
8233
8239template <>
8240constexpr inline int32_t EnumMax<QuantizationFlag>() noexcept
8241{
8242 return 1;
8243}
8244
8262enum class RuntimePlatform : int32_t
8263{
8266 kSAME_AS_BUILD = 0,
8267
8270 kWINDOWS_AMD64 = 1,
8271};
8272
8273namespace impl
8274{
8280template <>
8282{
8283 static constexpr int32_t kVALUE = 2;
8284};
8285} // namespace impl
8286
8293using BuilderFlags = uint32_t;
8294
8302enum class BuilderFlag : int32_t
8303{
8305 kFP16 = 0,
8306
8308 kINT8 = 1,
8309
8311 kDEBUG = 2,
8312
8314 kGPU_FALLBACK = 3,
8315
8317 kREFIT = 4,
8318
8321
8325 kTF32 = 6,
8326
8328 kSPARSE_WEIGHTS = 7,
8329
8336 kSAFETY_SCOPE = 8,
8337
8340
8344
8348 kDIRECT_IO = 11,
8349
8352
8359
8365
8371 kFP8 = 15,
8372
8376
8379 kBF16 = 17,
8380
8386
8388 kSTRIP_PLAN = 19,
8389
8392
8399 kREFIT_IDENTICAL = 20,
8400
8426 kWEIGHT_STREAMING = 21,
8427
8429 kINT4 = 22,
8430
8435 kREFIT_INDIVIDUAL = 23,
8436
8445 kSTRICT_NANS = 24,
8446
8448 kMONITOR_MEMORY = 25,
8449
8450};
8451
8457template <>
8458constexpr inline int32_t EnumMax<BuilderFlag>() noexcept
8459{
8460 return 26;
8461}
8462
8476class ITimingCache : public INoCopy
8477{
8478public:
8479 virtual ~ITimingCache() noexcept = default;
8480
8490 nvinfer1::IHostMemory* serialize() const noexcept
8491 {
8492 return mImpl->serialize();
8493 }
8494
8514 bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept
8515 {
8516 return mImpl->combine(inputCache, ignoreMismatch);
8517 }
8518
8524 bool reset() noexcept
8525 {
8526 return mImpl->reset();
8527 }
8528
8529protected:
8530 apiv::VTimingCache* mImpl;
8531};
8532
8540enum class MemoryPoolType : int32_t
8541{
8548 kWORKSPACE = 0,
8549
8557
8563 kDLA_LOCAL_DRAM = 2,
8564
8570 kDLA_GLOBAL_DRAM = 3,
8571
8579 kTACTIC_DRAM = 4,
8580
8594};
8595
8601template <>
8602constexpr inline int32_t EnumMax<MemoryPoolType>() noexcept
8603{
8604 return 6;
8605}
8606
8615enum class PreviewFeature : int32_t
8616{
8623
8628};
8629
8630namespace impl
8631{
8637template <>
8639{
8640 static constexpr int32_t kVALUE = 2;
8641};
8642} // namespace impl
8643
8654enum class HardwareCompatibilityLevel : int32_t
8655{
8658 kNONE = 0,
8659
8669 kAMPERE_PLUS = 1,
8670};
8671
8672namespace impl
8673{
8679template <>
8681{
8682 static constexpr int32_t kVALUE = 2;
8683};
8684} // namespace impl
8685
8686namespace v_1_0
8687{
8689{
8690public:
8691 IProgressMonitor() = default;
8692 virtual ~IProgressMonitor() noexcept = default;
8693
8697 InterfaceInfo getInterfaceInfo() const noexcept override
8698 {
8699 return InterfaceInfo{"IProgressMonitor", 1, 0};
8700 }
8701
8721 virtual void phaseStart(char const* phaseName, char const* parentPhase, int32_t nbSteps) noexcept = 0;
8722
8735 virtual bool stepComplete(char const* phaseName, int32_t step) noexcept = 0;
8736
8748 virtual void phaseFinish(char const* phaseName) noexcept = 0;
8749
8750}; // class IProgressMonitor
8751} // namespace v_1_0
8752
8773
8782{
8783public:
8784 virtual ~IBuilderConfig() noexcept = default;
8785
8794 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
8795 {
8796 mImpl->setAvgTimingIterations(avgTiming);
8797 }
8798
8806 int32_t getAvgTimingIterations() const noexcept
8807 {
8808 return mImpl->getAvgTimingIterations();
8809 }
8810
8819 void setEngineCapability(EngineCapability capability) noexcept
8820 {
8821 mImpl->setEngineCapability(capability);
8822 }
8823
8832 {
8833 return mImpl->getEngineCapability();
8834 }
8835
8844 {
8845 mImpl->setInt8Calibrator(calibrator);
8846 }
8847
8854 {
8855 return mImpl->getInt8Calibrator();
8856 }
8857
8870 void setFlags(BuilderFlags builderFlags) noexcept
8871 {
8872 mImpl->setFlags(builderFlags);
8873 }
8874
8882 BuilderFlags getFlags() const noexcept
8883 {
8884 return mImpl->getFlags();
8885 }
8886
8894 void clearFlag(BuilderFlag builderFlag) noexcept
8895 {
8896 mImpl->clearFlag(builderFlag);
8897 }
8898
8906 void setFlag(BuilderFlag builderFlag) noexcept
8907 {
8908 mImpl->setFlag(builderFlag);
8909 }
8910
8918 bool getFlag(BuilderFlag builderFlag) const noexcept
8919 {
8920 return mImpl->getFlag(builderFlag);
8921 }
8922
8935 void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept
8936 {
8937 mImpl->setDeviceType(layer, deviceType);
8938 }
8939
8945 DeviceType getDeviceType(ILayer const* layer) const noexcept
8946 {
8947 return mImpl->getDeviceType(layer);
8948 }
8949
8957 bool isDeviceTypeSet(ILayer const* layer) const noexcept
8958 {
8959 return mImpl->isDeviceTypeSet(layer);
8960 }
8961
8967 void resetDeviceType(ILayer const* layer) noexcept
8968 {
8969 mImpl->resetDeviceType(layer);
8970 }
8971
8977 bool canRunOnDLA(ILayer const* layer) const noexcept
8978 {
8979 return mImpl->canRunOnDLA(layer);
8980 }
8981
8993 void setDLACore(int32_t dlaCore) noexcept
8994 {
8995 mImpl->setDLACore(dlaCore);
8996 }
8997
9003 int32_t getDLACore() const noexcept
9004 {
9005 return mImpl->getDLACore();
9006 }
9007
9014 void setDefaultDeviceType(DeviceType deviceType) noexcept
9015 {
9016 mImpl->setDefaultDeviceType(deviceType);
9017 }
9018
9025 {
9026 return mImpl->getDefaultDeviceType();
9027 }
9028
9034 void reset() noexcept
9035 {
9036 mImpl->reset();
9037 }
9038
9046 void setProfileStream(const cudaStream_t stream) noexcept
9047 {
9048 return mImpl->setProfileStream(stream);
9049 }
9050
9058 cudaStream_t getProfileStream() const noexcept
9059 {
9060 return mImpl->getProfileStream();
9061 }
9062
9075 int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept
9076 {
9077 return mImpl->addOptimizationProfile(profile);
9078 }
9079
9088 int32_t getNbOptimizationProfiles() const noexcept
9089 {
9090 return mImpl->getNbOptimizationProfiles();
9091 }
9092
9101 {
9102 mImpl->setProfilingVerbosity(verbosity);
9103 }
9104
9114 {
9115 return mImpl->getProfilingVerbosity();
9116 }
9117
9123 {
9124 mImpl->setAlgorithmSelector(selector);
9125 }
9126
9131 {
9132 return mImpl->getAlgorithmSelector();
9133 }
9134
9149 {
9150 return mImpl->setCalibrationProfile(profile);
9151 }
9152
9161 {
9162 return mImpl->getCalibrationProfile();
9163 }
9164
9178 {
9179 mImpl->setQuantizationFlags(flags);
9180 }
9181
9190 {
9191 return mImpl->getQuantizationFlags();
9192 }
9193
9202 {
9203 mImpl->clearQuantizationFlag(flag);
9204 }
9205
9214 {
9215 mImpl->setQuantizationFlag(flag);
9216 }
9217
9225 bool getQuantizationFlag(QuantizationFlag flag) const noexcept
9226 {
9227 return mImpl->getQuantizationFlag(flag);
9228 }
9229
9247 bool setTacticSources(TacticSources tacticSources) noexcept
9248 {
9249 return mImpl->setTacticSources(tacticSources);
9250 }
9251
9263 {
9264 return mImpl->getTacticSources();
9265 }
9266
9281 nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept
9282 {
9283 return mImpl->createTimingCache(blob, size);
9284 }
9285
9304 bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept
9305 {
9306 return mImpl->setTimingCache(cache, ignoreMismatch);
9307 }
9308
9315 {
9316 return mImpl->getTimingCache();
9317 }
9318
9346 void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
9347 {
9348 mImpl->setMemoryPoolLimit(pool, poolSize);
9349 }
9350
9365 std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
9366 {
9367 return mImpl->getMemoryPoolLimit(pool);
9368 }
9369
9383 void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
9384 {
9385 mImpl->setPreviewFeature(feature, enable);
9386 }
9387
9397 bool getPreviewFeature(PreviewFeature feature) const noexcept
9398 {
9399 return mImpl->getPreviewFeature(feature);
9400 }
9401
9430 void setBuilderOptimizationLevel(int32_t level) noexcept
9431 {
9432 mImpl->setBuilderOptimizationLevel(level);
9433 }
9434
9443 {
9444 return mImpl->getBuilderOptimizationLevel();
9445 }
9446
9459 void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept
9460 {
9461 mImpl->setHardwareCompatibilityLevel(hardwareCompatibilityLevel);
9462 }
9463
9473 {
9474 return mImpl->getHardwareCompatibilityLevel();
9475 }
9476
9485 void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept
9486 {
9487 mImpl->setPluginsToSerialize(paths, nbPaths);
9488 }
9489
9498 char const* getPluginToSerialize(int32_t index) const noexcept
9499 {
9500 return mImpl->getPluginToSerialize(index);
9501 }
9502
9508 int32_t getNbPluginsToSerialize() const noexcept
9509 {
9510 return mImpl->getNbPluginsToSerialize();
9511 }
9512
9537 void setMaxAuxStreams(int32_t nbStreams) noexcept
9538 {
9539 mImpl->setMaxAuxStreams(nbStreams);
9540 }
9541
9547 int32_t getMaxAuxStreams() const noexcept
9548 {
9549 return mImpl->getMaxAuxStreams();
9550 }
9551
9563 void setProgressMonitor(IProgressMonitor* monitor) noexcept
9564 {
9565 return mImpl->setProgressMonitor(monitor);
9566 }
9567
9574 {
9575 return mImpl->getProgressMonitor();
9576 }
9577
9589 void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept
9590 {
9591 mImpl->setRuntimePlatform(runtimePlatform);
9592 }
9593
9602 {
9603 return mImpl->getRuntimePlatform();
9604 }
9605
9613 void setMaxNbTactics(int32_t maxNbTactics) noexcept
9614 {
9615 mImpl->setMaxNbTactics(maxNbTactics);
9616 }
9617
9625 int32_t getMaxNbTactics() const noexcept
9626 {
9627 return mImpl->getMaxNbTactics();
9628 }
9629
9630protected:
9631 apiv::VBuilderConfig* mImpl;
9632};
9633
9642
9652{
9657
9662 kSTRONGLY_TYPED = 1,
9663};
9664
9670template <>
9671constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>() noexcept
9672{
9673 return 2;
9674}
9675
9683class IBuilder : public INoCopy
9684{
9685public:
9686 virtual ~IBuilder() noexcept = default;
9687
9693 TRT_DEPRECATED bool platformHasFastFp16() const noexcept
9694 {
9695 return mImpl->platformHasFastFp16();
9696 }
9697
9704 {
9705 return mImpl->platformHasFastInt8();
9706 }
9707
9715 int32_t getMaxDLABatchSize() const noexcept
9716 {
9717 return mImpl->getMaxDLABatchSize();
9718 }
9719
9723 int32_t getNbDLACores() const noexcept
9724 {
9725 return mImpl->getNbDLACores();
9726 }
9727
9740 void setGpuAllocator(IGpuAllocator* allocator) noexcept
9741 {
9742 mImpl->setGpuAllocator(allocator);
9743 }
9744
9751 {
9752 return mImpl->createBuilderConfig();
9753 }
9754
9773 {
9774 return mImpl->createNetworkV2(flags);
9775 }
9776
9788 {
9789 return mImpl->createOptimizationProfile();
9790 }
9791
9806 void setErrorRecorder(IErrorRecorder* recorder) noexcept
9807 {
9808 mImpl->setErrorRecorder(recorder);
9809 }
9810
9822 {
9823 return mImpl->getErrorRecorder();
9824 }
9825
9829 void reset() noexcept
9830 {
9831 mImpl->reset();
9832 }
9833
9839 TRT_DEPRECATED bool platformHasTf32() const noexcept
9840 {
9841 return mImpl->platformHasTf32();
9842 }
9843
9859 {
9860 return mImpl->buildSerializedNetwork(network, config);
9861 }
9862
9879 {
9880 return mImpl->buildEngineWithConfig(network, config);
9881 }
9882
9900 bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
9901 {
9902 return mImpl->isNetworkSupported(network, config);
9903 }
9904
9910 ILogger* getLogger() const noexcept
9911 {
9912 return mImpl->getLogger();
9913 }
9914
9926 bool setMaxThreads(int32_t maxThreads) noexcept
9927 {
9928 return mImpl->setMaxThreads(maxThreads);
9929 }
9930
9940 int32_t getMaxThreads() const noexcept
9941 {
9942 return mImpl->getMaxThreads();
9943 }
9944
9951 {
9952 return mImpl->getPluginRegistry();
9953 }
9954
9955protected:
9956 apiv::VBuilder* mImpl;
9957};
9958
9959} // namespace nvinfer1
9960
9965extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept;
9966
9967namespace nvinfer1
9968{
9969namespace
9970{
9971
9979inline IBuilder* createInferBuilder(ILogger& logger) noexcept
9980{
9981 return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
9982}
9983
9984} // namespace
9985
9999 nvinfer1::EngineCapability capability) noexcept;
10000
10001namespace safe
10002{
10004class IPluginRegistry;
10005} // namespace safe
10006
10014extern "C" TENSORRTAPI nvinfer1::safe::IPluginRegistry* getBuilderSafePluginRegistry(
10015 nvinfer1::EngineCapability capability) noexcept;
10016
10017} // namespace nvinfer1
10018
10019#endif // NV_INFER_H
#define TENSORRTAPI
Definition: NvInferRuntimeBase.h:59
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeBase.h:91
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:45
#define TRT_DEPRECATED_ENUM
Definition: NvInferRuntimeBase.h:46
Definition: NvInferRuntimeBase.h:200
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeBase.h:203
An Activation layer in a network definition.
Definition: NvInfer.h:1342
void setBeta(float beta) noexcept
Set the beta parameter (must be finite).
Definition: NvInfer.h:1390
void setActivationType(ActivationType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1351
ActivationType getActivationType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1361
float getAlpha() const noexcept
Get the alpha parameter.
Definition: NvInfer.h:1399
virtual ~IActivationLayer() noexcept=default
float getBeta() const noexcept
Get the beta parameter.
Definition: NvInfer.h:1408
void setAlpha(float alpha) noexcept
Set the alpha parameter (must be finite).
Definition: NvInfer.h:1376
Describes the context and requirements, that could be fulfilled by one or more instances of IAlgorith...
Definition: NvInfer.h:8049
int32_t getNbOutputs() const noexcept
Return number of outputs of the algorithm.
Definition: NvInfer.h:8084
int32_t getNbInputs() const noexcept
Return number of inputs of the algorithm.
Definition: NvInfer.h:8076
char const * getName() const noexcept
Return name of the algorithm node.
Definition: NvInfer.h:8056
virtual ~IAlgorithmContext() noexcept=default
Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for input or output tensor.
Definition: NvInfer.h:8068
Describes a variation of execution of a layer. An algorithm is represented by IAlgorithmVariant and t...
Definition: NvInfer.h:8106
std::size_t getWorkspaceSize() const noexcept
The size of the GPU temporary memory in bytes which the algorithm uses at execution time.
Definition: NvInfer.h:8127
float getTimingMSec() const noexcept
The time in milliseconds to execute the algorithm.
Definition: NvInfer.h:8119
IAlgorithmIOInfo const * getAlgorithmIOInfoByIndex(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:8141
virtual ~IAlgorithm() noexcept=default
IAlgorithmVariant const & getAlgorithmVariant() const noexcept
Returns the algorithm variant.
Definition: NvInfer.h:8111
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:7956
virtual ~IAlgorithmIOInfo() noexcept=default
int64_t getVectorizedDim() const noexcept
Return the index of the vectorized dimension or -1 for non-vectorized formats.
Definition: NvInfer.h:7984
Dims getStrides() const noexcept
Return strides of the input/output tensor of algorithm. For vectorized formats, strides are given in ...
Definition: NvInfer.h:7974
DataType getDataType() const noexcept
Return DataType of the input/output of algorithm.
Definition: NvInfer.h:7963
int64_t getComponentsPerElement() const noexcept
Return the number of components per element. This is always 1 for non-vectorized formats.
Definition: NvInfer.h:7995
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:8017
virtual ~IAlgorithmVariant() noexcept=default
int64_t getTactic() const noexcept
Return tactic of the algorithm.
Definition: NvInfer.h:8030
int64_t getImplementation() const noexcept
Return implementation of the algorithm.
Definition: NvInfer.h:8022
An assertion layer in a network.
Definition: NvInfer.h:4933
void setMessage(char const *message) noexcept
Set the message to print if the assertion fails.
Definition: NvInfer.h:4943
char const * getMessage() const noexcept
Return the assertion message.
Definition: NvInfer.h:4953
virtual ~IAssertionLayer() noexcept=default
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:8782
void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
Set the memory size for the memory pool.
Definition: NvInfer.h:9346
void setQuantizationFlag(QuantizationFlag flag) noexcept
Set a single quantization flag.
Definition: NvInfer.h:9213
nvinfer1::ITimingCache * createTimingCache(void const *blob, std::size_t size) const noexcept
Create timing cache.
Definition: NvInfer.h:9281
void setPreviewFeature(PreviewFeature feature, bool enable) noexcept
Enable or disable a specific preview feature.
Definition: NvInfer.h:9383
TRT_DEPRECATED void setInt8Calibrator(IInt8Calibrator *calibrator) noexcept
Set Int8 Calibration interface.
Definition: NvInfer.h:8843
bool getPreviewFeature(PreviewFeature feature) const noexcept
Get status of preview feature.
Definition: NvInfer.h:9397
void clearQuantizationFlag(QuantizationFlag flag) noexcept
clear a quantization flag.
Definition: NvInfer.h:9201
int32_t getBuilderOptimizationLevel() noexcept
Get builder optimization level.
Definition: NvInfer.h:9442
bool setTacticSources(TacticSources tacticSources) noexcept
Set tactic sources.
Definition: NvInfer.h:9247
void setPluginsToSerialize(char const *const *paths, int32_t nbPaths) noexcept
Set the plugin libraries to be serialized with version-compatible engines.
Definition: NvInfer.h:9485
bool getQuantizationFlag(QuantizationFlag flag) const noexcept
Returns true if the quantization flag is set.
Definition: NvInfer.h:9225
TRT_DEPRECATED IInt8Calibrator * getInt8Calibrator() const noexcept
Get Int8 Calibration interface.
Definition: NvInfer.h:8853
std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
Get the memory size limit of the memory pool.
Definition: NvInfer.h:9365
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInfer.h:9003
int32_t getNbPluginsToSerialize() const noexcept
Get the number of plugin library paths to be serialized with version-compatible engines.
Definition: NvInfer.h:9508
void setDeviceType(ILayer const *layer, DeviceType deviceType) noexcept
Set the device that this layer must execute on.
Definition: NvInfer.h:8935
void setEngineCapability(EngineCapability capability) noexcept
Configure the builder to target specified EngineCapability flow.
Definition: NvInfer.h:8819
int32_t getMaxAuxStreams() const noexcept
Get the maximum number of auxiliary streams that TRT is allowed to use.
Definition: NvInfer.h:9547
bool getFlag(BuilderFlag builderFlag) const noexcept
Returns true if the build mode flag is set.
Definition: NvInfer.h:8918
void setQuantizationFlags(QuantizationFlags flags) noexcept
Set the quantization flags.
Definition: NvInfer.h:9177
void setMaxNbTactics(int32_t maxNbTactics) noexcept
Set the maximum number of tactics to time when there is a choice of tactics.
Definition: NvInfer.h:9613
void setProgressMonitor(IProgressMonitor *monitor) noexcept
Sets the progress monitor for building a network.
Definition: NvInfer.h:9563
void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:9100
IAlgorithmSelector * getAlgorithmSelector() const noexcept
Get Algorithm Selector.
Definition: NvInfer.h:9130
int32_t getNbOptimizationProfiles() const noexcept
Get number of optimization profiles.
Definition: NvInfer.h:9088
QuantizationFlags getQuantizationFlags() const noexcept
Get the quantization flags.
Definition: NvInfer.h:9189
nvinfer1::ITimingCache const * getTimingCache() const noexcept
Get the pointer to the timing cache from current IBuilderConfig.
Definition: NvInfer.h:9314
void reset() noexcept
Resets the builder configuration to defaults.
Definition: NvInfer.h:9034
bool setTimingCache(ITimingCache const &cache, bool ignoreMismatch) noexcept
Attach a timing cache to IBuilderConfig.
Definition: NvInfer.h:9304
char const * getPluginToSerialize(int32_t index) const noexcept
Get the plugin library path to be serialized with version-compatible engines.
Definition: NvInfer.h:9498
EngineCapability getEngineCapability() const noexcept
Query EngineCapability flow configured for the builder.
Definition: NvInfer.h:8831
void setAlgorithmSelector(IAlgorithmSelector *selector) noexcept
Set Algorithm Selector.
Definition: NvInfer.h:9122
RuntimePlatform getRuntimePlatform() const noexcept
Get the target platform for runtime execution.
Definition: NvInfer.h:9601
DeviceType getDefaultDeviceType() const noexcept
Get the default DeviceType which was set by setDefaultDeviceType.
Definition: NvInfer.h:9024
void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept
Set the target platform for runtime execution.
Definition: NvInfer.h:9589
int32_t getMaxNbTactics() const noexcept
Query the maximum number of tactics timed when there is a choice.
Definition: NvInfer.h:9625
BuilderFlags getFlags() const noexcept
Get the build mode flags for this builder config. Defaults to 0.
Definition: NvInfer.h:8882
void setFlags(BuilderFlags builderFlags) noexcept
Set the build mode flags to turn on builder options for this network.
Definition: NvInfer.h:8870
TacticSources getTacticSources() const noexcept
Get tactic sources.
Definition: NvInfer.h:9262
void resetDeviceType(ILayer const *layer) noexcept
reset the DeviceType for this layer
Definition: NvInfer.h:8967
void setDLACore(int32_t dlaCore) noexcept
Sets the DLA core used by the network. Defaults to -1.
Definition: NvInfer.h:8993
HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept
Get the hardware compatibility level.
Definition: NvInfer.h:9472
void clearFlag(BuilderFlag builderFlag) noexcept
clear a single build mode flag.
Definition: NvInfer.h:8894
int32_t addOptimizationProfile(IOptimizationProfile const *profile) noexcept
Add an optimization profile.
Definition: NvInfer.h:9075
IProgressMonitor * getProgressMonitor() const noexcept
Definition: NvInfer.h:9573
apiv::VBuilderConfig * mImpl
Definition: NvInfer.h:9631
TRT_DEPRECATED IOptimizationProfile const * getCalibrationProfile() noexcept
Get the current calibration profile.
Definition: NvInfer.h:9160
int32_t getAvgTimingIterations() const noexcept
Query the number of averaging iterations.
Definition: NvInfer.h:8806
void setDefaultDeviceType(DeviceType deviceType) noexcept
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
Definition: NvInfer.h:9014
void setFlag(BuilderFlag builderFlag) noexcept
Set a single build mode flag.
Definition: NvInfer.h:8906
TRT_DEPRECATED bool setCalibrationProfile(IOptimizationProfile const *profile) noexcept
Add a calibration profile.
Definition: NvInfer.h:9148
virtual ~IBuilderConfig() noexcept=default
DeviceType getDeviceType(ILayer const *layer) const noexcept
Get the device that this layer executes on.
Definition: NvInfer.h:8945
bool canRunOnDLA(ILayer const *layer) const noexcept
Checks if a layer can run on DLA.
Definition: NvInfer.h:8977
cudaStream_t getProfileStream() const noexcept
Get the CUDA stream that is used to profile this network.
Definition: NvInfer.h:9058
void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept
Set the hardware compatibility level.
Definition: NvInfer.h:9459
void setMaxAuxStreams(int32_t nbStreams) noexcept
Set the maximum number of auxiliary streams that TRT is allowed to use.
Definition: NvInfer.h:9537
ProfilingVerbosity getProfilingVerbosity() const noexcept
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:9113
bool isDeviceTypeSet(ILayer const *layer) const noexcept
whether the DeviceType has been explicitly set for this layer
Definition: NvInfer.h:8957
void setBuilderOptimizationLevel(int32_t level) noexcept
Set builder optimization level.
Definition: NvInfer.h:9430
void setProfileStream(const cudaStream_t stream) noexcept
Set the CUDA stream that is used to profile this network.
Definition: NvInfer.h:9046
Builds an engine from a network definition.
Definition: NvInfer.h:9684
int32_t getMaxDLABatchSize() const noexcept
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Definition: NvInfer.h:9715
int32_t getNbDLACores() const noexcept
Return the number of DLA engines available to this builder.
Definition: NvInfer.h:9723
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:9821
apiv::VBuilder * mImpl
Definition: NvInfer.h:9956
ILogger * getLogger() const noexcept
get the logger with which the builder was created
Definition: NvInfer.h:9910
bool isNetworkSupported(INetworkDefinition const &network, IBuilderConfig const &config) const noexcept
Checks that a network is within the scope of the IBuilderConfig settings.
Definition: NvInfer.h:9900
int32_t getMaxThreads() const noexcept
get the maximum number of threads that can be used by the builder.
Definition: NvInfer.h:9940
IPluginRegistry & getPluginRegistry() noexcept
get the local plugin registry that can be used by the builder.
Definition: NvInfer.h:9950
TRT_DEPRECATED bool platformHasFastInt8() const noexcept
Determine whether the platform has fast native int8.
Definition: NvInfer.h:9703
nvinfer1::IOptimizationProfile * createOptimizationProfile() noexcept
Create a new optimization profile.
Definition: NvInfer.h:9787
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInfer.h:9740
nvinfer1::INetworkDefinition * createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept
Create a network definition object.
Definition: NvInfer.h:9772
nvinfer1::IBuilderConfig * createBuilderConfig() noexcept
Create a builder configuration object.
Definition: NvInfer.h:9750
void reset() noexcept
Resets the builder state to default values.
Definition: NvInfer.h:9829
bool setMaxThreads(int32_t maxThreads) noexcept
Set the maximum number of threads.
Definition: NvInfer.h:9926
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:9806
nvinfer1::IHostMemory * buildSerializedNetwork(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds and serializes a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:9858
virtual ~IBuilder() noexcept=default
TRT_DEPRECATED bool platformHasTf32() const noexcept
Determine whether the platform has TF32 support.
Definition: NvInfer.h:9839
nvinfer1::ICudaEngine * buildEngineWithConfig(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:9878
A cast layer in a network.
Definition: NvInfer.h:3800
virtual ~ICastLayer() noexcept=default
apiv::VCastLayer * mImpl
Definition: NvInfer.h:3826
DataType getToType() const noexcept
Return cast layer output type.
Definition: NvInfer.h:3820
void setToType(DataType toType) noexcept
Set cast layer output type.
Definition: NvInfer.h:3809
A concatenation layer in a network definition.
Definition: NvInfer.h:2052
void setAxis(int32_t axis) noexcept
Set the axis along which concatenation occurs.
Definition: NvInfer.h:2065
int32_t getAxis() const noexcept
Get the axis along which concatenation occurs.
Definition: NvInfer.h:2075
virtual ~IConcatenationLayer() noexcept=default
This layer represents a condition input to an IIfConditional.
Definition: NvInfer.h:4459
virtual ~IConditionLayer() noexcept=default
Layer that represents a constant value.
Definition: NvInfer.h:3839
void setWeights(Weights weights) noexcept
Set the weights for the layer.
Definition: NvInfer.h:3849
Weights getWeights() const noexcept
Get the weights for the layer.
Definition: NvInfer.h:3859
void setDimensions(Dims const &dimensions) noexcept
Set the dimensions for the layer.
Definition: NvInfer.h:3871
apiv::VConstantLayer * mImpl
Definition: NvInfer.h:3889
virtual ~IConstantLayer() noexcept=default
Dims getDimensions() const noexcept
Get the dimensions for the layer.
Definition: NvInfer.h:3883
A convolution layer in a network definition.
Definition: NvInfer.h:1022
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1147
Weights getBiasWeights() const noexcept
Get the bias weights for the convolution.
Definition: NvInfer.h:1120
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1188
void setDilationNd(Dims const &dilation) noexcept
Set the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1292
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the convolution.
Definition: NvInfer.h:1278
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the convolution.
Definition: NvInfer.h:1248
Weights getKernelWeights() const noexcept
Get the kernel weights of the convolution.
Definition: NvInfer.h:1095
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride of the convolution.
Definition: NvInfer.h:1238
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1302
int64_t getNbOutputMaps() const noexcept
Get the number of output maps for the convolution.
Definition: NvInfer.h:1041
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the convolution.
Definition: NvInfer.h:1085
Dims getPostPadding() const noexcept
Get the post-padding.
Definition: NvInfer.h:1174
int64_t getNbGroups() const noexcept
Get the number of groups of the convolution.
Definition: NvInfer.h:1071
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1200
virtual ~IConvolutionLayer() noexcept=default
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups for a convolution.
Definition: NvInfer.h:1061
void setNbOutputMaps(int64_t nbOutputMaps) noexcept
Set the number of output maps for the convolution.
Definition: NvInfer.h:1031
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the convolution.
Definition: NvInfer.h:1110
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1223
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding of the convolution.
Definition: NvInfer.h:1266
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding of the convolution.
Definition: NvInfer.h:1137
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding of the convolution.
Definition: NvInfer.h:1164
void setKernelSizeNd(Dims const &kernelSize) noexcept
Set the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1213
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:2903
A deconvolution layer in a network definition.
Definition: NvInfer.h:2093
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the deconvolution.
Definition: NvInfer.h:2181
int64_t getNbGroups() const noexcept
Get the number of groups for a deconvolution.
Definition: NvInfer.h:2142
Weights getKernelWeights() const noexcept
Get the kernel weights for the deconvolution.
Definition: NvInfer.h:2166
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding of the deconvolution.
Definition: NvInfer.h:2208
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2323
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2389
Weights getBiasWeights() const noexcept
Get the bias weights for the deconvolution.
Definition: NvInfer.h:2191
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the deconvolution.
Definition: NvInfer.h:2156
int64_t getNbOutputMaps() const noexcept
Get the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2112
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2313
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:2245
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2296
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding of the deconvolution.
Definition: NvInfer.h:2235
void setKernelSizeNd(Dims const &kernelSize) noexcept
Set the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2286
virtual ~IDeconvolutionLayer() noexcept=default
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2341
void setNbOutputMaps(int64_t nbOutputMaps) noexcept
Set the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2102
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2353
void setDilationNd(Dims const &dilation) noexcept
Set the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2379
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:2259
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups for a deconvolution.
Definition: NvInfer.h:2132
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:2218
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:2271
A Dequantize layer in a network definition.
Definition: NvInfer.h:5519
void setToType(DataType toType) noexcept
Set the Dequantize layer output type.
Definition: NvInfer.h:5556
virtual ~IDequantizeLayer() noexcept=default
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5529
DataType getToType() const noexcept
Return the Dequantize layer output type.
Definition: NvInfer.h:5568
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5540
An Einsum layer in a network.
Definition: NvInfer.h:5616
bool setEquation(char const *equation) noexcept
Set the equation. The equation is a comma-separated list of subscript labels, where each label refers...
Definition: NvInfer.h:5627
virtual ~IEinsumLayer() noexcept=default
char const * getEquation() const noexcept
Return the equation.
Definition: NvInfer.h:5637
A elementwise layer in a network definition.
Definition: NvInfer.h:2465
virtual ~IElementWiseLayer() noexcept=default
apiv::VElementWiseLayer * mImpl
Definition: NvInfer.h:2494
ElementWiseOperation getOperation() const noexcept
Get the binary operation for the layer.
Definition: NvInfer.h:2488
void setOperation(ElementWiseOperation op) noexcept
Set the binary operation for the layer.
Definition: NvInfer.h:2476
Generate a tensor according to a specified mode.
Definition: NvInfer.h:5044
bool isAlphaBetaInt64() const noexcept
Return true if alpha/beta have type int64, false if they have type double.
Definition: NvInfer.h:5276
FillOperation getOperation() const noexcept
Get the fill operation for the layer.
Definition: NvInfer.h:5090
void setOperation(FillOperation op) noexcept
Set the fill operation for the layer.
Definition: NvInfer.h:5080
DataType getToType() const noexcept
Get the fill layer output type.
Definition: NvInfer.h:5305
void setAlphaInt64(int64_t alpha) noexcept
Set the alpha parameter with int64 datatype.
Definition: NvInfer.h:5219
void setBetaInt64(int64_t beta) noexcept
Set the beta parameter with int64 datatype.
Definition: NvInfer.h:5253
void setBeta(double beta) noexcept
Set the beta parameter.
Definition: NvInfer.h:5143
int64_t getAlphaInt64() const noexcept
Get the value of alpha parameter with int64 datatype.
Definition: NvInfer.h:5234
int64_t getBetaInt64() const noexcept
Get the value of beta parameter with int64 datatype.
Definition: NvInfer.h:5268
double getAlpha() const noexcept
Get the value of alpha parameter.
Definition: NvInfer.h:5124
void setDimensions(Dims const &dimensions) noexcept
Set the output tensor's dimensions.
Definition: NvInfer.h:5055
void setAlpha(double alpha) noexcept
Set the alpha parameter.
Definition: NvInfer.h:5109
void setToType(DataType toType) noexcept
Set the fill layer output type.
Definition: NvInfer.h:5293
Dims getDimensions() const noexcept
Get the output tensor's dimensions.
Definition: NvInfer.h:5070
double getBeta() const noexcept
Get the value of beta parameter.
Definition: NvInfer.h:5158
virtual ~IFillLayer() noexcept=default
A Gather layer in a network definition. Supports several kinds of gathering.
Definition: NvInfer.h:2598
void setGatherAxis(int32_t axis) noexcept
Set the axis used by GatherMode::kELEMENTS and GatherMode::kDEFAULT The axis must be less than the nu...
Definition: NvInfer.h:2609
void setNbElementWiseDims(int32_t elementWiseDims) noexcept
Set the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:2644
apiv::VGatherLayer * mImpl
Definition: NvInfer.h:2680
int32_t getNbElementWiseDims() const noexcept
Get the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:2654
void setMode(GatherMode mode) noexcept
Set the gather mode.
Definition: NvInfer.h:2664
int32_t getGatherAxis() const noexcept
Get the axis to gather on.
Definition: NvInfer.h:2621
GatherMode getMode() const noexcept
Get the gather mode.
Definition: NvInfer.h:2674
virtual ~IGatherLayer() noexcept=default
A GridSample layer in a network definition.
Definition: NvInfer.h:5837
void setInterpolationMode(InterpolationMode mode) noexcept
Set the grid sample interpolation mode.
Definition: NvInfer.h:5844
bool setSampleMode(SampleMode mode) noexcept
Set the sample mode.
Definition: NvInfer.h:5890
void setAlignCorners(bool alignCorners) noexcept
Set the align corners mode.
Definition: NvInfer.h:5866
apiv::VGridSampleLayer * mImpl
Definition: NvInfer.h:5908
SampleMode getSampleMode() const noexcept
Get the sample mode.
Definition: NvInfer.h:5902
InterpolationMode getInterpolationMode() const noexcept
Get the grid sample interpolation mode.
Definition: NvInfer.h:5856
bool getAlignCorners() const noexcept
Get the align corners mode.
Definition: NvInfer.h:5878
virtual ~IGridSampleLayer() noexcept=default
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:142
A layer that represents the identity function.
Definition: NvInfer.h:3787
apiv::VIdentityLayer * mImpl
Definition: NvInfer.h:3789
virtual ~IIdentityLayer() noexcept=default
This is a base class for Conditional boundary layers.
Definition: NvInfer.h:4438
IIfConditional * getConditional() const noexcept
Get a pointer to the IIfConditional associated with this boundary layer.
Definition: NvInfer.h:4443
virtual ~IIfConditionalBoundaryLayer() noexcept=default
Helper for constructing conditionally-executed subgraphs.
Definition: NvInfer.h:4520
IIfConditionalInputLayer * addInput(ITensor &input) noexcept
Add an If-conditional input.
Definition: NvInfer.h:4561
char const * getName() const noexcept
Return the name of the conditional.
Definition: NvInfer.h:4586
virtual ~IIfConditional() noexcept=default
IConditionLayer * setCondition(ITensor &condition) noexcept
Set the condition tensor for this If-Conditional construct.
Definition: NvInfer.h:4531
IIfConditionalOutputLayer * addOutput(ITensor &trueSubgraphOutput, ITensor &falseSubgraphOutput) noexcept
Add an If-conditional output.
Definition: NvInfer.h:4549
void setName(char const *name) noexcept
Set the name of the conditional.
Definition: NvInfer.h:4576
This layer represents an input to an IIfConditional.
Definition: NvInfer.h:4489
virtual ~IIfConditionalInputLayer() noexcept=default
This layer represents an output of an IIfConditional.
Definition: NvInfer.h:4476
virtual ~IIfConditionalOutputLayer() noexcept=default
Application-implemented interface for calibration.
Definition: NvInfer.h:7683
virtual TRT_DEPRECATED int32_t getBatchSize() const noexcept=0
Get the batch size used for calibration batches.
A layer to do iterations.
Definition: NvInfer.h:4751
virtual ~IIteratorLayer() noexcept=default
void setReverse(bool reverse) noexcept
Set iteration order to be reverse.
Definition: NvInfer.h:4778
bool getReverse() const noexcept
Check if the iteration order is reverse.
Definition: NvInfer.h:4788
int32_t getAxis() const noexcept
Get axis being iterated over.
Definition: NvInfer.h:4764
void setAxis(int32_t axis) noexcept
Set axis to iterate over.
Definition: NvInfer.h:4756
A LRN layer in a network definition.
Definition: NvInfer.h:1707
int64_t getWindowSize() const noexcept
Get the LRN window size.
Definition: NvInfer.h:1728
float getAlpha() const noexcept
Get the LRN alpha value.
Definition: NvInfer.h:1750
void setWindowSize(int64_t windowSize) noexcept
Set the LRN window size.
Definition: NvInfer.h:1718
void setK(float k) noexcept
Set the LRN K value.
Definition: NvInfer.h:1784
void setAlpha(float alpha) noexcept
Set the LRN alpha value.
Definition: NvInfer.h:1740
void setBeta(float beta) noexcept
Set the LRN beta value.
Definition: NvInfer.h:1762
virtual ~ILRNLayer() noexcept=default
float getBeta() const noexcept
Get the LRN beta value.
Definition: NvInfer.h:1772
float getK() const noexcept
Get the LRN K value.
Definition: NvInfer.h:1794
Base class for all layer classes in a network definition.
Definition: NvInfer.h:550
bool precisionIsSet() const noexcept
whether the computational precision has been set for this layer
Definition: NvInfer.h:692
void setMetadata(char const *metadata) noexcept
Set the metadata for this layer.
Definition: NvInfer.h:808
void setPrecision(DataType dataType) noexcept
Set the preferred or required computational precision of this layer in a weakly-typed network.
Definition: NvInfer.h:668
void setName(char const *name) noexcept
Set the name of a layer.
Definition: NvInfer.h:571
void resetPrecision() noexcept
reset the computational precision for this layer
Definition: NvInfer.h:702
int32_t getNbInputs() const noexcept
Get the number of inputs of a layer.
Definition: NvInfer.h:589
char const * getMetadata() const noexcept
Get the metadata of the layer.
Definition: NvInfer.h:821
DataType getOutputType(int32_t index) const noexcept
get the output type of this layer
Definition: NvInfer.h:764
DataType getPrecision() const noexcept
get the computational precision of this layer
Definition: NvInfer.h:680
char const * getName() const noexcept
Return the name of a layer.
Definition: NvInfer.h:581
int32_t getNbOutputs() const noexcept
Get the number of outputs of a layer.
Definition: NvInfer.h:610
bool outputTypeIsSet(int32_t index) const noexcept
whether the output type has been set for this layer
Definition: NvInfer.h:778
ITensor * getOutput(int32_t index) const noexcept
Get the layer output corresponding to the given index.
Definition: NvInfer.h:620
void setInput(int32_t index, ITensor &tensor) noexcept
Replace an input of this layer with a specific tensor.
Definition: NvInfer.h:637
void resetOutputType(int32_t index) noexcept
reset the output type for this layer
Definition: NvInfer.h:790
ITensor * getInput(int32_t index) const noexcept
Get the layer input corresponding to the given index.
Definition: NvInfer.h:602
void setOutputType(int32_t index, DataType dataType) noexcept
Set the output type of this layer in a weakly-typed network.
Definition: NvInfer.h:749
LayerType getType() const noexcept
Return the type of a layer.
Definition: NvInfer.h:557
virtual ~ILayer() noexcept=default
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntime.h:1464
This is a base class for Loop boundary layers.
Definition: NvInfer.h:4415
virtual ~ILoopBoundaryLayer() noexcept=default
ILoop * getLoop() const noexcept
Get a pointer to ILoop associated with this boundary layer.
Definition: NvInfer.h:4420
Helper for creating a recurrent subgraph.
Definition: NvInfer.h:4808
void setName(char const *name) noexcept
Set the name of the loop.
Definition: NvInfer.h:4878
ITripLimitLayer * addTripLimit(ITensor &tensor, TripLimit limit) noexcept
Add a trip-count limiter, based on the given tensor.
Definition: NvInfer.h:4837
IIteratorLayer * addIterator(ITensor &tensor, int32_t axis=0, bool reverse=false) noexcept
Return layer that subscripts tensor by loop iteration.
Definition: NvInfer.h:4850
ILoopOutputLayer * addLoopOutput(ITensor &tensor, LoopOutput outputKind, int32_t axis=0) noexcept
Make an output for this loop, based on the given tensor.
Definition: NvInfer.h:4863
virtual ~ILoop() noexcept=default
char const * getName() const noexcept
Return the name of the loop.
Definition: NvInfer.h:4888
IRecurrenceLayer * addRecurrence(ITensor &initialValue) noexcept
Create a recurrence layer for this loop with initialValue as its first input.
Definition: NvInfer.h:4816
An ILoopOutputLayer is the sole way to get output from a loop.
Definition: NvInfer.h:4651
virtual ~ILoopOutputLayer() noexcept=default
int32_t getAxis() const noexcept
Get axis being concatenated over.
Definition: NvInfer.h:4681
LoopOutput getLoopOutput() const noexcept
Get which kind a loop output has.
Definition: NvInfer.h:4656
void setAxis(int32_t axis) noexcept
Set where to insert the contenation axis. Ignored if getLoopOutput() is kLAST_VALUE.
Definition: NvInfer.h:4673
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:3679
apiv::VMatrixMultiplyLayer * mImpl
Definition: NvInfer.h:3707
virtual ~IMatrixMultiplyLayer() noexcept=default
MatrixOperation getOperation(int32_t index) const noexcept
Get the operation for an input tensor.
Definition: NvInfer.h:3701
void setOperation(int32_t index, MatrixOperation op) noexcept
Set the operation for an input tensor.
Definition: NvInfer.h:3689
A non-maximum suppression layer in a network definition.
Definition: NvInfer.h:5985
virtual ~INMSLayer() noexcept=default
void setTopKBoxLimit(int32_t limit) noexcept
Set the TopK box limit parameter for the layer.
Definition: NvInfer.h:6022
void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept
Set the bounding box format parameter for the layer.
Definition: NvInfer.h:5996
BoundingBoxFormat getBoundingBoxFormat() const noexcept
Get the bounding box format parameter for the layer.
Definition: NvInfer.h:6008
apiv::VNMSLayer * mImpl
Definition: NvInfer.h:6058
int32_t getTopKBoxLimit() const noexcept
Get the TopK box limit parameter for the layer.
Definition: NvInfer.h:6032
A network definition for input to the builder.
Definition: NvInfer.h:6281
IPluginV2Layer * addPluginV2(ITensor *const *inputs, int32_t nbInputs, IPluginV2 &plugin) noexcept
Add a plugin layer to the network using the IPluginV2 interface.
Definition: NvInfer.h:6883
IConcatenationLayer * addConcatenation(ITensor *const *inputs, int32_t nbInputs) noexcept
Add a concatenation layer to the network.
Definition: NvInfer.h:6474
IShuffleLayer * addShuffle(ITensor &input) noexcept
Add a shuffle layer to the network.
Definition: NvInfer.h:6537
INormalizationLayer * addNormalization(ITensor &input, ITensor &scale, ITensor &bias, uint32_t axesMask) noexcept
Add a normalization layer to the network.
Definition: NvInfer.h:7585
void setName(char const *name) noexcept
Sets the name of the network.
Definition: NvInfer.h:6945
bool markDebug(ITensor &tensor) noexcept
Mark a tensor as a debug tensor.
Definition: NvInfer.h:6353
ILRNLayer * addLRN(ITensor &input, int64_t window, float alpha, float beta, float k) noexcept
Add a LRN layer to the network.
Definition: NvInfer.h:6418
ITopKLayer * addTopK(ITensor &input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
Add a TopK layer to the network.
Definition: NvInfer.h:6697
IAssertionLayer * addAssertion(ITensor &condition, char const *message) noexcept
Add an assertion layer to the network.
Definition: NvInfer.h:7261
IConvolutionLayer * addConvolutionNd(ITensor &input, int64_t nbOutputMaps, Dims const &kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension convolution layer to the network.
Definition: NvInfer.h:7080
ICastLayer * addCast(ITensor &input, DataType toType) noexcept
Add a cast layer.
Definition: NvInfer.h:6837
IScaleLayer * addScaleNd(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
Add a multi-dimension scale layer to the network.
Definition: NvInfer.h:7159
char const * getName() const noexcept
Returns the name associated with the network.
Definition: NvInfer.h:6959
IParametricReLULayer * addParametricReLU(ITensor &input, ITensor &slope) noexcept
Add a parametric ReLU layer to the network.
Definition: NvInfer.h:7058
ITensor * getOutput(int32_t index) const noexcept
Get the output tensor specified by the given index.
Definition: NvInfer.h:6638
ITensor * getInput(int32_t index) const noexcept
Get the input tensor specified by the given index.
Definition: NvInfer.h:6608
IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale, DataType outputType) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7428
bool unmarkOutputForShapes(ITensor &tensor) noexcept
Undo markOutputForShapes.
Definition: NvInfer.h:7040
IFillLayer * addFill(Dims const &dimensions, FillOperation op, DataType outputType) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7312
ILoop * addLoop() noexcept
Add a loop to the network.
Definition: NvInfer.h:7190
IActivationLayer * addActivation(ITensor &input, ActivationType type) noexcept
Add an activation layer to the network.
Definition: NvInfer.h:6399
TRT_DEPRECATED IFillLayer * addFill(Dims const &dimensions, FillOperation op) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7286
ISliceLayer * addSlice(ITensor &input, Dims const &start, Dims const &size, Dims const &stride) noexcept
Add a slice layer to the network.
Definition: NvInfer.h:6921
virtual ~INetworkDefinition() noexcept=default
TRT_DEPRECATED IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7469
virtual IBuilder & getBuilder() const noexcept
Return the builder from which this INetworkDefinition was created.
Definition: NvInfer.h:7596
INMSLayer * addNMS(ITensor &boxes, ITensor &scores, ITensor &maxOutputBoxesPerClass) noexcept
Add a non-maximum suppression layer to the network.
Definition: NvInfer.h:7542
ILayer * getLayer(int32_t index) const noexcept
Get the layer specified by the given index.
Definition: NvInfer.h:6580
bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept
Returns true if the network definition creation flag is set.
Definition: NvInfer.h:7011
IIfConditional * addIfConditional() noexcept
Add an if-then-else to the network.
Definition: NvInfer.h:7205
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:7386
IReverseSequenceLayer * addReverseSequence(ITensor &input, ITensor &sequenceLens) noexcept
Add a ReverseSequence layer to the network.
Definition: NvInfer.h:7559
int32_t getNbInputs() const noexcept
Get the number of inputs in the network.
Definition: NvInfer.h:6592
NetworkDefinitionCreationFlags getFlags() const noexcept
Get the network definition creation flags for this network definition object. Defaults to 0.
Definition: NvInfer.h:6999
IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale, DataType outputType) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7490
IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
Add a reduce layer to the network.
Definition: NvInfer.h:6664
IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation) noexcept
Add a unary layer to the network.
Definition: NvInfer.h:6523
IGridSampleLayer * addGridSample(ITensor &input, ITensor &grid) noexcept
Add a GridSample layer to the network.
Definition: NvInfer.h:7524
void removeTensor(ITensor &tensor) noexcept
remove a tensor from the network definition.
Definition: NvInfer.h:6852
bool areWeightsMarkedRefittable(char const *name) const noexcept
Whether the weight has been marked as refittable.
Definition: NvInfer.h:7634
ISelectLayer * addSelect(ITensor &condition, ITensor &thenInput, ITensor &elseInput) noexcept
Add a select layer to the network.
Definition: NvInfer.h:7244
IScatterLayer * addScatter(ITensor &data, ITensor &indices, ITensor &updates, ScatterMode mode) noexcept
Add a Scatter layer to the network with specified mode and axis=0.
Definition: NvInfer.h:7448
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInfer.h:6566
TRT_DEPRECATED bool hasImplicitBatchDimension() const noexcept
Query whether the network was created with an implicit batch dimension.
Definition: NvInfer.h:6989
apiv::VNetworkDefinition * mImpl
Definition: NvInfer.h:7640
bool markOutputForShapes(ITensor &tensor) noexcept
Enable tensor's value to be computed by IExecutionContext::getShapeBinding.
Definition: NvInfer.h:7028
IOneHotLayer * addOneHot(ITensor &indices, ITensor &values, ITensor &depth, int32_t axis) noexcept
Add a OneHot layer to the network.
Definition: NvInfer.h:6554
IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
Add a Scale layer to the network.
Definition: NvInfer.h:6444
IPluginV3Layer * addPluginV3(ITensor *const *inputs, int32_t nbInputs, ITensor *const *shapeInputs, int32_t nbShapeInputs, IPluginV3 &plugin) noexcept
Add a plugin layer implementing the IPluginV3 interface to the network.
Definition: NvInfer.h:6901
void unmarkOutput(ITensor &tensor) noexcept
unmark a tensor as a network output.
Definition: NvInfer.h:6864
IIdentityLayer * addIdentity(ITensor &input) noexcept
Add an identity layer.
Definition: NvInfer.h:6822
IGatherLayer * addGatherV2(ITensor &data, ITensor &indices, GatherMode mode) noexcept
Add gather with specified mode, axis=0 and nbElementWiseDims=0.
Definition: NvInfer.h:6729
IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op) noexcept
Add an elementwise layer to the network.
Definition: NvInfer.h:6501
IConstantLayer * addConstant(Dims const &dimensions, Weights weights) noexcept
Add a constant layer to the network.
Definition: NvInfer.h:6808
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:7371
IPoolingLayer * addPoolingNd(ITensor &input, PoolingType type, Dims const &windowSize) noexcept
Add a multi-dimension pooling layer to the network.
Definition: NvInfer.h:7100
IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds) noexcept
Add a RaggedSoftMax layer to the network.
Definition: NvInfer.h:6748
IShapeLayer * addShape(ITensor &input) noexcept
Add a shape layer to the network.
Definition: NvInfer.h:6975
IGatherLayer * addGather(ITensor &data, ITensor &indices, int32_t axis) noexcept
Add gather with mode GatherMode::kDEFAULT and specified axis and nbElementWiseDims=0.
Definition: NvInfer.h:6713
bool unmarkWeightsRefittable(char const *name) noexcept
Unmark weights as refittable when the builder flag kREFIT_INDIVIDUAL is set.
Definition: NvInfer.h:7621
bool markWeightsRefittable(char const *name) noexcept
Mark weights as refittable when the builder flag kREFIT_INDIVIDUAL is set.
Definition: NvInfer.h:7609
IDeconvolutionLayer * addDeconvolutionNd(ITensor &input, int64_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension deconvolution layer to the network.
Definition: NvInfer.h:7122
IResizeLayer * addResize(ITensor &input) noexcept
Add a resize layer to the network.
Definition: NvInfer.h:7176
IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1) noexcept
Add a MatrixMultiply layer to the network.
Definition: NvInfer.h:6769
ISoftMaxLayer * addSoftMax(ITensor &input) noexcept
Add a SoftMax layer to the network.
Definition: NvInfer.h:6457
bool isDebugTensor(nvinfer1::ITensor const &tensor) const noexcept
Check if a tensor is marked as debug tensor.
Definition: NvInfer.h:6379
bool unmarkDebug(ITensor &tensor) noexcept
Unmark a tensor as a debug tensor.
Definition: NvInfer.h:6369
IEinsumLayer * addEinsum(ITensor *const *inputs, int32_t nbInputs, char const *equation) noexcept
Add an Einsum layer to the network.
Definition: NvInfer.h:7506
void markOutput(ITensor &tensor) noexcept
Mark a tensor as a network output.
Definition: NvInfer.h:6335
IPaddingLayer * addPaddingNd(ITensor &input, Dims const &prePadding, Dims const &postPadding) noexcept
Add a padding layer to the network. Only 2D padding is currently supported.
Definition: NvInfer.h:7328
INonZeroLayer * addNonZero(ITensor &input) noexcept
Add a nonzero layer to the network.
Definition: NvInfer.h:6784
TRT_DEPRECATED IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7407
int32_t getNbOutputs() const noexcept
Get the number of outputs in the network.
Definition: NvInfer.h:6622
bool setWeightsName(Weights weights, char const *name) noexcept
Associate a name with all current uses of the given weights.
Definition: NvInfer.h:7352
Forward declaration of IEngineInspector for use by other interfaces.
Definition: NvInferRuntime.h:51
Definition: NvInfer.h:3733
virtual ~INonZeroLayer() noexcept=default
A normalization layer in a network definition.
Definition: NvInfer.h:6147
float getEpsilon() const noexcept
Get the epsilon value used for the normalization calculation.
Definition: NvInfer.h:6166
uint32_t getAxes() const noexcept
Get the axes value used for the normalization calculation.
Definition: NvInfer.h:6186
virtual ~INormalizationLayer() noexcept=default
void setEpsilon(float eps) noexcept
Set the epsilon value used for the normalization calculation.
Definition: NvInfer.h:6156
DataType getComputePrecision() const noexcept
Get the compute precision of this layer.
Definition: NvInfer.h:6253
apiv::VNormalizationLayer * mImpl
Definition: NvInfer.h:6259
int64_t getNbGroups() const noexcept
Get the number of groups used to split the channels for the normalization calculation.
Definition: NvInfer.h:6217
void setAxes(uint32_t axesMask) noexcept
Set the reduction axes for the normalization calculation.
Definition: NvInfer.h:6176
void setComputePrecision(DataType type) noexcept
Set the compute precision of this layer.
Definition: NvInfer.h:6243
void setNbGroups(int64_t nbGroups) noexcept
Set the number of groups used to split the channels in the normalization calculation.
Definition: NvInfer.h:6207
A OneHot layer in a network definition.
Definition: NvInfer.h:5801
apiv::VOneHotLayer * mImpl
Definition: NvInfer.h:5822
void setAxis(int32_t axis) noexcept
Set the axis parameter.
Definition: NvInfer.h:5808
int32_t getAxis() const noexcept
Get the value of the axis parameter.
Definition: NvInfer.h:5816
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:2517
Layer that represents a padding operation.
Definition: NvInfer.h:2957
Dims getPostPaddingNd() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3006
void setPrePaddingNd(Dims const &padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:2968
virtual ~IPaddingLayer() noexcept=default
void setPostPaddingNd(Dims const &padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:2994
Dims getPrePaddingNd() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:2980
apiv::VPaddingLayer * mImpl
Definition: NvInfer.h:3012
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:3903
apiv::VParametricReLULayer * mImpl
Definition: NvInfer.h:3905
virtual ~IParametricReLULayer() noexcept=default
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:56
Plugin class for user-implemented layers.
Definition: NvInferRuntimePlugin.h:134
Layer type for pluginV2.
Definition: NvInfer.h:2694
virtual ~IPluginV2Layer() noexcept=default
apiv::VPluginV2Layer * mImpl
Definition: NvInfer.h:2707
IPluginV2 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:2701
Layer type for V3 plugins.
Definition: NvInfer.h:2721
virtual ~IPluginV3Layer() noexcept=default
IPluginV3 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:2728
apiv::VPluginV3Layer * mImpl
Definition: NvInfer.h:2734
A Pooling layer in a network definition.
Definition: NvInfer.h:1456
PoolingType getPoolingType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1475
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1608
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:1584
bool getAverageCountExcludesPadding() const noexcept
Get whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1528
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1556
void setPoolingType(PoolingType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1465
void setWindowSizeNd(Dims const &windowSize) noexcept
Set the multi-dimension window size for pooling.
Definition: NvInfer.h:1621
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1597
Dims getWindowSizeNd() const noexcept
Get the multi-dimension window size for pooling.
Definition: NvInfer.h:1631
void setAverageCountExcludesPadding(bool exclusive) noexcept
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1517
void setPaddingNd(Dims const &padding) noexcept
Set the multi-dimension padding for pooling.
Definition: NvInfer.h:1675
float getBlendFactor() const noexcept
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1503
void setStrideNd(Dims const &stride) noexcept
Set the multi-dimension stride for pooling.
Definition: NvInfer.h:1646
Dims getStrideNd() const noexcept
Get the multi-dimension stride for pooling.
Definition: NvInfer.h:1656
virtual ~IPoolingLayer() noexcept=default
Dims getPaddingNd() const noexcept
Get the multi-dimension padding for pooling.
Definition: NvInfer.h:1687
void setPostPadding(Dims const &padding) noexcept
Set the multi-dimension post-padding for pooling.
Definition: NvInfer.h:1574
void setPrePadding(Dims const &padding) noexcept
Set the multi-dimension pre-padding for pooling.
Definition: NvInfer.h:1546
void setBlendFactor(float blendFactor) noexcept
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1490
A Quantize layer in a network definition.
Definition: NvInfer.h:5389
void setToType(DataType toType) noexcept
Set the Quantize layer output type.
Definition: NvInfer.h:5426
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5410
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5399
virtual ~IQuantizeLayer() noexcept=default
DataType getToType() const noexcept
Return the Quantize layer output type.
Definition: NvInfer.h:5438
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:3754
apiv::VRaggedSoftMaxLayer * mImpl
Definition: NvInfer.h:3756
virtual ~IRaggedSoftMaxLayer() noexcept=default
A recurrence layer in a network definition.
Definition: NvInfer.h:4604
virtual ~IRecurrenceLayer() noexcept=default
Layer that represents a reduction across a non-bool tensor.
Definition: NvInfer.h:2877
void setKeepDimensions(bool keepDimensions) noexcept
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:2924
void setOperation(ReduceOperation op) noexcept
Set the reduce operation for the layer.
Definition: NvInfer.h:2884
ReduceOperation getOperation() const noexcept
Get the reduce operation for the layer.
Definition: NvInfer.h:2894
virtual ~IReduceLayer() noexcept=default
uint32_t getReduceAxes() const noexcept
Get the axes over which to reduce for the layer.
Definition: NvInfer.h:2914
void setReduceAxes(uint32_t reduceAxes) noexcept
Set the axes over which to reduce.
Definition: NvInfer.h:2904
apiv::VReduceLayer * mImpl
Definition: NvInfer.h:2940
bool getKeepDimensions() const noexcept
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:2934
A resize layer in a network definition.
Definition: NvInfer.h:4092
void setSelectorForSinglePixel(ResizeSelector selector) noexcept
Set coordinate selector function when resized to single pixel.
Definition: NvInfer.h:4253
void setNearestRounding(ResizeRoundMode value) noexcept
Set rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4277
virtual ~IResizeLayer() noexcept=default
int32_t getScales(int32_t size, float *scales) const noexcept
Copies resize scales to scales[0, ..., nbScales-1], where nbScales is the number of scales that were ...
Definition: NvInfer.h:4171
void setOutputDimensions(Dims const &dimensions) noexcept
Set the output dimensions.
Definition: NvInfer.h:4112
void setCubicCoeff(float A) noexcept
Set the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:4309
void setScales(float const *scales, int32_t nbScales) noexcept
Set the resize scales.
Definition: NvInfer.h:4152
float getCubicCoeff() const noexcept
Get the coefficient 'A' used in cubic interpolation.
Definition: NvInfer.h:4319
ResizeSelector getSelectorForSinglePixel() const noexcept
Get the coordinate selector function when resized to single pixel.
Definition: NvInfer.h:4263
InterpolationMode getResizeMode() const noexcept
Get resize mode for an input tensor.
Definition: NvInfer.h:4193
void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept
Set coordinate transformation function.
Definition: NvInfer.h:4228
void setExcludeOutside(bool excludeFlag) noexcept
Set the state for excluding outside pixels.
Definition: NvInfer.h:4332
void setResizeMode(InterpolationMode interpolationMode) noexcept
Set resize mode for an input tensor.
Definition: NvInfer.h:4183
Dims getOutputDimensions() const noexcept
Get the output dimensions.
Definition: NvInfer.h:4122
ResizeRoundMode getNearestRounding() const noexcept
Get rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4287
bool getExcludeOutside() const noexcept
Get the state for excluding outside pixels.
Definition: NvInfer.h:4342
ResizeCoordinateTransformation getCoordinateTransformation() const noexcept
Get coordinate transformation function.
Definition: NvInfer.h:4238
A ReverseSequence layer in a network definition.
Definition: NvInfer.h:6075
void setSequenceAxis(int32_t sequenceAxis) noexcept
Set the sequence axis. Default is 0.
Definition: NvInfer.h:6108
int32_t getBatchAxis() const noexcept
Return the batch axis. Return 1 if no batch axis was set.
Definition: NvInfer.h:6095
apiv::VReverseSequenceLayer * mImpl
Definition: NvInfer.h:6124
int32_t getSequenceAxis() const noexcept
Return the sequence axis. Return 0 if no sequence axis was set.
Definition: NvInfer.h:6118
void setBatchAxis(int32_t batchAxis) noexcept
Set the batch axis. Default is 1.
Definition: NvInfer.h:6085
virtual ~IReverseSequenceLayer() noexcept=default
A Scale layer in a network definition.
Definition: NvInfer.h:1853
Weights getScale() const noexcept
Get the scale value.
Definition: NvInfer.h:1910
Weights getPower() const noexcept
Get the power value.
Definition: NvInfer.h:1930
void setScale(Weights scale) noexcept
Set the scale value.
Definition: NvInfer.h:1900
void setPower(Weights power) noexcept
Set the power value.
Definition: NvInfer.h:1920
ScaleMode getMode() const noexcept
Get the scale mode.
Definition: NvInfer.h:1870
void setShift(Weights shift) noexcept
Set the shift value.
Definition: NvInfer.h:1880
void setChannelAxis(int32_t channelAxis) noexcept
Set the channel axis.
Definition: NvInfer.h:1966
Weights getShift() const noexcept
Get the shift value.
Definition: NvInfer.h:1890
virtual ~IScaleLayer() noexcept=default
void setMode(ScaleMode mode) noexcept
Set the scale mode.
Definition: NvInfer.h:1860
int32_t getChannelAxis() const noexcept
Get the channel axis.
Definition: NvInfer.h:1945
A scatter layer in a network definition. Supports several kinds of scattering.
Definition: NvInfer.h:5729
void setMode(ScatterMode mode) noexcept
Set the scatter mode.
Definition: NvInfer.h:5736
apiv::VScatterLayer * mImpl
Definition: NvInfer.h:5770
void setAxis(int32_t axis) noexcept
Set the axis used by ScatterMode::kELEMENTS.
Definition: NvInfer.h:5756
int32_t getAxis() const noexcept
Get the axis.
Definition: NvInfer.h:5764
ScatterMode getMode() const noexcept
Get the scatter mode.
Definition: NvInfer.h:5746
virtual ~IScatterLayer() noexcept=default
Select elements from two data tensors based on a condition tensor.
Definition: NvInfer.h:4911
virtual ~ISelectLayer() noexcept=default
Layer type for getting shape of a tensor.
Definition: NvInfer.h:3482
virtual ~IShapeLayer() noexcept=default
apiv::VShapeLayer * mImpl
Definition: NvInfer.h:3484
Layer type for shuffling data.
Definition: NvInfer.h:3045
apiv::VShuffleLayer * mImpl
Definition: NvInfer.h:3203
void setFirstTranspose(Permutation permutation) noexcept
Set the permutation applied by the first transpose operation.
Definition: NvInfer.h:3056
void setSecondTranspose(Permutation permutation) noexcept
Set the permutation applied by the second transpose operation.
Definition: NvInfer.h:3156
Dims getReshapeDimensions() const noexcept
Get the reshaped dimensions.
Definition: NvInfer.h:3109
void setReshapeDimensions(Dims const &dimensions) noexcept
Set the reshaped dimensions.
Definition: NvInfer.h:3096
Permutation getFirstTranspose() const noexcept
Get the permutation applied by the first transpose operation.
Definition: NvInfer.h:3068
virtual ~IShuffleLayer() noexcept=default
Permutation getSecondTranspose() const noexcept
Get the permutation applied by the second transpose operation.
Definition: NvInfer.h:3168
bool getZeroIsPlaceholder() const noexcept
Get meaning of 0 in reshape dimensions.
Definition: NvInfer.h:3197
void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
Set meaning of 0 in reshape dimensions.
Definition: NvInfer.h:3184
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:3297
void setStride(Dims const &stride) noexcept
Set the stride for computing the output slice data.
Definition: NvInfer.h:3366
apiv::VSliceLayer * mImpl
Definition: NvInfer.h:3465
virtual ~ISliceLayer() noexcept=default
void setSize(Dims const &size) noexcept
Set the dimensions of the output slice.
Definition: NvInfer.h:3337
void setAxes(Dims const &axes) noexcept
Set the axes for this ISliceLayer.
Definition: NvInfer.h:3444
void setStart(Dims const &start) noexcept
Set the start offset that the slice layer uses to create the output slice.
Definition: NvInfer.h:3308
Dims getStart() const noexcept
Get the start offset for the slice layer.
Definition: NvInfer.h:3323
void setMode(SampleMode mode) noexcept
Set the slice mode.
Definition: NvInfer.h:3391
Dims getSize() const noexcept
Get dimensions of the output slice.
Definition: NvInfer.h:3352
SampleMode getMode() const noexcept
Get the slice mode.
Definition: NvInfer.h:3401
Dims getStride() const noexcept
Get the stride for the output slice.
Definition: NvInfer.h:3381
Dims getAxes() const noexcept
Get the axes for this ISliceLayer.
Definition: NvInfer.h:3459
A Softmax layer in a network definition.
Definition: NvInfer.h:1997
void setAxes(uint32_t axes) noexcept
Set the axis along which softmax is computed. Currently, only one axis can be set.
Definition: NvInfer.h:2019
uint32_t getAxes() const noexcept
Get the axis along which softmax occurs.
Definition: NvInfer.h:2029
virtual ~ISoftMaxLayer() noexcept=default
A tensor in a network definition.
Definition: NvInfer.h:181
void setAllowedFormats(TensorFormats formats) noexcept
Set allowed formats for an input or output tensor. By default all formats are allowed....
Definition: NvInfer.h:426
TensorLocation getLocation() const noexcept
Get the storage location of a tensor.
Definition: NvInfer.h:345
void setDimensions(Dims const &dimensions) noexcept
Set the dimensions of a tensor.
Definition: NvInfer.h:228
void resetDynamicRange() noexcept
Undo effect of setDynamicRange.
Definition: NvInfer.h:384
void setName(char const *name) noexcept
Set the tensor name.
Definition: NvInfer.h:197
bool isExecutionTensor() const noexcept
Whether the tensor is an execution tensor.
Definition: NvInfer.h:491
void setType(DataType type) noexcept
Set the data type of a tensor.
Definition: NvInfer.h:257
bool dynamicRangeIsSet() const noexcept
Query whether dynamic range is set.
Definition: NvInfer.h:376
char const * getName() const noexcept
Get the tensor name.
Definition: NvInfer.h:209
bool isShapeTensor() const noexcept
Whether the tensor is a shape tensor.
Definition: NvInfer.h:470
float getDynamicRangeMax() const noexcept
Get maximum of dynamic range.
Definition: NvInfer.h:404
bool isNetworkInput() const noexcept
Whether the tensor is a network input.
Definition: NvInfer.h:294
TRT_DEPRECATED void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
Set whether to enable broadcast of tensor across the implicit batch dimension.
Definition: NvInfer.h:319
TRT_DEPRECATED bool setDynamicRange(float min, float max) noexcept
Set dynamic range for the tensor.
Definition: NvInfer.h:286
TRT_DEPRECATED bool getBroadcastAcrossBatch() const noexcept
Check if tensor is broadcast across the implicit batch dimension.
Definition: NvInfer.h:333
bool isNetworkOutput() const noexcept
Whether the tensor is a network output.
Definition: NvInfer.h:302
DataType getType() const noexcept
Get the data type of a tensor.
Definition: NvInfer.h:269
apiv::VTensor * mImpl
Definition: NvInfer.h:538
float getDynamicRangeMin() const noexcept
Get minimum of dynamic range.
Definition: NvInfer.h:394
virtual ~ITensor() noexcept=default
void setDimensionName(int32_t index, char const *name) noexcept
Name a dimension of an input tensor.
Definition: NvInfer.h:517
char const * getDimensionName(int32_t index) const noexcept
Get the name of an input dimension.
Definition: NvInfer.h:532
TRT_DEPRECATED void setLocation(TensorLocation location) noexcept
Set the storage location of a tensor.
Definition: NvInfer.h:364
Dims getDimensions() const noexcept
Get the dimensions of a tensor.
Definition: NvInfer.h:242
TensorFormats getAllowedFormats() const noexcept
Get a bitmask of TensorFormat values that the tensor supports. For a shape tensor,...
Definition: NvInfer.h:439
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:8477
bool combine(ITimingCache const &inputCache, bool ignoreMismatch) noexcept
Combine input timing cache into local instance.
Definition: NvInfer.h:8514
virtual ~ITimingCache() noexcept=default
apiv::VTimingCache * mImpl
Definition: NvInfer.h:8530
bool reset() noexcept
Empty the timing cache.
Definition: NvInfer.h:8524
Layer that represents a TopK reduction.
Definition: NvInfer.h:3522
void setK(int32_t k) noexcept
Set the static k value for the layer.
Definition: NvInfer.h:3553
void setReduceAxes(uint32_t reduceAxes) noexcept
Set which axes to reduce for the layer.
Definition: NvInfer.h:3577
TopKOperation getOperation() const noexcept
Get the operation for the layer.
Definition: NvInfer.h:3539
apiv::VTopKLayer * mImpl
Definition: NvInfer.h:3609
void setOperation(TopKOperation op) noexcept
Set the operation for the layer.
Definition: NvInfer.h:3529
int32_t getK() const noexcept
Get the k value for the layer.
Definition: NvInfer.h:3567
uint32_t getReduceAxes() const noexcept
Get the axes to reduce for the layer.
Definition: NvInfer.h:3587
virtual ~ITopKLayer() noexcept=default
A layer that represents a trip-count limiter.
Definition: NvInfer.h:4725
TripLimit getTripLimit() const noexcept
Get a trip limiter type.
Definition: NvInfer.h:4730
virtual ~ITripLimitLayer() noexcept=default
Layer that represents an unary operation.
Definition: NvInfer.h:2802
void setOperation(UnaryOperation op) noexcept
Set the unary operation for the layer.
Definition: NvInfer.h:2811
apiv::VUnaryLayer * mImpl
Definition: NvInfer.h:2827
UnaryOperation getOperation() const noexcept
Get the unary operation for the layer.
Definition: NvInfer.h:2821
virtual ~IUnaryLayer() noexcept=default
An Interface class for version control.
Definition: NvInferRuntimeBase.h:260
Version information associated with a TRT interface.
Definition: NvInferRuntimeBase.h:225
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:124
Definition: NvInfer.h:8154
virtual int32_t selectAlgorithms(IAlgorithmContext const &context, IAlgorithm const *const *choices, int32_t nbChoices, int32_t *selection) noexcept=0
Select Algorithms for a layer from the given list of algorithm choices.
virtual void reportAlgorithms(IAlgorithmContext const *const *algoContexts, IAlgorithm const *const *algoChoices, int32_t nbAlgorithms) noexcept=0
Called by TensorRT to report choices it made.
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:8159
virtual ~IAlgorithmSelector() noexcept=default
Definition: NvInferRuntimeBase.h:397
Definition: NvInferRuntime.h:1532
~IInt8EntropyCalibrator2() noexcept override=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7802
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7794
Definition: NvInfer.h:7749
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7762
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7754
~IInt8EntropyCalibrator() noexcept override=default
Definition: NvInfer.h:7868
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7881
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7873
virtual double getQuantile() const noexcept=0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
Definition: NvInfer.h:7829
~IInt8MinMaxCalibrator() noexcept override=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7842
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInfer.h:7834
Definition: NvInferPluginBase.h:199
Definition: NvInfer.h:8689
virtual bool stepComplete(char const *phaseName, int32_t step) noexcept=0
Signal that a step of an optimizer phase has finished.
virtual ~IProgressMonitor() noexcept=default
virtual void phaseFinish(char const *phaseName) noexcept=0
Signal that a phase of the optimizer has finished.
virtual void phaseStart(char const *phaseName, char const *parentPhase, int32_t nbSteps) noexcept=0
Signal that a phase of the optimizer has started.
IBuilder * createInferBuilder(ILogger &logger) noexcept
Create an instance of an IBuilder class.
Definition: NvInfer.h:9979
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2733
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:3997
@ kFORMULA
Use formula to map the original index.
@ kUPPER
Select the upper left pixel.
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:76
nvinfer1::IPluginRegistry * getBuilderPluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for building a Standard engine, or nullptr if no registry exists.
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8541
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1810
@ kUNIFORM
Identical coefficients across all elements of the tensor.
@ kCHANNEL
Per-channel coefficients.
RuntimePlatform
Describes the intended runtime platform (operating system and CPU architecture) for the execution of ...
Definition: NvInfer.h:8263
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:8215
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:8655
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:5920
@ kCENTER_SIZES
(x_center, y_center, width, height) where (x_center, y_center) is the center point of the box
@ kCORNER_PAIRS
(x1, y1, x2, y2) where (x1, y1) and (x2, y2) are any pair of diagonal corners
constexpr int32_t EnumMax< BuilderFlag >() noexcept
Definition: NvInfer.h:8458
constexpr int32_t EnumMax< LayerType >() noexcept
Definition: NvInfer.h:114
constexpr int32_t EnumMax< CalibrationAlgoType >() noexcept
Definition: NvInfer.h:7664
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2755
@ kISINF
Return true if input value equals +/- infinity for floating-point data type.
@ kCOSH
Hyperbolic cosine.
@ kACOSH
Inverse hyperbolic cosine.
@ kERF
Gauss error function.
@ kISNAN
Return true if input value is a NaN for floating-point data type.
@ kACOS
Inverse cosine.
@ kABS
Absolute value.
@ kSINH
Hyperbolic sine.
@ kROUND
Round to nearest even for floating-point data type.
@ kATANH
Inverse hyperbolic tangent.
@ kASINH
Inverse hyperbolic sine.
@ kSIGN
Sign, If input > 0, output 1; if input < 0, output -1; if input == 0, output 0.
@ kEXP
Exponentiation.
@ kATAN
Inverse tangent.
constexpr int32_t EnumMax< ReduceOperation >() noexcept
Definition: NvInfer.h:2864
constexpr int32_t EnumMax< TripLimit >() noexcept
Definition: NvInfer.h:4394
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:133
@ kSELU
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
@ kTANH
TanH activation.
@ kSCALED_TANH
Scaled tanh activation: alpha*tanh(beta*x)
@ kRELU
Rectified linear activation.
@ kELU
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
@ kLEAKY_RELU
LeakyRelu activation: x>=0 ? x : alpha * x.
@ kSOFTSIGN
Softsign activation: x / (1+|x|)
@ kHARD_SIGMOID
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
@ kTHRESHOLDED_RELU
Thresholded ReLU activation: x>alpha ? x : 0.
@ kSIGMOID
Sigmoid activation.
@ kCLIP
Clip activation: max(alpha, min(beta, x))
@ kGELU_TANH
GELU tanh activation: 0.5 * x * (1 + tanh(sqrt(2/pi) * (0.044715F * pow(x, 3) + x)))
@ kGELU_ERF
GELU erf activation: 0.5 * x * (1 + erf(sqrt(0.5) * x))
@ kSOFTPLUS
Parametric softplus activation: alpha*log(exp(beta*x)+1)
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:4972
@ kRANDOM_UNIFORM
Randomly draw values from a uniform distribution.
@ kRANDOM_NORMAL
Randomly draw values from a normal distribution.
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4027
@ kHALF_UP
Round half up.
@ kHALF_DOWN
Round half down.
nvinfer1::safe::IPluginRegistry * getBuilderSafePluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for building a Safety engine, or nullptr if no registry exists.
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:988
@ kSAME_LOWER
Use SAME padding, with prePadding >= postPadding.
@ kEXPLICIT_ROUND_DOWN
Use explicit padding, rounding output size down.
@ kEXPLICIT_ROUND_UP
Use explicit padding, rounding output size up.
@ kSAME_UPPER
Use SAME padding, with prePadding <= postPadding.
TripLimit
Definition: NvInfer.h:4382
@ kWHILE
Tensor is a scalar of type kBOOL. Loop terminates when value is false.
@ kCOUNT
Tensor is a scalar of type kINT32 or kINT64 that contains the trip count.
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:9641
PreviewFeature
Define preview features.
Definition: NvInfer.h:8616
constexpr int32_t EnumMax< GatherMode >() noexcept
Definition: NvInfer.h:2516
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:133
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kFP16 ...
Definition: NvInfer.h:8293
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:1227
constexpr int32_t EnumMax< ScaleMode >() noexcept
Definition: NvInfer.h:1822
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7651
@ kENTROPY_CALIBRATION_2
Entropy calibration.
@ kLEGACY_CALIBRATION
Legacy calibration.
@ kENTROPY_CALIBRATION
Legacy entropy calibration.
@ kMINMAX_CALIBRATION
Minmax calibration.
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
@ kGRID_SAMPLE
Grid sample layer.
@ kRAGGED_SOFTMAX
Ragged softmax layer.
@ kDECONVOLUTION
Deconvolution layer.
@ kREDUCE
Reduce layer.
@ kASSERTION
Assertion layer.
@ kTOPK
TopK layer.
@ kRESIZE
Resize Layer.
@ kCAST
Cast layer.
@ kPADDING
Padding layer.
@ kMATRIX_MULTIPLY
Matrix multiply layer.
@ kCONDITION
Condition layer.
@ kCONDITIONAL_INPUT
Conditional Input layer.
@ kIDENTITY
Identity layer.
@ kNORMALIZATION
Normalization layer.
@ kQUANTIZE
Quantize layer.
@ kSCATTER
Scatter layer.
@ kCONVOLUTION
Convolution layer.
@ kPARAMETRIC_RELU
Parametric ReLU layer.
@ kCONCATENATION
Concatenation layer.
@ kONE_HOT
OneHot layer.
@ kREVERSE_SEQUENCE
Reverse sequence layer.
@ kSLICE
Slice layer.
@ kEINSUM
Einsum layer.
@ kSOFTMAX
SoftMax layer.
@ kSHAPE
Shape layer.
@ kRECURRENCE
Loop Recurrence layer.
@ kDEQUANTIZE
Dequantize layer.
@ kSHUFFLE
Shuffle layer.
@ kPLUGIN_V3
PluginV3 layer.
@ kITERATOR
Loop Iterator layer.
@ kPOOLING
Pooling layer.
@ kTRIP_LIMIT
Loop Trip limit layer.
@ kSCALE
Scale layer.
@ kGATHER
Gather layer.
@ kUNARY
UnaryOp operation Layer.
@ kACTIVATION
Activation layer.
@ kELEMENTWISE
Elementwise layer.
@ kSELECT
Select layer.
@ kPLUGIN_V2
PluginV2 layer.
@ kLOOP_OUTPUT
Loop output layer.
@ kCONDITIONAL_OUTPUT
Conditional Output layer.
@ kCONSTANT
Constant layer.
@ kNON_ZERO
NonZero layer.
@ kFILL
Fill layer.
@ kPLUGIN
Plugin layer.
constexpr int32_t EnumMax< QuantizationFlag >() noexcept
Definition: NvInfer.h:8240
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3213
@ kCLAMP
Out of bounds indices are clamped to bounds.
@ kSTRICT_BOUNDS
Fail with error when the coordinates are out of bounds.
@ kWRAP
Coordinates wrap around periodically.
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2504
@ kDEFAULT
Similar to ONNX Gather.
@ kELEMENT
Similar to ONNX GatherElements.
@ kND
Similar to ONNX GatherND.
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:125
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2745
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:9652
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2414
@ kSUB
Subtract the second element from the first.
@ kSUM
Sum of the two elements.
@ kPROD
Product of the two elements.