TensorRT 8.2.1
NvInferImpl.h
1/*
2 * Copyright 1993-2021 NVIDIA Corporation. All rights reserved.
3 *
4 * NOTICE TO LICENSEE:
5 *
6 * This source code and/or documentation ("Licensed Deliverables") are
7 * subject to NVIDIA intellectual property rights under U.S. and
8 * international Copyright laws.
9 *
10 * These Licensed Deliverables contained herein is PROPRIETARY and
11 * CONFIDENTIAL to NVIDIA and is being provided under the terms and
12 * conditions of a form of NVIDIA software license agreement by and
13 * between NVIDIA and Licensee ("License Agreement") or electronically
14 * accepted by Licensee. Notwithstanding any terms or conditions to
15 * the contrary in the License Agreement, reproduction or disclosure
16 * of the Licensed Deliverables to any third party without the express
17 * written consent of NVIDIA is prohibited.
18 *
19 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
20 * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
21 * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS
22 * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
23 * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
24 * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
25 * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
26 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
27 * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
28 * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
29 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
30 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
31 * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
32 * OF THESE LICENSED DELIVERABLES.
33 *
34 * U.S. Government End Users. These Licensed Deliverables are a
35 * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
36 * 1995), consisting of "commercial computer software" and "commercial
37 * computer software documentation" as such terms are used in 48
38 * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
39 * only as a commercial end item. Consistent with 48 C.F.R.12.212 and
40 * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
41 * U.S. Government End Users acquire the Licensed Deliverables with
42 * only those rights set forth herein.
43 *
44 * Any use of the Licensed Deliverables in individual and commercial
45 * software must include, in the user documentation and internal
46 * comments to the code, the above Disclaimer and U.S. Government End
47 * Users Notice.
48 */
49
50#ifndef NV_INFER_IMPL_H
51#define NV_INFER_IMPL_H
52
53#include "NvInferLegacyDims.h"
54#include "NvInferRuntimeCommon.h"
55
56// @cond SuppressDoxyWarnings
57
58namespace nvinfer1
59{
60
61class IActivationLayer;
62class IAlgorithm;
63class IAlgorithmContext;
64class IAlgorithmIOInfo;
65class IAlgorithmSelector;
66class IAlgorithmVariant;
67class IAssertionLayer;
68class IBuilderConfig;
69class IConcatenationLayer;
70class IIfConditional;
71class IConditionLayer;
72class IIfConditionalOutputLayer;
73class IIfConditionalInputLayer;
74class IConstantLayer;
75class IConvolutionLayer;
76class ICudaEngine;
77class IDeconvolutionLayer;
78class IDequantizeLayer;
79class IDimensionExpr;
80class IEinsumLayer;
81class IElementWiseLayer;
82class IExecutionContext;
83class IFillLayer;
84class IFullyConnectedLayer;
85class IGatherLayer;
86class IHostMemory;
87class IIdentityLayer;
88class IIfConditional;
89class IInt8Calibrator;
90class IIteratorLayer;
91class ILayer;
92class ILoop;
93class ILoopOutputLayer;
94class ILRNLayer;
95class IMatrixMultiplyLayer;
96class INetworkDefinition;
97class IOptimizationProfile;
98class IPaddingLayer;
99class IParametricReLULayer;
100class IPlugin;
101class IPluginExt;
102class IPluginFactory;
103class IPluginLayer;
104class IPluginV2Layer;
105class IPoolingLayer;
106class IProfiler;
107class IQuantizeLayer;
108class IRaggedSoftMaxLayer;
109class IRecurrenceLayer;
110class IReduceLayer;
111class IResizeLayer;
112class IRNNv2Layer;
113class IScaleLayer;
114class IScatterLayer;
115class ISelectLayer;
116class IShapeLayer;
117class IShuffleLayer;
118class ISliceLayer;
119class ISoftMaxLayer;
120class IEngineInspector;
121class ITensor;
122class ITimingCache;
123class ITopKLayer;
124class ITripLimitLayer;
125class IUnaryLayer;
126struct Permutation;
127class Weights;
128
129enum class ActivationType : int32_t;
130enum class BuilderFlag : int32_t;
131enum class CalibrationAlgoType : int32_t;
132enum class DeviceType : int32_t;
133enum class DimensionOperation : int32_t;
134enum class ElementWiseOperation : int32_t;
135enum class EngineCapability : int32_t;
136enum class FillOperation : int32_t;
137enum class GatherMode : int32_t;
138enum class LayerInformationFormat : int32_t;
139enum class LayerType : int32_t;
140enum class LoopOutput : int32_t;
141enum class MatrixOperation : int32_t;
142enum class NetworkDefinitionCreationFlag : int32_t;
143enum class OptProfileSelector : int32_t;
144enum class PaddingMode : int32_t;
145enum class PoolingType : int32_t;
146enum class ProfilingVerbosity : int32_t;
147enum class QuantizationFlag : int32_t;
148enum class ReduceOperation : int32_t;
149enum class ResizeCoordinateTransformation : int32_t;
150enum class ResizeMode : int32_t;
151enum class ResizeRoundMode : int32_t;
152enum class ResizeSelector : int32_t;
153enum class RNNDirection : int32_t;
154enum class RNNGateType : int32_t;
155enum class RNNInputMode : int32_t;
156enum class RNNOperation : int32_t;
157enum class ScaleMode : int32_t;
158enum class ScatterMode : int32_t;
159enum class SliceMode : int32_t;
160enum class TensorLocation : int32_t;
161enum class TopKOperation : int32_t;
162enum class TripLimit : int32_t;
163enum class UnaryOperation : int32_t;
164enum class WeightsRole : int32_t;
165
166using TacticSources = uint32_t;
167using TensorFormats = uint32_t;
168using BuilderFlags = uint32_t;
169using NetworkDefinitionCreationFlags = uint32_t;
170using QuantizationFlags = uint32_t;
171
179
180namespace apiv
181{
182
183class VRoot
184{
185public:
186 virtual ~VRoot() noexcept = default;
187};
188
189class VHostMemory : public VRoot
190{
191public:
192 virtual void* data() const noexcept = 0;
193 virtual std::size_t size() const noexcept = 0;
194 virtual DataType type() const noexcept = 0;
195};
196
197class VDimensionExpr : public VRoot
198{
199public:
200 virtual bool isConstant() const = 0;
201 virtual int32_t getConstantValue() const = 0;
202};
203
204class VExprBuilder : public VRoot
205{
206public:
207 virtual const IDimensionExpr* constant(int32_t value) = 0;
208 virtual const IDimensionExpr* operation(
209 DimensionOperation op, const IDimensionExpr& first, const IDimensionExpr& second)
210 = 0;
211};
212
213class VRuntime : public VRoot
214{
215public:
216 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(
217 const void* blob, std::size_t size, IPluginFactory* pluginFactory) noexcept
218 = 0;
219 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
220 virtual int32_t getDLACore() const noexcept = 0;
221 virtual int32_t getNbDLACores() const noexcept = 0;
222 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
223 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
224 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
225 virtual ILogger* getLogger() const noexcept = 0;
226};
227
228class VRefitter : public VRoot
229{
230public:
231 virtual bool setWeights(const char* layerName, WeightsRole role, const Weights weights) noexcept = 0;
232 virtual bool refitCudaEngine() noexcept = 0;
233 virtual int32_t getMissing(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
234 virtual int32_t getAll(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
235 virtual bool setDynamicRange(const char* tensorName, float min, float max) noexcept = 0;
236 virtual float getDynamicRangeMin(const char* tensorName) const noexcept = 0;
237 virtual float getDynamicRangeMax(const char* tensorName) const noexcept = 0;
238 virtual int32_t getTensorsWithDynamicRange(int32_t size, const char** tensorNames) const noexcept = 0;
239 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
240 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
241 virtual bool setNamedWeights(const char* name, Weights weights) noexcept = 0;
242 virtual int32_t getMissingWeights(int32_t size, const char** weightsNames) noexcept = 0;
243 virtual int32_t getAllWeights(int32_t size, const char** weightsNames) noexcept = 0;
244 virtual ILogger* getLogger() const noexcept = 0;
245};
246
247class VOptimizationProfile : public VRoot
248{
249public:
250 virtual bool setDimensions(const char* inputName, OptProfileSelector select, Dims dims) noexcept = 0;
251 virtual Dims getDimensions(const char* inputName, OptProfileSelector select) const noexcept = 0;
252 virtual bool setShapeValues(
253 const char* inputName, OptProfileSelector select, const int32_t* values, int32_t nbValues) noexcept
254 = 0;
255 virtual int32_t getNbShapeValues(const char* inputName) const noexcept = 0;
256 virtual int32_t const* getShapeValues(const char* inputName, OptProfileSelector select) const noexcept = 0;
257 virtual bool setExtraMemoryTarget(float target) noexcept = 0;
258 virtual float getExtraMemoryTarget() const noexcept = 0;
259 virtual bool isValid() const noexcept = 0;
260};
261
262class VCudaEngine : public VRoot
263{
264public:
265 virtual int32_t getNbBindings() const noexcept = 0;
266 virtual int32_t getBindingIndex(const char* name) const noexcept = 0;
267 virtual const char* getBindingName(int32_t bindingIndex) const noexcept = 0;
268 virtual bool bindingIsInput(int32_t bindingIndex) const noexcept = 0;
269 virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
270 virtual DataType getBindingDataType(int32_t bindingIndex) const noexcept = 0;
271 virtual int32_t getMaxBatchSize() const noexcept = 0;
272 virtual int32_t getNbLayers() const noexcept = 0;
273 virtual IHostMemory* serialize() const noexcept = 0;
274 virtual IExecutionContext* createExecutionContext() noexcept = 0;
275 virtual TensorLocation getLocation(int32_t bindingIndex) const noexcept = 0;
276 virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
277 virtual size_t getDeviceMemorySize() const noexcept = 0;
278 virtual bool isRefittable() const noexcept = 0;
279 virtual int32_t getBindingBytesPerComponent(int32_t bindingIndex) const noexcept = 0;
280 virtual int32_t getBindingComponentsPerElement(int32_t bindingIndex) const noexcept = 0;
281 virtual TensorFormat getBindingFormat(int32_t bindingIndex) const noexcept = 0;
282 virtual const char* getBindingFormatDesc(int32_t bindingIndex) const noexcept = 0;
283 virtual int32_t getBindingVectorizedDim(int32_t bindingIndex) const noexcept = 0;
284 virtual const char* getName() const noexcept = 0;
285 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
286 virtual Dims getProfileDimensions(int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const
287 noexcept
288 = 0;
289 virtual const int32_t* getProfileShapeValues(
290 int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
291 = 0;
292 virtual bool isShapeBinding(int32_t bindingIndex) const noexcept = 0;
293 virtual bool isExecutionBinding(int32_t bindingIndex) const noexcept = 0;
294 virtual EngineCapability getEngineCapability() const noexcept = 0;
295 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
296 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
297 virtual bool hasImplicitBatchDimension() const noexcept = 0;
298 virtual TacticSources getTacticSources() const noexcept = 0;
299 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
300 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
301};
302
303class VExecutionContext : public VRoot
304{
305public:
306 virtual bool execute(int32_t batchSize, void* const* bindings) noexcept = 0;
307 virtual bool enqueue(
308 int32_t batchSize, void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept
309 = 0;
310 virtual void setDebugSync(bool sync) noexcept = 0;
311 virtual bool getDebugSync() const noexcept = 0;
312 virtual void setProfiler(IProfiler*) noexcept = 0;
313 virtual IProfiler* getProfiler() const noexcept = 0;
314 virtual const ICudaEngine& getEngine() const noexcept = 0;
315 virtual void setName(const char* name) noexcept = 0;
316 virtual const char* getName() const noexcept = 0;
317 virtual void setDeviceMemory(void* memory) noexcept = 0;
318 virtual Dims getStrides(int32_t bindingIndex) const noexcept = 0;
319 virtual bool setOptimizationProfile(int32_t profileIndex) noexcept = 0;
320 virtual int32_t getOptimizationProfile() const noexcept = 0;
321 virtual bool setBindingDimensions(int32_t bindingIndex, Dims dimensions) noexcept = 0;
322 virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
323 virtual bool setInputShapeBinding(int32_t bindingIndex, int32_t const* data) noexcept = 0;
324 virtual bool getShapeBinding(int32_t bindingIndex, int32_t* data) const noexcept = 0;
325 virtual bool allInputDimensionsSpecified() const noexcept = 0;
326 virtual bool allInputShapesSpecified() const noexcept = 0;
327 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
328 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
329 virtual bool executeV2(void* const* bindings) noexcept = 0;
330 virtual bool enqueueV2(void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept = 0;
331 virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
332 virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
333 virtual bool getEnqueueEmitsProfile() const noexcept = 0;
334 virtual bool reportToProfiler() const noexcept = 0;
335};
336
337class VEngineInspector : public VRoot
338{
339public:
340 virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
341 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
342 virtual AsciiChar const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) noexcept = 0;
343 virtual AsciiChar const* getEngineInformation(LayerInformationFormat format) noexcept = 0;
344 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
345 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
346};
347
348class VTensor : public VRoot
349{
350public:
351 virtual void setName(const char* name) noexcept = 0;
352 virtual const char* getName() const noexcept = 0;
353 virtual void setDimensions(Dims dimensions) noexcept = 0;
354 virtual Dims getDimensions() const noexcept = 0;
355 virtual void setType(DataType type) noexcept = 0;
356 virtual DataType getType() const noexcept = 0;
357 virtual bool setDynamicRange(float min, float max) noexcept = 0;
358 virtual bool isNetworkInput() const noexcept = 0;
359 virtual bool isNetworkOutput() const noexcept = 0;
360 virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
361 virtual bool getBroadcastAcrossBatch() const noexcept = 0;
362 virtual TensorLocation getLocation() const noexcept = 0;
363 virtual void setLocation(TensorLocation location) noexcept = 0;
364 virtual bool dynamicRangeIsSet() const noexcept = 0;
365 virtual void resetDynamicRange() noexcept = 0;
366 virtual float getDynamicRangeMin() const noexcept = 0;
367 virtual float getDynamicRangeMax() const noexcept = 0;
368 virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
369 virtual TensorFormats getAllowedFormats() const noexcept = 0;
370 virtual bool isShapeTensor() const noexcept = 0;
371 virtual bool isExecutionTensor() const noexcept = 0;
372};
373class VLayer : public VRoot
374{
375public:
376 virtual LayerType getType() const noexcept = 0;
377 virtual void setName(const char* name) noexcept = 0;
378 virtual const char* getName() const noexcept = 0;
379 virtual int32_t getNbInputs() const noexcept = 0;
380 virtual ITensor* getInput(int32_t index) const noexcept = 0;
381 virtual int32_t getNbOutputs() const noexcept = 0;
382 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
383 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
384 virtual void setPrecision(DataType dataType) noexcept = 0;
385 virtual DataType getPrecision() const noexcept = 0;
386 virtual bool precisionIsSet() const noexcept = 0;
387 virtual void resetPrecision() noexcept = 0;
388 virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
389 virtual DataType getOutputType(int32_t index) const noexcept = 0;
390 virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
391 virtual void resetOutputType(int32_t index) noexcept = 0;
392};
393
394class VConvolutionLayer : public VRoot
395{
396public:
397 virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
398 virtual DimsHW getKernelSize() const noexcept = 0;
399 virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
400 virtual int32_t getNbOutputMaps() const noexcept = 0;
401 virtual void setStride(DimsHW stride) noexcept = 0;
402 virtual DimsHW getStride() const noexcept = 0;
403 virtual void setPadding(DimsHW padding) noexcept = 0;
404 virtual DimsHW getPadding() const noexcept = 0;
405 virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
406 virtual int32_t getNbGroups() const noexcept = 0;
407 virtual void setKernelWeights(Weights weights) noexcept = 0;
408 virtual Weights getKernelWeights() const noexcept = 0;
409 virtual void setBiasWeights(Weights weights) noexcept = 0;
410 virtual Weights getBiasWeights() const noexcept = 0;
411 virtual void setDilation(DimsHW dilation) noexcept = 0;
412 virtual DimsHW getDilation() const noexcept = 0;
413 virtual void setPrePadding(Dims padding) noexcept = 0;
414 virtual Dims getPrePadding() const noexcept = 0;
415 virtual void setPostPadding(Dims padding) noexcept = 0;
416 virtual Dims getPostPadding() const noexcept = 0;
417 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
418 virtual PaddingMode getPaddingMode() const noexcept = 0;
419 virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
420 virtual Dims getKernelSizeNd() const noexcept = 0;
421 virtual void setStrideNd(Dims stride) noexcept = 0;
422 virtual Dims getStrideNd() const noexcept = 0;
423 virtual void setPaddingNd(Dims padding) noexcept = 0;
424 virtual Dims getPaddingNd() const noexcept = 0;
425 virtual void setDilationNd(Dims dilation) noexcept = 0;
426 virtual Dims getDilationNd() const noexcept = 0;
427};
428
429class VFullyConnectedLayer : public VRoot
430{
431public:
432 virtual void setNbOutputChannels(int32_t nbOutputs) noexcept = 0;
433 virtual int32_t getNbOutputChannels() const noexcept = 0;
434 virtual void setKernelWeights(Weights weights) noexcept = 0;
435 virtual Weights getKernelWeights() const noexcept = 0;
436 virtual void setBiasWeights(Weights weights) noexcept = 0;
437 virtual Weights getBiasWeights() const noexcept = 0;
438};
439
440class VActivationLayer : public VRoot
441{
442public:
443 virtual void setActivationType(ActivationType type) noexcept = 0;
444 virtual ActivationType getActivationType() const noexcept = 0;
445 virtual void setAlpha(float alpha) noexcept = 0;
446 virtual void setBeta(float beta) noexcept = 0;
447 virtual float getAlpha() const noexcept = 0;
448 virtual float getBeta() const noexcept = 0;
449};
450
451class VPoolingLayer : public VRoot
452{
453public:
454 virtual void setPoolingType(PoolingType type) noexcept = 0;
455 virtual PoolingType getPoolingType() const noexcept = 0;
456 virtual void setWindowSize(DimsHW windowSize) noexcept = 0;
457 virtual DimsHW getWindowSize() const noexcept = 0;
458 virtual void setStride(DimsHW stride) noexcept = 0;
459 virtual DimsHW getStride() const noexcept = 0;
460 virtual void setPadding(DimsHW padding) noexcept = 0;
461 virtual DimsHW getPadding() const noexcept = 0;
462 virtual void setBlendFactor(float blendFactor) noexcept = 0;
463 virtual float getBlendFactor() const noexcept = 0;
464 virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
465 virtual bool getAverageCountExcludesPadding() const noexcept = 0;
466 virtual void setPrePadding(Dims padding) noexcept = 0;
467 virtual Dims getPrePadding() const noexcept = 0;
468 virtual void setPostPadding(Dims padding) noexcept = 0;
469 virtual Dims getPostPadding() const noexcept = 0;
470 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
471 virtual PaddingMode getPaddingMode() const noexcept = 0;
472 virtual void setWindowSizeNd(Dims windowSize) noexcept = 0;
473 virtual Dims getWindowSizeNd() const noexcept = 0;
474 virtual void setStrideNd(Dims stride) noexcept = 0;
475 virtual Dims getStrideNd() const noexcept = 0;
476 virtual void setPaddingNd(Dims padding) noexcept = 0;
477 virtual Dims getPaddingNd() const noexcept = 0;
478};
479
480class VLRNLayer : public VRoot
481{
482public:
483 virtual void setWindowSize(int32_t windowSize) noexcept = 0;
484 virtual int32_t getWindowSize() const noexcept = 0;
485 virtual void setAlpha(float alpha) noexcept = 0;
486 virtual float getAlpha() const noexcept = 0;
487 virtual void setBeta(float beta) noexcept = 0;
488 virtual float getBeta() const noexcept = 0;
489 virtual void setK(float k) noexcept = 0;
490 virtual float getK() const noexcept = 0;
491};
492
493class VScaleLayer : public VRoot
494{
495public:
496 virtual void setMode(ScaleMode mode) noexcept = 0;
497 virtual ScaleMode getMode() const noexcept = 0;
498 virtual void setShift(Weights shift) noexcept = 0;
499 virtual Weights getShift() const noexcept = 0;
500 virtual void setScale(Weights scale) noexcept = 0;
501 virtual Weights getScale() const noexcept = 0;
502 virtual void setPower(Weights power) noexcept = 0;
503 virtual Weights getPower() const noexcept = 0;
504 virtual int32_t getChannelAxis() const noexcept = 0;
505 virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
506};
507
508class VSoftMaxLayer : public VRoot
509{
510public:
511 virtual void setAxes(uint32_t axes) noexcept = 0;
512 virtual uint32_t getAxes() const noexcept = 0;
513};
514
515class VConcatenationLayer : public VRoot
516{
517public:
518 virtual void setAxis(int32_t axis) noexcept = 0;
519 virtual int32_t getAxis() const noexcept = 0;
520};
521
522class VDeconvolutionLayer : public VRoot
523{
524public:
525 virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
526 virtual DimsHW getKernelSize() const noexcept = 0;
527 virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
528 virtual int32_t getNbOutputMaps() const noexcept = 0;
529 virtual void setStride(DimsHW stride) noexcept = 0;
530 virtual DimsHW getStride() const noexcept = 0;
531 virtual void setPadding(DimsHW padding) noexcept = 0;
532 virtual DimsHW getPadding() const noexcept = 0;
533 virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
534 virtual int32_t getNbGroups() const noexcept = 0;
535 virtual void setKernelWeights(Weights weights) noexcept = 0;
536 virtual Weights getKernelWeights() const noexcept = 0;
537 virtual void setBiasWeights(Weights weights) noexcept = 0;
538 virtual Weights getBiasWeights() const noexcept = 0;
539 virtual void setPrePadding(Dims padding) noexcept = 0;
540 virtual Dims getPrePadding() const noexcept = 0;
541 virtual void setPostPadding(Dims padding) noexcept = 0;
542 virtual Dims getPostPadding() const noexcept = 0;
543 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
544 virtual PaddingMode getPaddingMode() const noexcept = 0;
545 virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
546 virtual Dims getKernelSizeNd() const noexcept = 0;
547 virtual void setStrideNd(Dims stride) noexcept = 0;
548 virtual Dims getStrideNd() const noexcept = 0;
549 virtual void setPaddingNd(Dims padding) noexcept = 0;
550 virtual Dims getPaddingNd() const noexcept = 0;
551 virtual void setDilationNd(Dims dilation) noexcept = 0;
552 virtual Dims getDilationNd() const noexcept = 0;
553};
554
555class VElementWiseLayer : public VRoot
556{
557public:
558 virtual void setOperation(ElementWiseOperation op) noexcept = 0;
559 virtual ElementWiseOperation getOperation() const noexcept = 0;
560};
561
562class VGatherLayer : public VRoot
563{
564public:
565 virtual void setGatherAxis(int32_t axis) noexcept = 0;
566 virtual int32_t getGatherAxis() const noexcept = 0;
567 virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
568 virtual int32_t getNbElementWiseDims() const noexcept = 0;
569 virtual void setMode(GatherMode mode) noexcept = 0;
570 virtual GatherMode getMode() const noexcept = 0;
571};
572
573class VRNNv2Layer : public VRoot
574{
575public:
576 virtual int32_t getLayerCount() const noexcept = 0;
577 virtual int32_t getHiddenSize() const noexcept = 0;
578 virtual int32_t getMaxSeqLength() const noexcept = 0;
579 virtual int32_t getDataLength() const noexcept = 0;
580 virtual void setSequenceLengths(ITensor& seqLengths) noexcept = 0;
581 virtual ITensor* getSequenceLengths() const noexcept = 0;
582 virtual void setOperation(RNNOperation op) noexcept = 0;
583 virtual RNNOperation getOperation() const noexcept = 0;
584 virtual void setInputMode(RNNInputMode op) noexcept = 0;
585 virtual RNNInputMode getInputMode() const noexcept = 0;
586 virtual void setDirection(RNNDirection op) noexcept = 0;
587 virtual RNNDirection getDirection() const noexcept = 0;
588 virtual void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept = 0;
589 virtual Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
590 virtual void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept = 0;
591 virtual Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
592 virtual void setHiddenState(ITensor& hidden) noexcept = 0;
593 virtual ITensor* getHiddenState() const noexcept = 0;
594 virtual void setCellState(ITensor& cell) noexcept = 0;
595 virtual ITensor* getCellState() const noexcept = 0;
596};
597
598class VPluginLayer : public VRoot
599{
600public:
601 virtual IPlugin& getPlugin() noexcept = 0;
602};
603
604class VPluginV2Layer : public VRoot
605{
606public:
607 virtual IPluginV2& getPlugin() noexcept = 0;
608};
609
610class VUnaryLayer : public VRoot
611{
612public:
613 virtual void setOperation(UnaryOperation op) noexcept = 0;
614 virtual UnaryOperation getOperation() const noexcept = 0;
615};
616
617class VReduceLayer : public VRoot
618{
619public:
620 virtual void setOperation(ReduceOperation op) noexcept = 0;
621 virtual ReduceOperation getOperation() const noexcept = 0;
622 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
623 virtual uint32_t getReduceAxes() const noexcept = 0;
624 virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
625 virtual bool getKeepDimensions() const noexcept = 0;
626};
627
628class VPaddingLayer : public VRoot
629{
630public:
631 virtual void setPrePadding(DimsHW padding) noexcept = 0;
632 virtual DimsHW getPrePadding() const noexcept = 0;
633 virtual void setPostPadding(DimsHW padding) noexcept = 0;
634 virtual DimsHW getPostPadding() const noexcept = 0;
635 virtual void setPrePaddingNd(Dims padding) noexcept = 0;
636 virtual Dims getPrePaddingNd() const noexcept = 0;
637 virtual void setPostPaddingNd(Dims padding) noexcept = 0;
638 virtual Dims getPostPaddingNd() const noexcept = 0;
639};
640
641class VShuffleLayer : public VRoot
642{
643public:
644 virtual void setFirstTranspose(const Permutation& permutation) noexcept = 0;
645 virtual const Permutation& getFirstTranspose() const noexcept = 0;
646 virtual void setReshapeDimensions(Dims dimensions) noexcept = 0;
647 virtual Dims getReshapeDimensions() const noexcept = 0;
648 virtual void setSecondTranspose(const Permutation& permutation) noexcept = 0;
649 virtual const Permutation& getSecondTranspose() const noexcept = 0;
650 virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) = 0;
651 virtual bool getZeroIsPlaceholder() const = 0;
652};
653
654class VSliceLayer : public VRoot
655{
656public:
657 virtual void setStart(Dims start) noexcept = 0;
658 virtual Dims getStart() const noexcept = 0;
659 virtual void setSize(Dims size) noexcept = 0;
660 virtual Dims getSize() const noexcept = 0;
661 virtual void setStride(Dims stride) noexcept = 0;
662 virtual Dims getStride() const noexcept = 0;
663 virtual void setMode(SliceMode mode) noexcept = 0;
664 virtual SliceMode getMode() const noexcept = 0;
665};
666
667class VShapeLayer : public VRoot
668{
669public:
670};
671
672class VTopKLayer : public VRoot
673{
674public:
675 virtual void setOperation(TopKOperation op) noexcept = 0;
676 virtual TopKOperation getOperation() const noexcept = 0;
677 virtual void setK(int32_t k) noexcept = 0;
678 virtual int32_t getK() const noexcept = 0;
679 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
680 virtual uint32_t getReduceAxes() const noexcept = 0;
681};
682
683class VMatrixMultiplyLayer : public VRoot
684{
685public:
686 virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
687 virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
688};
689
690class VRaggedSoftMaxLayer : public VRoot
691{
692public:
693};
694
695class VIdentityLayer : public VRoot
696{
697public:
698};
699
700class VConstantLayer : public VRoot
701{
702public:
703 virtual void setWeights(Weights weights) noexcept = 0;
704 virtual Weights getWeights() const noexcept = 0;
705 virtual void setDimensions(Dims dimensions) noexcept = 0;
706 virtual Dims getDimensions() const noexcept = 0;
707};
708
709class VParametricReLULayer : public VRoot
710{
711public:
712};
713
714class VResizeLayer : public VRoot
715{
716public:
717 virtual void setOutputDimensions(Dims dimensions) noexcept = 0;
718 virtual Dims getOutputDimensions() const noexcept = 0;
719 virtual void setScales(const float* scales, int32_t nbScales) noexcept = 0;
720 virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
721 virtual void setResizeMode(ResizeMode resizeMode) noexcept = 0;
722 virtual ResizeMode getResizeMode() const noexcept = 0;
723 virtual void setAlignCorners(bool alignCorners) noexcept = 0;
724 virtual bool getAlignCorners() const noexcept = 0;
725 virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
726 virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
727 virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
728 virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
729 virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
730 virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
731};
732
733class VLoopBoundaryLayer : public VRoot
734{
735public:
736 virtual ILoop* getLoop() const noexcept = 0;
737};
738
739class VRecurrenceLayer : public VRoot
740{
741public:
742};
743
744class VLoopOutputLayer : public VRoot
745{
746public:
747 virtual LoopOutput getLoopOutput() const noexcept = 0;
748 virtual void setAxis(int32_t axis) noexcept = 0;
749 virtual int32_t getAxis() const noexcept = 0;
750};
751
752class VTripLimitLayer : public VRoot
753{
754public:
755 virtual TripLimit getTripLimit() const noexcept = 0;
756};
757
758class VIteratorLayer : public VRoot
759{
760public:
761 virtual void setAxis(int32_t axis) noexcept = 0;
762 virtual int32_t getAxis() const noexcept = 0;
763 virtual void setReverse(bool reverse) noexcept = 0;
764 virtual bool getReverse() const noexcept = 0;
765};
766class VLoop : public VRoot
767{
768public:
769 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
770 virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
771 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
772 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
773 virtual void setName(const char* name) noexcept = 0;
774 virtual const char* getName() const noexcept = 0;
775};
776
777class VConditionalBoundaryLayer : public VRoot
778{
779public:
780 virtual IIfConditional* getConditional() const noexcept = 0;
781};
782
783class VConditionLayer : public VRoot
784{
785public:
786};
787
788class VConditionalInputLayer : public VRoot
789{
790public:
791};
792
793class VConditionalOutputLayer : public VRoot
794{
795public:
796};
797
798class VIfConditional : public VRoot
799{
800public:
801 virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
802 virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
803 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
804 virtual void setName(const char* name) noexcept = 0;
805 virtual const char* getName() const noexcept = 0;
806};
807
808class VSelectLayer : public VRoot
809{
810};
811
812class VAssertionLayer : public VRoot
813{
814public:
815 virtual void setMessage(const char* message) noexcept = 0;
816 virtual const char* getMessage() const noexcept = 0;
817};
818
819class VFillLayer : public VRoot
820{
821public:
822 virtual void setDimensions(Dims dimensions) noexcept = 0;
823 virtual Dims getDimensions() const noexcept = 0;
824 virtual void setOperation(FillOperation op) noexcept = 0;
825 virtual FillOperation getOperation() const noexcept = 0;
826 virtual void setAlpha(double alpha) noexcept = 0;
827 virtual double getAlpha() const noexcept = 0;
828 virtual void setBeta(double beta) noexcept = 0;
829 virtual double getBeta() const noexcept = 0;
830};
831
832class VQuantizeLayer : public VRoot
833{
834public:
835 virtual int32_t getAxis() const noexcept = 0;
836 virtual void setAxis(int32_t axis) noexcept = 0;
837};
838
839class VDequantizeLayer : public VRoot
840{
841public:
842 virtual int32_t getAxis() const noexcept = 0;
843 virtual void setAxis(int32_t axis) noexcept = 0;
844};
845
846class VScatterLayer : public VRoot
847{
848public:
849 virtual void setMode(ScatterMode mode) noexcept = 0;
850 virtual ScatterMode getMode() const noexcept = 0;
851 virtual void setAxis(int32_t axis) noexcept = 0;
852 virtual int32_t getAxis() const noexcept = 0;
853}; // class VScatterLayer
854
855class VEinsumLayer : public VRoot
856{
857public:
858 virtual bool setEquation(const char* equation) noexcept = 0;
859 virtual const char* getEquation() const noexcept = 0;
860};
861
862class VNetworkDefinition : public VRoot
863{
864public:
865 virtual ITensor* addInput(const char* name, DataType type, Dims dimensions) noexcept = 0;
866 virtual void markOutput(ITensor& tensor) noexcept = 0;
867 virtual IConvolutionLayer* addConvolution(
868 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
869 = 0;
870 virtual IFullyConnectedLayer* addFullyConnected(
871 ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
872 = 0;
873 virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
874 virtual IPoolingLayer* addPooling(ITensor& input, PoolingType type, DimsHW windowSize) noexcept = 0;
875 virtual ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept = 0;
876 virtual IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
877 = 0;
878 virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
879 virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
880 virtual IDeconvolutionLayer* addDeconvolution(
881 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
882 = 0;
883 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
884 virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
885 virtual IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept = 0;
886 virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
887 virtual int32_t getNbLayers() const noexcept = 0;
888 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
889 virtual int32_t getNbInputs() const noexcept = 0;
890 virtual ITensor* getInput(int32_t index) const noexcept = 0;
891 virtual int32_t getNbOutputs() const noexcept = 0;
892 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
893 virtual IReduceLayer* addReduce(
894 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
895 = 0;
896 virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
897 virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
898 virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
899 virtual IMatrixMultiplyLayer* addMatrixMultiply(
900 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
901 = 0;
902 virtual IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept = 0;
903 virtual IRNNv2Layer* addRNNv2(
904 ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
905 = 0;
906 virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
907 virtual void removeTensor(ITensor& tensor) noexcept = 0;
908 virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
909 virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
910 virtual ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept = 0;
911 virtual void setName(const char* name) noexcept = 0;
912 virtual const char* getName() const noexcept = 0;
913 virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
914 virtual bool hasImplicitBatchDimension() const noexcept = 0;
915 virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
916 virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
917 virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
918 virtual IConvolutionLayer* addConvolutionNd(
919 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
920 = 0;
921 virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept = 0;
922 virtual IDeconvolutionLayer* addDeconvolutionNd(
923 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
924 = 0;
925 virtual IScaleLayer* addScaleNd(
926 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
927 = 0;
928 virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
929 virtual bool hasExplicitPrecision() const noexcept = 0;
930 virtual ILoop* addLoop() noexcept = 0;
931 virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
932 virtual IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept = 0;
933 virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept = 0;
934 virtual bool setWeightsName(Weights weights, const char* name) noexcept = 0;
935 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
936 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
937 virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
938 virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
939 virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept = 0;
940 virtual IIfConditional* addIfConditional() noexcept = 0;
941 virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept = 0;
942 virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, const char* equation) noexcept = 0;
943 virtual IAssertionLayer* addAssertion(ITensor& condition, const char* message) noexcept = 0;
944};
945
946class VAlgorithmIOInfo : public VRoot
947{
948public:
949 virtual TensorFormat getTensorFormat() const noexcept = 0;
950 virtual DataType getDataType() const noexcept = 0;
951 virtual Dims getStrides() const noexcept = 0;
952};
953
954class VAlgorithmVariant : public VRoot
955{
956public:
957 virtual int64_t getImplementation() const noexcept = 0;
958 virtual int64_t getTactic() const noexcept = 0;
959};
960
961class VAlgorithmContext : public VRoot
962{
963public:
964 virtual const char* getName() const noexcept = 0;
965 virtual Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept = 0;
966 virtual int32_t getNbInputs() const noexcept = 0;
967 virtual int32_t getNbOutputs() const noexcept = 0;
968};
969
970class VAlgorithm : public VRoot
971{
972public:
973 virtual const IAlgorithmIOInfo& getAlgorithmIOInfo(int32_t index) const noexcept = 0;
974 virtual const IAlgorithmVariant& getAlgorithmVariant() const noexcept = 0;
975 virtual float getTimingMSec() const noexcept = 0;
976 virtual std::size_t getWorkspaceSize() const noexcept = 0;
977 virtual const IAlgorithmIOInfo* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
978};
979
980class VTimingCache : public VRoot
981{
982public:
983 virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
984 virtual bool combine(const ITimingCache& inputCache, bool ignoreMismatch) noexcept = 0;
985 virtual bool reset() noexcept = 0;
986};
987
988class VBuilderConfig : public VRoot
989{
990public:
991 virtual void setMinTimingIterations(int32_t minTiming) noexcept = 0;
992 virtual int32_t getMinTimingIterations() const noexcept = 0;
993 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
994 virtual int32_t getAvgTimingIterations() const noexcept = 0;
995 virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
996 virtual EngineCapability getEngineCapability() const noexcept = 0;
997 virtual void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
998 virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
999 virtual void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept = 0;
1000 virtual std::size_t getMaxWorkspaceSize() const noexcept = 0;
1001 virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
1002 virtual BuilderFlags getFlags() const noexcept = 0;
1003 virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
1004 virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
1005 virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
1006 virtual void setDeviceType(const ILayer* layer, DeviceType deviceType) noexcept = 0;
1007 virtual DeviceType getDeviceType(const ILayer* layer) const noexcept = 0;
1008 virtual bool isDeviceTypeSet(const ILayer* layer) const noexcept = 0;
1009 virtual void resetDeviceType(const ILayer* layer) noexcept = 0;
1010 virtual bool canRunOnDLA(const ILayer* layer) const noexcept = 0;
1011 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
1012 virtual int32_t getDLACore() const noexcept = 0;
1013 virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
1014 virtual DeviceType getDefaultDeviceType() const noexcept = 0;
1015 virtual void reset() noexcept = 0;
1016 virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
1017 virtual cudaStream_t getProfileStream() const noexcept = 0;
1018 virtual int32_t addOptimizationProfile(const IOptimizationProfile* profile) noexcept = 0;
1019 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1020 virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
1021 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
1022 virtual void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept = 0;
1023 virtual IAlgorithmSelector* getAlgorithmSelector() const noexcept = 0;
1024 virtual bool setCalibrationProfile(const IOptimizationProfile* profile) noexcept = 0;
1025 virtual const IOptimizationProfile* getCalibrationProfile() noexcept = 0;
1026 virtual void setQuantizationFlags(QuantizationFlags flags) noexcept = 0;
1027 virtual QuantizationFlags getQuantizationFlags() const noexcept = 0;
1028 virtual void clearQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1029 virtual void setQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1030 virtual bool getQuantizationFlag(QuantizationFlag flag) const noexcept = 0;
1031 virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
1032 virtual TacticSources getTacticSources() const noexcept = 0;
1033 virtual nvinfer1::ITimingCache* createTimingCache(const void* blob, std::size_t size) const noexcept = 0;
1034 virtual bool setTimingCache(const ITimingCache& cache, bool ignoreMismatch) noexcept = 0;
1035 virtual const nvinfer1::ITimingCache* getTimingCache() const noexcept = 0;
1036};
1037
1038class VBuilder : public VRoot
1039{
1040public:
1041 virtual void setMaxBatchSize(int32_t batchSize) noexcept = 0;
1042 virtual int32_t getMaxBatchSize() const noexcept = 0;
1043 virtual bool platformHasFastFp16() const noexcept = 0;
1044 virtual bool platformHasFastInt8() const noexcept = 0;
1045 virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1046 virtual int32_t getNbDLACores() const noexcept = 0;
1047 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
1048 virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1049 virtual nvinfer1::ICudaEngine* buildEngineWithConfig(INetworkDefinition& network, IBuilderConfig& config) noexcept
1050 = 0;
1051 virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
1052 virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1053 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1054 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1055 virtual void reset() noexcept = 0;
1056 virtual bool platformHasTf32() const noexcept = 0;
1057 virtual nvinfer1::IHostMemory* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config) noexcept
1058 = 0;
1059 virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1060 virtual ILogger* getLogger() const noexcept = 0;
1061};
1062
1063} // namespace apiv
1064} // namespace nvinfer1
1065
1066// @endcond
1067
1068#endif // NV_INFER_RUNTIME_IMPL_H
getLogger
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
Dims
Structure to define the dimensions of a tensor.
nvinfer1::ICudaEngine
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1310
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:175
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::TacticSources
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:1275
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4634
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:106
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:223
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2118
nvinfer1::QuantizationFlags
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:7707
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3559
nvinfer1::RNNInputMode
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3205
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:155
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5458
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4660
nvinfer1::AsciiChar
char_t AsciiChar
AsciiChar is the type used by TensorRT to represent valid ASCII characters.
Definition: NvInferRuntimeCommon.h:125
nvinfer1::RNNGateType
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3225
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:977
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4971
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:8479
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:151
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:7737
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:633
nvinfer1::CalibrationAlgoType
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7261
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:90
nvinfer1::SliceMode
SliceMode
Controls how ISliceLayer handles out of bounds coordinates.
Definition: NvInfer.h:4047
nvinfer1::GatherMode
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2900
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:611
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:147
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:1287
nvinfer1::ResizeMode
ResizeMode
Enumerates various modes of resize in the resize layer. Resize mode set using setResizeMode().
Definition: NvInfer.h:4561
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:8491
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2814
nvinfer1::QuantizationFlag
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:7717
nvinfer1::RNNDirection
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3177
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7747
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:221
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4260
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3646
nvinfer1::RNNOperation
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:3155
nvinfer1::LayerInformationFormat
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:2419
nvinfer1::ScatterMode
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5896
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4356
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4584
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4951
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1652
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:247
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1058