nvmedia_vpi.h

Go to the documentation of this file.

/*
 * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. All
 * information contained herein is proprietary and confidential to NVIDIA
 * Corporation.  Any use, reproduction, or disclosure without the written
 * permission of NVIDIA Corporation is prohibited.
 */


#ifndef _NvMediaVPI_H
#define _NvMediaVPI_H

#ifdef __cplusplus
extern "C" {
#endif

#include "nvmedia_core.h"
#include "nvmedia_image.h"
#include "nvmedia_array.h"
#include "nvmedia_image_pyramid.h"
#include "nvmedia_cvscratchpad.h"
#include "nvmedia_surface.h"

#define NVMEDIA_VPI_VERSION_MAJOR   2

#define NVMEDIA_VPI_VERSION_MINOR   6

#define NVMEDIA_VPI_MAX_QUEUED_TASKS 64

typedef struct NvMediaVPI NvMediaVPI;

typedef enum {
    NVMEDIA_VPI_ARRAY_TYPE_KEYPOINT = 0,
    NVMEDIA_VPI_ARRAY_TYPE_BBOX_XFORM = 1,
    NVMEDIA_VPI_ARRAY_TYPE_TRANSFORM = 2,
        /* If more types are needed, add above this and update this. */
    NVMEDIA_VPI_ARRAY_NUM_TYPES = 3,
} NvMediaVPIArrayType;

typedef struct {
    float x;
    float y;
} NvMediaVPIPoint2Df;

typedef struct {
    uint32_t x;
    uint32_t y;
} NvMediaVPIPoint2DFrac;

typedef struct {
    float x;
    float y;
    float width;
    float height;
    uint8_t trackingStatus;
    uint8_t updateTemplate;
    uint8_t reserved1;
    uint8_t reserved2;
} NvMediaVPIAABB;

typedef struct {
    float mat3[3][3];
} NvMediaVPI2DTransform;

typedef struct {
    float deltaScale;
    float deltaX;
    float deltaY;
} NvMediaVPITranslationWithScale;

typedef struct {
    NvMediaVPI2DTransform transform;
    float width;
    float height;
    uint8_t trackingStatus;
    uint8_t templateStatus;
    uint8_t reserved1;
    uint8_t reserved2;
} NvMediaVPIBoundingBoxWithTransform;

NvMediaStatus
NvMediaVPIGetNumEngines(
    uint32_t *numEngines
);

NvMediaStatus
NvMediaVPIGetVersion(
    NvMediaVersion *version
);

NvMediaVPI *
NvMediaVPICreate(
    const uint32_t Id,
    const uint32_t maxQueuedTasks
);

NvMediaStatus
NvMediaVPIDestroy(
     NvMediaVPI *vpi
);

NvMediaStatus
NvMediaVPIFlush(
     NvMediaVPI *vpi
);

NvMediaStatus
NvMediaVPIImageRegister(
    NvMediaVPI *vpi,
    NvMediaImage *image
);

NvMediaStatus
NvMediaVPIImageUnregister(
    NvMediaVPI *vpi,
    NvMediaImage *image
);

NvMediaStatus
NvMediaVPIArrayRegister(
    NvMediaVPI *vpi,
    NvMediaArray *array
);

NvMediaStatus
NvMediaVPIArrayUnregister(
    NvMediaVPI *vpi,
    NvMediaArray *array
);

NvMediaStatus
NvMediaVPIScratchpadRegister(
    NvMediaVPI *vpi,
    NvMediaCVScratchpad *scratchpad
);

NvMediaStatus
NvMediaVPIScratchpadUnregister(
    NvMediaVPI *vpi,
    NvMediaCVScratchpad *scratchpad
);

NvMediaArray *
NvMediaVPIArrayCreate(
    NvMediaDevice *device,
    NvMediaVPIArrayType type,
    uint32_t numElements,
    const NvMediaArrayAllocAttr *attrs,
    uint32_t numAttrs
);

NvMediaStatus
NvMediaVPIPyramidRegister(
    NvMediaVPI *vpi,
    NvMediaImagePyramid *pyramid
);

NvMediaStatus
NvMediaVPIPyramidUnregister(
    NvMediaVPI *vpi,
    NvMediaImagePyramid *pyramid
);

typedef struct NvMediaVPIProcessStereoDescriptor NvMediaVPIProcessStereoDescriptor;

typedef enum {
    NVMEDIA_VPI_STEREO_DISPARITY = 0,
    NVMEDIA_VPI_MV,
    NVMEDIA_VPI_MV_HINTS,
} NvMediaVPIOFSTOutputType;

NvMediaVPIProcessStereoDescriptor *
NvMediaVPICreateProcessStereoPairDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const NvMediaVPIOFSTOutputType outputType
);

NvMediaStatus
NvMediaVPIProcessStereoPairDesc(
    NvMediaVPI *vpi,
    NvMediaVPIProcessStereoDescriptor *descriptor,
    NvMediaImage *left,
    NvMediaImage *right,
    NvMediaImage *output
);

typedef struct NvMediaVPIConvertMVDescriptor NvMediaVPIConvertMVDescriptor;

NvMediaVPIConvertMVDescriptor *
NvMediaVPICreateConvertMVDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const NvMediaSurfaceType type,
    const float strength,
    const float scale
);

NvMediaStatus
NvMediaVPIConvertMVDesc(
    NvMediaVPI *vpi,
    NvMediaVPIConvertMVDescriptor *descriptor,
    NvMediaImage *inputMVImage,
    NvMediaImage *inputColor,
    NvMediaImage *output,
    const NvMediaVPIOFSTOutputType outputType
);


typedef struct NvMediaVPIStereoPostprocessDescriptor NvMediaVPIStereoPostprocessDescriptor;

NvMediaVPIStereoPostprocessDescriptor *
NvMediaVPICreateStereoPostprocessDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const float scale
);

typedef struct {
    uint16_t confidenceThresh;
    uint32_t windowDimension;
} NvMediaVPIStereoPostprocessParams;

NvMediaStatus
NvMediaVPIStereoPostprocessDesc(
    NvMediaVPI *vpi,
    NvMediaVPIStereoPostprocessDescriptor *descriptor,
    NvMediaVPIStereoPostprocessParams *params,
    NvMediaImage *disparityInput,
    NvMediaImage *confidenceInput,
    NvMediaImage *disparityOutput,
    NvMediaImage *confidenceOutput
);

typedef struct NvMediaVPIStereoPreprocessDescriptor NvMediaVPIStereoPreprocessDescriptor;

NvMediaVPIStereoPreprocessDescriptor *
NvMediaVPICreateStereoPreprocessDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const NvMediaVPIOFSTOutputType outputType
);

typedef struct {
    uint8_t numberOfHintsPerMB;
    uint8_t fractionalBits;
    uint8_t confidenceThresh;
    uint8_t disparityDiffToConfidenceShift;
} NvMediaVPIStereoPreprocessParams;

#define NVMEDIA_VPI_CONFIDENCE_TABLE_MIN_CAPACITY (128)

NvMediaStatus
NvMediaVPIStereoPreprocessDesc(
    NvMediaVPI *vpi,
    NvMediaVPIStereoPreprocessDescriptor *descriptor,
    NvMediaVPIStereoPreprocessParams *params,
    NvMediaArray *confidenceTable,
    NvMediaImage *leftImage,
    NvMediaImage *rightImage,
    NvMediaImage *output0,
    NvMediaImage *output1
);

#define NVMEDIA_VPI_HINT_ARRAY_MIN_CAPACITY_BYTES (261376)

NvMediaStatus
NvMediaVPIStereoPreprocessDescEx(
    NvMediaVPI *vpi,
    NvMediaVPIStereoPreprocessDescriptor *descriptor,
    NvMediaVPIStereoPreprocessParams *params,
    NvMediaArray *confidenceTable,
    NvMediaImage *leftImage,
    NvMediaImage *rightImage,
    NvMediaImage *leftDisparity,
    NvMediaImage *rightDisparity,
    NvMediaImage *confidenceMap,
    NvMediaArray *hintsArray
);

typedef enum {
    /* Use a 2D grid of 8x8 cells with no minimal distance. */
    NVMEDIA_VPI_2D_GRID_8x8_CELL_NO_MIN_DISTANCE = 0
} NvMediaVPINonMaxSuppressionType;

typedef enum {
    NVMEDIA_VPI_HARRIS_FILTER_TYPE_DEFAULT = 0,
    NVMEDIA_VPI_HARRIS_FILTER_TYPE_USER
} NvMediaVPIHarrisGradientFilterType;

/*
 * \brief NvMediaVPIGetKeyPointsHarrisDescriptor.
 */
typedef struct NvMediaVPIGetKeyPointsHarrisDescriptor NvMediaVPIGetKeyPointsHarrisDescriptor;

NvMediaVPIGetKeyPointsHarrisDescriptor *
NvMediaVPICreateGetKeyPointsHarrisDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const NvMediaVPINonMaxSuppressionType nonMaxSuppressionType);

typedef struct {
    const uint32_t gradientSize;
    const NvMediaVPIHarrisGradientFilterType gradientFilterType;
    const int16_t *horizontalFilterCoefficients;
    const uint16_t *verticalFilterCoefficients;
    const uint32_t blockSize;
    const float strengthThresh;
    const float sensitivity;
    const NvMediaVPINonMaxSuppressionType nonMaxSuppressionType;
    const float minDistance;
} NvMediaVPIGetKeyPointsHarrisParams;

NvMediaStatus
NvMediaVPIGetKeyPointsHarrisDesc(
    NvMediaVPI *vpi,
    NvMediaVPIGetKeyPointsHarrisDescriptor *descriptor,
    const NvMediaVPIGetKeyPointsHarrisParams *params,
    NvMediaImage *input,
    NvMediaImage *mask,
    NvMediaArray *trackedKeypoints,
    NvMediaArray *trackedScores,
    NvMediaArray *keypoints,
    NvMediaArray *scores
);

/*
 * \brief NvMediaVPIGetKeyPointsFastDescriptor.
 */
typedef struct NvMediaVPIGetKeyPointsFastDescriptor NvMediaVPIGetKeyPointsFastDescriptor;

NvMediaVPIGetKeyPointsFastDescriptor *
NvMediaVPICreateGetKeyPointsFastDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const NvMediaSurfaceType type
);

NvMediaStatus
NvMediaVPIGetKeyPointsFastDesc(
    NvMediaVPI *vpi,
    NvMediaVPIGetKeyPointsFastDescriptor *descriptor,
    NvMediaImage *input,
    const uint32_t strengthThresh,
    const NvMediaBool nonmaxSupression,
    NvMediaArray *keypoints,
    NvMediaArray *scores
);

typedef enum {
    NVMEDIA_VPI_TERMINATION_CRITERION_ITERATIONS = 0,
    NVMEDIA_VPI_TERMINATION_CRITERION_EPSILON,
    NVMEDIA_VPI_TERMINATION_CRITERION_BOTH
} NvMediaVPITerminationCriterion;

typedef struct {
    const NvMediaVPITerminationCriterion termination;
    float epsilon;
    uint32_t numPoints;
    uint32_t numIterations;
    uint32_t windowDimension;
    int32_t calcError;
    uint32_t numLevels;
} NvMediaVPIGetSparseFlowPyrLKParams;

typedef struct NvMediaVPIGetSparseFlowPyrLKDescriptor NvMediaVPIGetSparseFlowPyrLKDescriptor;

NvMediaVPIGetSparseFlowPyrLKDescriptor *
NvMediaVPICreateGetSparseFlowPyrLKDescriptor(
    NvMediaVPI *vpi,
    const NvMediaVPIGetSparseFlowPyrLKParams *params
);

NvMediaStatus NvMediaVPIGetSparseFlowPyrLKDesc(
    NvMediaVPI *vpi,
    NvMediaVPIGetSparseFlowPyrLKDescriptor *descriptor,
    const NvMediaVPIGetSparseFlowPyrLKParams *params,
    NvMediaImagePyramid *oldImages,
    NvMediaImagePyramid *newImages,
    NvMediaArray *oldPoints,
    NvMediaArray *newPoints,
    NvMediaArray *newStatus
);

typedef struct NvMediaVPIKLTDescriptor NvMediaVPIKLTDescriptor;

NvMediaVPIKLTDescriptor *
NvMediaVPICreateKLTDescriptor(
    NvMediaVPI *vpi,
    const uint32_t width,
    const uint32_t height,
    const NvMediaSurfaceType type
);

typedef enum NvMediaVPIKltTrackType
{
    NVMEDIA_VPI_KLT_INVERSE_COMPOSITIONAL = 0,
} NvMediaVPIKltTrackType;

typedef struct {
    uint32_t numIterationsScaling;
    uint32_t numIterationsTranslation;
    uint32_t numIterationsCoarse;
    uint32_t numIterationsFine;
    float maxPixelTolerance;
    float thresholdUpdate;
    float thresholdKill;
    float thresholdStop;
    float maxScaleChange;
    NvMediaVPIKltTrackType trackingType;
} NvMediaVPIKLTParams;

NvMediaStatus
NvMediaVPIKLTDesc(
    NvMediaVPI *vpi,
    NvMediaVPIKLTDescriptor *descriptor,
    const NvMediaVPIKLTParams *params,
    NvMediaImage *referenceImage,
    NvMediaImage *templateImage,
    NvMediaArray *inputBoxList,
    NvMediaArray *predictedBoxList,
    NvMediaArray *outputBoxList,
    NvMediaArray *estimationList
);

NvMediaStatus
NvMediaVPIGetScratchpadSize(
    void *descriptor,
    uint32_t *scratchpadSizeBytes
);

NvMediaStatus
NvMediaVPIKLTFastDesc(
    NvMediaVPI *vpi,
    NvMediaVPIKLTDescriptor *descriptor,
    NvMediaCVScratchpad *scratchpad,
    const NvMediaVPIKLTParams *params,
    NvMediaImage *referenceImage,
    NvMediaImage *templateImage,
    NvMediaArray *inputBoxList,
    NvMediaArray *predictedBoxList,
    NvMediaArray *outputBoxList,
    NvMediaArray *estimationList
);

NvMediaStatus
NvMediaVPIGetImagePyramid(
    NvMediaVPI *vpi,
    NvMediaImage *input,
    NvMediaImagePyramid *output
);

NvMediaStatus
NvMediaVPIConvolveImage(
    NvMediaVPI *vpi,
    NvMediaImage *input,
    const float *kernelData,
    const uint32_t kernelWidth,
    const uint32_t kernelHeight,
    NvMediaImage *output
);

typedef struct NvMediaVPIConvolveImageSeparableDescriptor \
               NvMediaVPIConvolveImageSeparableDescriptor;

NvMediaVPIConvolveImageSeparableDescriptor *
NvMediaVPICreateConvolveImageSeparableDescriptor(
    NvMediaVPI *vpi,
    NvMediaSurfaceType type,
    const float *kernelX,
    const uint32_t kernelXSize,
    const float *kernelY,
    const uint32_t kernelYSize
);

NvMediaStatus
NvMediaVPIConvolveImageSeparable(
    NvMediaVPI *vpi,
    NvMediaImage *input,
    const float *kernelX,
    const uint32_t kernelXSize,
    const float *kernelY,
    const uint32_t kernelYSize,
    NvMediaImage *output
);

NvMediaStatus
NvMediaVPIFilterImageBox(
    NvMediaVPI *vpi,
    NvMediaImage *input,
    const uint32_t windowWidth,
    const uint32_t windowHeight,
    NvMediaImage *output
);

NvMediaStatus
NvMediaVPIArrayGetPoint2Df(
    NvMediaArray *array,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaVPIPoint2Df *destPtr
);

NvMediaStatus
NvMediaVPIArrayUpdatePoint2Df(
    NvMediaVPIPoint2Df *inputPtr,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaArray *destArray
);

NvMediaStatus
NvMediaVPIArrayGetAABB(
    NvMediaArray *array,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaVPIAABB *destPtr
);

NvMediaStatus
NvMediaVPIArrayUpdateAABB(
    NvMediaVPIAABB *inputPtr,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaArray *destArray
);

NvMediaStatus
NvMediaVPIArrayGetBoundingBoxWithTransform(
    NvMediaArray *array,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaVPIBoundingBoxWithTransform *destPtr
);

NvMediaStatus
NvMediaVPIArrayUpdateBoundingBoxWithTransform(
    NvMediaVPIBoundingBoxWithTransform *inputPtr,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaArray *destArray
);

NvMediaStatus
NvMediaVPIArrayGetTranslationWithScale(
    NvMediaArray *array,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaVPITranslationWithScale *destPtr
);

NvMediaStatus
NvMediaVPIArrayUpdateTranslationWithScale(
    NvMediaVPITranslationWithScale *inputPtr,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaArray *destArray
);

NvMediaStatus
NvMediaVPIArrayGet2DTransform(
    NvMediaArray *array,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaVPI2DTransform *destPtr
);

NvMediaStatus
NvMediaVPIArrayUpdate2DTransform(
    NvMediaVPI2DTransform *inputPtr,
    const uint32_t startIndex,
    const uint32_t numElements,
    NvMediaArray *destArray
);

NvMediaStatus
NvMediaVPIDestroyDescriptor(
    NvMediaVPI *vpi,
    void *descriptor
);

/*
 * \defgroup history_NvMediaVPI History
 * Provides change history for the NVIDIA VPI API.
 *
 * \section history_NvMediaVPI Version History
 *
 * <b> Version 1.0 </b> March 2, 2018
 * - Initial API definitions.
 *
 * <b> Version 2.0 </b> June 5, 2018
 * - Update Harris tracking API.
 *
 * <b> Version 2.1 </b> July 12, 2018
 * - Add KLT API, and array accessor API's.
 *
 * <b> Version 2.2 </b> July 12, 2018
 * - Add API to query number of available engines.
 *
 * <b> Version 2.3 </b> Sept 27, 2018
 * - Add KLT Fast API
 *
 * <b> Version 2.4 </b> Feb 21, 2019
 * - Add NvMediaVPIStereoPreprocessDescEx API
 *
 * <b> Version 2.5 </b> March 16, 2019
 * - Added required header include nvmedia_surface.h
 *
 * <b> Version 2.6 </b> March 22, 2019
 * - Unnecessary header include nvmedia_common.h has been removed
 *
 */

#ifdef __cplusplus
}
#endif

#endif /* _NVMEDIA_VPI_H */