29 #include <opencv2/core/version.hpp>
30 #include <opencv2/imgcodecs.hpp>
31 #include <opencv2/imgproc/imgproc.hpp>
32 #include <opencv2/videoio.hpp>
48 #define CHECK_STATUS(STMT) \
51 VPIStatus status = (STMT); \
52 if (status != VPI_SUCCESS) \
54 char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH]; \
55 vpiGetLastStatusMessage(buffer, sizeof(buffer)); \
56 std::ostringstream ss; \
57 ss << vpiStatusGetName(status) << ": " << buffer; \
58 throw std::runtime_error(ss.str()); \
94 throw std::runtime_error(
"Image type not supported");
100 if (cvimg.type() == CV_16U)
102 cvimg.convertTo(out, CV_8U);
107 cvtColor(cvimg, out, cv::COLOR_GRAY2BGR);
123 static std::vector<cv::Vec3b> colors;
129 for (
size_t i = 0; i < colors.size(); ++i)
131 colors[i] = cv::Vec3b(rand.uniform(0, 180), 255, 255);
133 cvtColor(colors, colors, cv::COLOR_HSV2BGR);
139 if (pboxes[i].trackingStatus == 1)
145 x = pboxes[i].bbox.xform.mat3[0][2] + ppreds[i].mat3[0][2];
146 y = pboxes[i].bbox.xform.mat3[1][2] + ppreds[i].mat3[1][2];
147 w = pboxes[i].bbox.width * pboxes[i].bbox.xform.mat3[0][0] * ppreds[i].mat3[0][0];
148 h = pboxes[i].bbox.height * pboxes[i].bbox.xform.mat3[1][1] * ppreds[i].mat3[1][1];
150 rectangle(out, cv::Rect(x, y, w, h), cv::Scalar(colors[i][0], colors[i][1], colors[i][2]), 2);
159 int main(
int argc,
char *argv[])
163 cv::Mat cvTemplate, cvReference;
166 VPIArray inputBoxList = NULL, inputPredList = NULL;
181 throw std::runtime_error(std::string(
"Usage: ") + argv[0] +
" <cpu|pva|cuda> <input_video> <bbox descr>");
184 std::string strBackend = argv[1];
185 std::string strInputVideo = argv[2];
186 std::string strInputBBoxes = argv[3];
189 cv::VideoCapture invid;
190 if (!invid.open(strInputVideo))
192 throw std::runtime_error(
"Can't open '" + strInputVideo +
"'");
196 int w = invid.get(cv::CAP_PROP_FRAME_WIDTH);
197 int h = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
198 int fourcc = cv::VideoWriter::fourcc(
'M',
'P',
'E',
'G');
199 double fps = invid.get(cv::CAP_PROP_FPS);
201 cv::VideoWriter outVideo(
"klt_" + strBackend +
".mp4", fourcc, fps, cv::Size(w, h));
202 if (!outVideo.isOpened())
204 throw std::runtime_error(
"Can't create output video");
212 std::vector<VPIKLTTrackedBoundingBox> bboxes;
213 int32_t bboxesSize = 0;
214 std::vector<VPIHomographyTransform2D> preds;
215 int32_t predsSize = 0;
219 std::map<int, size_t> bboxes_size_at_frame;
227 std::ifstream in(strInputBBoxes);
230 throw std::runtime_error(
"Can't open '" + strInputBBoxes +
"'");
234 int frame, x, y, w, h;
235 while (in >> frame >> x >> y >> w >> h)
237 if (bboxes.size() == 64)
239 throw std::runtime_error(
"Too many bounding boxes");
260 bboxes.push_back(track);
264 xform.
mat3[0][0] = 1;
265 xform.
mat3[1][1] = 1;
266 xform.
mat3[2][2] = 1;
267 preds.push_back(xform);
269 bboxes_size_at_frame[frame] = bboxes.size();
272 if (!in && !in.eof())
274 throw std::runtime_error(
"Can't parse bounding boxes, stopped at bbox #" +
275 std::to_string(bboxes.size()));
296 if (strBackend ==
"cpu")
300 else if (strBackend ==
"cuda")
304 else if (strBackend ==
"pva")
310 throw std::runtime_error(
"Backend '" + strBackend +
311 "' not recognized, it must be either cpu, cuda or pva.");
319 auto fetchFrame = [&invid, &nextFrame, backend]() {
321 if (!invid.read(frame))
327 if (frame.channels() == 3)
329 cvtColor(frame, frame, cv::COLOR_BGR2GRAY);
338 frame.convertTo(aux, CV_16U);
343 assert(frame.type() == CV_8U);
352 cvTemplate = fetchFrame();
376 size_t curNumBoxes = 0;
380 size_t curFrame = nextFrame - 1;
383 auto tmp = --bboxes_size_at_frame.upper_bound(curFrame);
384 size_t bbox_count = tmp->second;
386 assert(bbox_count >= curNumBoxes &&
"input bounding boxes must be sorted by frame");
389 if (curNumBoxes != bbox_count)
396 for (
size_t i = 0; i < bbox_count - curNumBoxes; ++i)
398 std::cout << curFrame <<
" -> new " << curNumBoxes + i << std::endl;
400 assert(bbox_count <= bboxes.capacity());
401 assert(bbox_count <= preds.capacity());
403 curNumBoxes = bbox_count;
407 outVideo << WriteKLTBoxes(imgTemplate, inputBoxList, inputPredList);
410 cvReference = fetchFrame();
413 if (cvReference.data == NULL)
425 imgReference, outputBoxList, outputEstimList, ¶ms));
448 for (
size_t b = 0; b < curNumBoxes; ++b)
451 if (updated_bbox[b].trackingStatus)
454 if (bboxes[b].trackingStatus == 0)
456 std::cout << curFrame <<
" -> dropped " << b << std::endl;
457 bboxes[b].trackingStatus = 1;
464 if (updated_bbox[b].templateStatus)
466 std::cout << curFrame <<
" -> update " << b << std::endl;
476 bboxes[b] = updated_bbox[b];
479 bboxes[b].templateStatus = 1;
483 preds[b].
mat3[0][0] = 1;
484 preds[b].mat3[1][1] = 1;
485 preds[b].mat3[2][2] = 1;
490 bboxes[b].templateStatus = 0;
505 std::swap(imgTemplate, imgReference);
506 std::swap(cvTemplate, cvReference);
509 catch (std::exception &e)
511 std::cerr << e.what() << std::endl;
Functions and structures for dealing with VPI arrays.
Functions and structures for dealing with VPI images.
Declares functions that implement the KLT Feature Tracker algorithm.
Functions for handling OpenCV interoperability with VPI.
Declaration of VPI status codes handling functions.
Declares functions dealing with VPI streams.
VPIArrayBufferType bufferType
Type of array buffer.
void * data
Points to the first element of the array.
VPIArrayBuffer buffer
Stores the array contents.
int32_t * sizePointer
Points to the number of elements in the array.
VPIArrayBufferAOS aos
Array stored in array-of-structures layout.
int32_t capacity
Maximum number of elements that the array can hold.
VPIArrayType type
Type of each array element.
VPIStatus vpiArraySetSize(VPIArray array, int32_t size)
Set the array size in elements.
VPIStatus vpiArrayUnlock(VPIArray array)
Releases the lock on array object.
VPIStatus vpiArrayLockData(VPIArray array, VPILockMode mode, VPIArrayBufferType bufType, VPIArrayData *data)
Acquires the lock on an array object and returns the array contents.
VPIStatus vpiArrayCreateWrapper(const VPIArrayData *data, uint64_t flags, VPIArray *array)
Create an array object by wrapping an existing host memory block.
void vpiArrayDestroy(VPIArray array)
Destroy an array instance.
VPIStatus vpiArrayCreate(int32_t capacity, VPIArrayType type, uint64_t flags, VPIArray *array)
Create an empty array instance.
VPIStatus vpiArrayLock(VPIArray array, VPILockMode mode)
Acquires the lock on an array object.
struct VPIArrayImpl * VPIArray
A handle to an array.
@ VPI_ARRAY_TYPE_KLT_TRACKED_BOUNDING_BOX
VPIKLTTrackedBoundingBox element.
@ VPI_ARRAY_TYPE_HOMOGRAPHY_TRANSFORM_2D
VPIHomographyTransform2D element.
@ VPI_ARRAY_BUFFER_HOST_AOS
Host-accessible array-of-structures.
Stores information about array characteristics and contents.
VPIImageBuffer buffer
Stores the image contents.
VPIImagePlanePitchLinear planes[VPI_MAX_PLANE_COUNT]
Data of all image planes in pitch-linear layout.
VPIImageBufferPitchLinear pitch
Image stored in pitch-linear layout.
void * data
Pointer to the first row of this plane.
VPIImageFormat format
Image format.
VPIImageBufferType bufferType
Type of image buffer.
int32_t height
Height of this plane in pixels.
int32_t width
Width of this plane in pixels.
int32_t pitchBytes
Difference in bytes of beginning of one row and the beginning of the previous.
void vpiImageDestroy(VPIImage img)
Destroy an image instance.
struct VPIImageImpl * VPIImage
A handle to an image.
VPIStatus vpiImageLockData(VPIImage img, VPILockMode mode, VPIImageBufferType bufType, VPIImageData *data)
Acquires the lock on an image object and returns the image contents.
VPIStatus vpiImageGetFormat(VPIImage img, VPIImageFormat *format)
Get the image format.
VPIStatus vpiImageUnlock(VPIImage img)
Releases the lock on an image object.
@ VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR
Host-accessible with planes in pitch-linear memory layout.
Stores the image plane contents.
Stores information about image characteristics and content.
int8_t templateStatus
Status of the template related to this bounding box.
int8_t trackingStatus
Tracking status of this bounding box.
VPIBoundingBox bbox
Bounding box being tracked.
VPIStatus vpiCreateKLTFeatureTracker(uint64_t backends, int32_t imageWidth, int32_t imageHeight, VPIImageFormat imageFormat, const VPIKLTFeatureTrackerCreationParams *params, VPIPayload *payload)
Creates payload for vpiSubmitKLTFeatureTracker.
VPIStatus vpiSubmitKLTFeatureTracker(VPIStream stream, uint64_t backend, VPIPayload payload, VPIImage templateImage, VPIArray inputBoxList, VPIArray inputPredictionList, VPIImage referenceImage, VPIArray outputBoxList, VPIArray outputEstimationList, const VPIKLTFeatureTrackerParams *params)
Runs KLT Feature Tracker on two frames.
VPIStatus vpiInitKLTFeatureTrackerParams(VPIKLTFeatureTrackerParams *params)
Initialize VPIKLTFeatureTrackerParams with default values.
Structure that defines the parameters for vpiCreateKLTFeatureTracker.
Stores a bounding box that is being tracked by KLT Tracker.
VPIStatus vpiImageCreateWrapperOpenCVMat(const cv::Mat &mat, VPIImageFormat fmt, uint64_t flags, VPIImage *img)
Wraps a cv::Mat in an VPIImage with the given image format.
VPIStatus vpiImageSetWrappedOpenCVMat(VPIImage img, const cv::Mat &mat)
Redefines the wrapped cv::Mat of an existing VPIImage wrapper.
struct VPIPayloadImpl * VPIPayload
A handle to an algorithm payload.
void vpiPayloadDestroy(VPIPayload payload)
Deallocates the payload object and all associated resources.
struct VPIStreamImpl * VPIStream
A handle to a stream.
VPIStatus vpiStreamSync(VPIStream stream)
Blocks the calling thread until all submitted commands in this stream queue are done (queue is empty)...
VPIBackend
VPI Backend types.
void vpiStreamDestroy(VPIStream stream)
Destroy a stream instance and deallocate all HW resources.
VPIStatus vpiStreamCreate(uint64_t flags, VPIStream *stream)
Create a stream instance.
@ VPI_BACKEND_CUDA
CUDA backend.
@ VPI_BACKEND_PVA
PVA backend.
@ VPI_BACKEND_CPU
CPU backend.
float width
Bounding box width.
float height
Bounding box height.
VPIHomographyTransform2D xform
Defines the bounding box top left corner and its homography.
float mat3[3][3]
3x3 homogeneous matrix that defines the homography.
@ VPI_LOCK_READ_WRITE
Lock memory for reading and writing.
@ VPI_LOCK_READ
Lock memory only for reading.