Perspective Warp

Overview

The Perspective Warp sample application takes an input video and outputs a video where for each frame, a different perspective warp is applied. The result is a perspective bouncy effect. Sample application could be modified to get the input from a camera and apply the effect in real-time.

Instructions

The command line parameters are:

<backend> <input video>

where

• backend: Either cpu, cuda or vic (on Jetson devices only). It defines the backend that will perform the processing.
• input video: video file to the effect applied on; it accepts .mp4, .avi and possibly others, depending on OpenCV's support.

VPI samples installer includes some sample videos that can be used as input. They are found in /opt/nvidia/vpi1/samples/assets/ directory.

Here's one invocation example:

• C++

  ./vpi_sample_10_perspwarp cuda ../assets/noisy.mp4

• Python

  python main.py cuda ../assets/noisy.mp4

The application will process noisy.mp4 and create perspwarp_cuda.mp4 with time-variant perspective warps applied to input frames.

Note

If using OpenCV-2.4 or older (i.e. on Ubuntu 16.04), output file is perspwarp_cuda.avi.

Results

Input video	Perspective effect

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++ Python

import cv2
import sys
import vpi
import numpy as np
from math import sin, cos, pi
from argparse import ArgumentParser
 
# ----------------------------
# Parse command line arguments
 
parser = ArgumentParser()
parser.add_argument('backend', choices=['cpu', 'cuda','vic'],
                    help='Backend to be used for processing')
 
parser.add_argument('input',
                    help='Input video to be denoised')
 
args = parser.parse_args();
 
if args.backend == 'cuda':
    backend = vpi.Backend.CUDA
elif args.backend == 'cpu':
    backend = vpi.Backend.CPU
else:
    assert args.backend == 'vic'
    backend = vpi.Backend.VIC
 
# -----------------------------
# Open input and output videos
 
inVideo = cv2.VideoCapture(args.input)
 
if int(cv2.__version__.split('.')[0]) >= 3:
    extOutputVideo = '.mp4'
    fourcc = cv2.VideoWriter_fourcc(*'avc1')
    inSize = (int(inVideo.get(cv2.CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.CAP_PROP_FRAME_HEIGHT)))
    fps = inVideo.get(cv2.CAP_PROP_FPS)
else:
    # MP4 support with OpenCV-2.4 has issues, we'll use
    # avi/mpeg instead.
    extOutputVideo = '.avi'
    fourcc = cv2.cv.CV_FOURCC('M','P','E','G')
    inSize = (int(inVideo.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT)))
    fps = inVideo.get(cv2.cv.CV_CAP_PROP_FPS)
 
outVideo = cv2.VideoWriter('perspwarp_python'+str(sys.version_info[0])+'_'+args.backend+extOutputVideo,
                            fourcc, fps, inSize)
 
#--------------------------------------------------------------
# Main processing loop
curFrame = 1
while True:
    print("Frame: {}".format(curFrame))
    curFrame+=1
 
    # Read one input frame
    ret, cvFrame = inVideo.read()
    if not ret:
        break
 
    # Convert it to NV12_ER format to be used by VPI
    with vpi.Backend.CUDA:
        frame = vpi.asimage(cvFrame).convert(vpi.Format.NV12_ER)
 
    # Calculate the transformation to be applied ------------
 
    # Move image's center to origin of coordinate system
    T1 = np.array([[1, 0, -frame.width/2.0],
                   [0, 1, -frame.height/2.0],
                   [0, 0, 1]])
 
    # Apply some time-dependent perspective transform
    v1 = sin(curFrame/30.0*2*pi/2)*0.0005
    v2 = cos(curFrame/30.0*2*pi/3)*0.0005
    P = np.array([[0.66, 0, 0],
                  [0, 0.66, 0],
                  [v1, v2, 1]])
 
    # Move image's center back to where it was
    T2 = np.array([[1, 0, frame.width/2.0],
                   [0, 1, frame.height/2.0],
                   [0, 0, 1]])
 
    # Do perspective warp using the backend passed in the command line.
    with backend:
        frame = frame.perspwarp(np.matmul(T2, np.matmul(P, T1)))
 
    # Convert it to RGB8 for output using the CUDA backend
    with vpi.Backend.CUDA:
        frame = frame.convert(vpi.Format.RGB8)
 
    # Write the denoised frame to the output video
    with frame.rlock():
        outVideo.write(frame.cpu())
 
# vim: ts=8:sw=4:sts=4:et:ai

#include <opencv2/core/version.hpp>
#if CV_MAJOR_VERSION >= 3
#    include <opencv2/imgcodecs.hpp>
#    include <opencv2/videoio.hpp>
#else
#    include <opencv2/highgui/highgui.hpp>
#endif
 
#include <opencv2/imgproc/imgproc.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/PerspectiveWarp.h>
 
#include <algorithm>
#include <cstring> // for memset
#include <fstream>
#include <iostream>
#include <random>
#include <sstream>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
static void MatrixMultiply(VPIPerspectiveTransform &r, const VPIPerspectiveTransform &a,
                           const VPIPerspectiveTransform &b)
{
    for (int i = 0; i < 3; ++i)
    {
        for (int j = 0; j < 3; ++j)
        {
            r[i][j] = a[i][0] * b[0][j];
            for (int k = 1; k < 3; ++k)
            {
                r[i][j] += a[i][k] * b[k][j];
            }
        }
    }
}
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvFrame;
 
    // VPI objects that will be used
    VPIStream stream  = NULL;
    VPIImage imgInput = NULL, imgOutput = NULL;
    VPIPayload warp   = NULL;
    VPIImage frameBGR = NULL;
 
    int retval = 0;
 
    try
    {
        // =============================
        // Parse command line parameters
 
        if (argc != 3)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|vic|cuda> <input_video>");
        }
 
        std::string strBackend    = argv[1];
        std::string strInputVideo = argv[2];
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cpu")
        {
            backend = VPI_BACKEND_CPU;
        }
        else if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else if (strBackend == "vic")
        {
            backend = VPI_BACKEND_VIC;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend +
                                     "' not recognized, it must be either cpu, cuda or vic.");
        }
 
        // ===============================
        // Prepare input and output videos
 
        // Load the input video
        cv::VideoCapture invid;
        if (!invid.open(strInputVideo))
        {
            throw std::runtime_error("Can't open '" + strInputVideo + "'");
        }
 
        // Open the output video for writing using input's characteristics
#if CV_MAJOR_VERSION >= 3
        int w                      = invid.get(cv::CAP_PROP_FRAME_WIDTH);
        int h                      = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
        int fourcc                 = cv::VideoWriter::fourcc('a', 'v', 'c', '1');
        double fps                 = invid.get(cv::CAP_PROP_FPS);
        std::string extOutputVideo = ".mp4";
#else
        // MP4 support with OpenCV-2.4 has issues, we'll use
        // avi/mpeg instead.
        int w                      = invid.get(CV_CAP_PROP_FRAME_WIDTH);
        int h                      = invid.get(CV_CAP_PROP_FRAME_HEIGHT);
        int fourcc                 = CV_FOURCC('M', 'P', 'E', 'G');
        double fps                 = invid.get(CV_CAP_PROP_FPS);
        std::string extOutputVideo = ".avi";
#endif
 
        cv::VideoWriter outVideo("perspwarp_" + strBackend + extOutputVideo, fourcc, fps, cv::Size(w, h));
        if (!outVideo.isOpened())
        {
            throw std::runtime_error("Can't create output video");
        }
 
        // =================================
        // Allocate all VPI resources needed
 
        // Create the stream for the given backend. We'll be using CUDA for image format conversion.
        CHECK_STATUS(vpiStreamCreate(backend | VPI_BACKEND_CUDA, &stream));
 
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgInput));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgOutput));
 
        // Create a Perspective Warp payload.
        CHECK_STATUS(vpiCreatePerspectiveWarp(backend, &warp));
 
        VPIPerspectiveTransform xform;
        memset(&xform, 0, sizeof(xform));
 
        // ====================
        // Main processing loop
 
        int curFrame = 1;
        while (invid.read(cvFrame))
        {
            printf("Frame: %d\n", curFrame++);
 
            if (frameBGR == NULL)
            {
                // Ceate a VPIImage that wraps the frame
                CHECK_STATUS(vpiImageCreateOpenCVMatWrapper(cvFrame, 0, &frameBGR));
            }
            else
            {
                // reuse existing VPIImage wrapper to wrap the new frame.
                CHECK_STATUS(vpiImageSetWrappedOpenCVMat(frameBGR, cvFrame));
            }
 
            // First convert it to NV12 using CUDA
            CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, frameBGR, imgInput, NULL));
 
            // move image's center to origin of coordinate system
            VPIPerspectiveTransform t1 = {{1, 0, -w / 2.0f}, {0, 1, -h / 2.0f}, {0, 0, 1}};
 
            // Apply some time-dependent perspective transform
            float v1                  = sin(curFrame / 30.0 * 2 * M_PI / 2) * 0.0005f;
            float v2                  = cos(curFrame / 30.0 * 2 * M_PI / 3) * 0.0005f;
            VPIPerspectiveTransform P = {{0.66, 0, 0}, {0, 0.66, 0}, {v1, v2, 1}};
 
            // move image's center back to where it was.
            VPIPerspectiveTransform t2 = {{1, 0, w / 2.0f}, {0, 1, h / 2.0f}, {0, 0, 1}};
 
            // Apply the transforms defined above.
            VPIPerspectiveTransform tmp;
            MatrixMultiply(tmp, P, t1);
            MatrixMultiply(xform, t2, tmp);
 
            // Do perspective warp using the backend passed in the command line.
            CHECK_STATUS(vpiSubmitPerspectiveWarp(stream, 0, warp, imgInput, xform, imgOutput, VPI_INTERP_LINEAR,
                                                  VPI_BORDER_ZERO, 0));
 
            // Convert output back to BGR using CUDA
            CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, imgOutput, frameBGR, NULL));
            CHECK_STATUS(vpiStreamSync(stream));
 
            // Now add it to the output video stream
            VPIImageData imgdata;
            CHECK_STATUS(vpiImageLock(frameBGR, VPI_LOCK_READ, &imgdata));
 
            cv::Mat outFrame;
            CHECK_STATUS(vpiImageDataExportOpenCVMat(imgdata, &outFrame));
            outVideo << outFrame;
 
            CHECK_STATUS(vpiImageUnlock(frameBGR));
        }
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // =========================
    // Destroy all VPI resources
 
    vpiStreamDestroy(stream);
    vpiPayloadDestroy(warp);
    vpiImageDestroy(imgInput);
    vpiImageDestroy(imgOutput);
    vpiImageDestroy(frameBGR);
 
    return retval;
}
 
// vim: ts=8:sw=4:sts=4:et:ai