Image View

Overview

The Image View application splits the input image in 4 views, in a 2x2 grid, processing each image view differently, potentially in parallel and using different backends. The processing consumes the same input parent image and produce a single output, saving the result as an image file on disk. The user defines the input image for processing. The backends used for each processing, and each algorithm and their parameters, are fixed as follows:

1. CPU Bilateral filter
2. CPU Image flip in horizontal direction
3. CUDA Image flip in vertical direction
4. CUDA Image flip in both directions

Instructions

The command line parameters are:

<input image>

where

• input image: input image file name; it accepts png, jpeg and possibly others.

Here's one example:

• C++

  ./vpi_sample_15_image_view ../assets/kodim08.png

• Python

  python3 main.py ../assets/kodim08.png

This uses one of the provided sample images. You can try with other images, respecting the constraints imposed by each algorithm running on each image view.

Results

Input image	Output image

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++ Python

import sys
import vpi
import numpy as np
from PIL import Image
from argparse import ArgumentParser
 
# Parse command line arguments
parser = ArgumentParser()
parser.add_argument('input', help='Image to be used as input')
 
args = parser.parse_args();
 
# Load input parent image into a vpi.Image
try:
    input = vpi.asimage(np.asarray(Image.open(args.input)))
except IOError:
    sys.exit("Input file not found")
except:
    sys.exit("Error with input file")
 
# Create 4 streams for independent processing
streams  = []
for _ in range(4):
    streams.append(vpi.Stream())
 
# Create input parent image and output as grayscale images in the CPU using numpy
np_parent = np.zeros((input.height, input.width), np.uint8)
np_output = np.zeros((input.height, input.width), np.uint8)
 
# Wrap the input parent and output using VPI, this is important
# because views on Tegra devices can only be created on wrapped CPU or
# CUDA buffers.  If an image is created with vpi.Image, and there is a
# different backend available, the view cannot be created.
parent = vpi.asimage(np_parent)
output = vpi.asimage(np_output)
 
# Convert input parent image to grayscale
with vpi.Backend.CPU, streams[0]:
    input.convert(out=parent)
 
# Calculate views size: width x height
viewSize = (input.width // 2, input.height // 2)
 
# Define clip bounds for each view
clipBounds = vpi.RectangleI(0, 0, viewSize[0], viewSize[1])
 
# Create 4 input and output child image views
parViews = []
outViews = []
for i in range(4):
    clipBounds.x = (i % 2) * viewSize[0]
    clipBounds.y = (i // 2) * viewSize[1]
    parViews.append(parent.view(clipBounds))
    outViews.append(output.view(clipBounds))
 
# Run one algorithm on each view:
# Each algorithm runs on a different stream, potentially in parallel, all streams working together
# consuming the same input to produce a single output, using CPU and CUDA backends
 
with vpi.Backend.CPU:
    parViews[0].bilateral_filter(3, 45, 35, out=outViews[0], border=vpi.Border.ZERO, stream=streams[0])
    parViews[1].image_flip(vpi.Flip.HORIZ, out=outViews[1], stream=streams[1])
 
with vpi.Backend.CUDA:
    parViews[2].image_flip(vpi.Flip.VERT, out=outViews[2], stream=streams[2])
    parViews[3].image_flip(vpi.Flip.BOTH, out=outViews[3], stream=streams[3])
 
# Sync all streams to guarantee they finish before reading the output image
for i in range(4):
    streams[i].sync()
 
# Save result to disk
Image.fromarray(output.cpu()).save('output_views_python'+str(sys.version_info[0])+'.png')

#include <opencv2/core/version.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Event.h>
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/BilateralFilter.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/ImageFlip.h>
 
#include <cstring> // for memset
#include <iostream>
#include <sstream>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvImage;
 
    // VPI objects that will be used
    VPIImage imageBGR    = NULL;
    VPIImage inParent    = NULL;
    VPIImage outParent   = NULL;
    VPIEvent parentEvent = NULL;
    VPIImage inViews[4]  = {};
    VPIImage outViews[4] = {};
    VPIStream streams[4] = {};
 
    int retval = 0;
 
    // Views on Tegra devices need parent images to have only CPU or CUDA backends enabled.
    int imgFlags = VPI_BACKEND_CPU | VPI_BACKEND_CUDA;
 
    try
    {
        if (argc != 2)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <input image>");
        }
 
        std::string strInputFileName = argv[1];
 
        // Load the input image
        cvImage = cv::imread(strInputFileName);
        if (cvImage.empty())
        {
            throw std::runtime_error("Can't open '" + strInputFileName + "'");
        }
 
        // Wrap the loaded image into a VPIImage object to be used by VPI
        CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvImage, 0, &imageBGR));
 
        // Retrieve the width and height of the loaded parent image
        const int parentWidth = cvImage.cols, parentHeight = cvImage.rows;
 
        // Calculate the width and height of the child image views
        const int viewWidth = parentWidth / 2, viewHeight = parentHeight / 2;
 
        if ((viewWidth == 0) || (viewHeight == 0))
        {
            throw std::runtime_error(std::string("Input image too small, it must be at least 2x2"));
        }
 
        // Create the input parent image as a single unsigned 8-bit channel
        CHECK_STATUS(vpiImageCreate(parentWidth, parentHeight, VPI_IMAGE_FORMAT_U8, imgFlags, &inParent));
 
        // Create the output parent image, single unsigned 8-bit channel
        CHECK_STATUS(vpiImageCreate(parentWidth, parentHeight, VPI_IMAGE_FORMAT_U8, imgFlags, &outParent));
 
        // Create 1 event for the parent image
        CHECK_STATUS(vpiEventCreate(0, &parentEvent));
 
        for (int i = 0; i < 4; ++i)
        {
            // Create 4 streams to execute algorithms on
            CHECK_STATUS(vpiStreamCreate(0, &streams[i]));
 
            // Define the clip bounds to be the rectangular region of each view inside the parent image
            VPIRectangleI clipBounds;
 
            clipBounds.x      = static_cast<int>(i % 2) * viewWidth;
            clipBounds.y      = static_cast<int>(i / 2) * viewHeight;
            clipBounds.width  = viewWidth;
            clipBounds.height = viewHeight;
 
            // Create each input child image view
            CHECK_STATUS(vpiImageCreateView(inParent, &clipBounds, imgFlags, &inViews[i]));
 
            // Create each output child image view
            CHECK_STATUS(vpiImageCreateView(outParent, &clipBounds, imgFlags, &outViews[i]));
        }
 
        // Run one algorithm on each view:
        // Each algorithm runs on a different stream, potentially in parallel, all streams working together
        // consuming the same input to produce a single output, using CPU and CUDA backends
 
        // Convert the loaded parent image to grayscale
        CHECK_STATUS(vpiSubmitConvertImageFormat(streams[0], VPI_BACKEND_CPU, imageBGR, inParent, NULL));
 
        // Record the parent event announcing the conversion above has finished
        CHECK_STATUS(vpiEventRecord(parentEvent, streams[0]));
 
        // Wait the conversion above to finish, then release all streams for view processing
        CHECK_STATUS(vpiStreamWaitEvent(streams[1], parentEvent));
        CHECK_STATUS(vpiStreamWaitEvent(streams[2], parentEvent));
        CHECK_STATUS(vpiStreamWaitEvent(streams[3], parentEvent));
 
        // Submit an algorithm on 1st view
        CHECK_STATUS(
            vpiSubmitBilateralFilter(streams[0], VPI_BACKEND_CPU, inViews[0], outViews[0], 3, 45, 35, VPI_BORDER_ZERO));
 
        // Submit an algorithm on 2nd view
        CHECK_STATUS(vpiSubmitImageFlip(streams[1], VPI_BACKEND_CPU, inViews[1], outViews[1], VPI_FLIP_HORIZ));
 
        // Submit an algorithm on 3rd view
        CHECK_STATUS(vpiSubmitImageFlip(streams[2], VPI_BACKEND_CUDA, inViews[2], outViews[2], VPI_FLIP_VERT));
 
        // Submit an algorithm on 4th view
        CHECK_STATUS(vpiSubmitImageFlip(streams[3], VPI_BACKEND_CUDA, inViews[3], outViews[3], VPI_FLIP_BOTH));
 
        // Synchronize all streams to make sure all views finished processing
        CHECK_STATUS(vpiStreamSync(streams[0]));
        CHECK_STATUS(vpiStreamSync(streams[1]));
        CHECK_STATUS(vpiStreamSync(streams[2]));
        CHECK_STATUS(vpiStreamSync(streams[3]));
 
        // Retrieve the output parent image contents and write it to disk
 
        // Lock output parent image to retrieve its data, checking it consists of
        // host-accessible memory buffers in pitch-linear layout
        VPIImageData outData;
        CHECK_STATUS(vpiImageLockData(outParent, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &outData));
        assert(outData.bufferType == VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR);
 
        cv::Mat cvOut;
        vpiImageDataExportOpenCVMat(outData, &cvOut);
        imwrite("output_views.png", cvOut);
 
        // Done handling output parent image, don't forget to unlock it
        CHECK_STATUS(vpiImageUnlock(outParent));
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // Clean up
 
    // Make sure all streams are synchronized before destroying the objects possibly in use
    for (int i = 0; i < 4; ++i)
    {
        if (streams[i] != NULL)
        {
            vpiStreamSync(streams[i]);
        }
    }
 
    // Make sure to destroy all views before destroying their parents
    for (int i = 0; i < 4; ++i)
    {
        vpiStreamDestroy(streams[i]);
 
        vpiImageDestroy(inViews[i]);
        vpiImageDestroy(outViews[i]);
    }
 
    vpiEventDestroy(parentEvent);
 
    vpiImageDestroy(imageBGR);
    vpiImageDestroy(inParent);
    vpiImageDestroy(outParent);
 
    return retval;
}