Harris Corners Detector

Overview

This application detects Harris corners from a given input image. The corners are then drawn onto the input image and the result is saved as an image on disk.

Instructions

The command line parameters are:

<backend> <input image>

where

• backend: either cpu, cuda or pva; it defines the backend that will perform the processing.
• input image: input image file name, it accepts png, jpeg and possibly others.

Here's one example:

• C++

  ./vpi_sample_03_harris_corners pva ../assets/kodim08.png

• Python

  python3 main.py pva ../assets/kodim08.png

This is using the PVA backend and one of the provided sample images. You can try with other input images, respecting the constraints imposed by the algorithm. If your stream doesn't support PVA, an error is printed. In this case, just try another backend.

Results

Input image	Output image, Harris keypoints

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++ Python

import sys
import vpi
import numpy as np
from argparse import ArgumentParser
import cv2
 
 
# ----------------------------
# Parse command line arguments
 
parser = ArgumentParser()
parser.add_argument('backend', choices=['cpu','cuda','pva'],
                    help='Backend to be used for processing')
 
parser.add_argument('input',
                    help='Input image on which harris corners will be detected')
 
args = parser.parse_args();
 
if args.backend == 'cpu':
    backend = vpi.Backend.CPU
elif args.backend == 'cuda':
    backend = vpi.Backend.CUDA
else:
    assert args.backend == 'pva'
    backend = vpi.Backend.PVA
 
# --------------------------------------------------------------
# Load input into a vpi.Image and convert it to grayscale, signed 16bpp
with vpi.Backend.CUDA:
    input = vpi.asimage(cv2.imread(args.input), vpi.Format.BGR8).convert(vpi.Format.S16)
 
with backend:
    corners, scores = input.harriscorners(sensitivity=0.01)
 
# ---------------------------------------
# Render the keypoints in the output image
 
out = input.convert(vpi.Format.BGR8, backend=vpi.Backend.CUDA)
 
if corners.size > 0:
    with out.lock_cpu() as out_data, scores.lock_cpu() as scores_data, corners.lock_cpu() as corners_data:
        cmap = cv2.applyColorMap(np.arange(0, 256, dtype=np.uint8), cv2.COLORMAP_HOT)
 
        maxscore = scores_data.max()
 
        for i in range(corners.size):
            color = tuple([int(x) for x in cmap[255*scores_data[i]//maxscore,0]])
            kpt = tuple(corners_data[i].astype(np.int16))
            cv2.circle(out_data, kpt, 3, color, -1)
 
# -------------------
# Save result to disk
cv2.imwrite('harris_corners_python'+str(sys.version_info[0])+'_'+args.backend+'.png', out.cpu())

#include <opencv2/core/version.hpp>
#if CV_MAJOR_VERSION >= 3
#    include <opencv2/imgcodecs.hpp>
#else
#    include <opencv2/contrib/contrib.hpp> // for applyColorMap
#    include <opencv2/highgui/highgui.hpp>
#endif
 
#include <opencv2/imgproc/imgproc.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Array.h>
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/HarrisCorners.h>
 
#include <cstdio>
#include <cstring> // for memset
#include <iostream>
#include <sstream>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
static cv::Mat DrawKeypoints(cv::Mat img, VPIKeypointF32 *kpts, uint32_t *scores, int numKeypoints)
{
    cv::Mat out;
    img.convertTo(out, CV_8UC1);
    cvtColor(out, out, cv::COLOR_GRAY2BGR);
 
    if (numKeypoints == 0)
    {
        return out;
    }
 
    // prepare our colormap
    cv::Mat cmap(1, 256, CV_8UC3);
    {
        cv::Mat gray(1, 256, CV_8UC1);
        for (int i = 0; i < 256; ++i)
        {
            gray.at<unsigned char>(0, i) = i;
        }
        applyColorMap(gray, cmap, cv::COLORMAP_HOT);
    }
 
    float maxScore = *std::max_element(scores, scores + numKeypoints);
 
    for (int i = 0; i < numKeypoints; ++i)
    {
        cv::Vec3b color = cmap.at<cv::Vec3b>(scores[i] / maxScore * 255);
        circle(out, cv::Point(kpts[i].x, kpts[i].y), 3, cv::Scalar(color[0], color[1], color[2]), -1);
    }
 
    return out;
}
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvImage;
 
    // VPI objects that will be used
    VPIImage imgInput     = NULL;
    VPIImage imgGrayscale = NULL;
    VPIArray keypoints    = NULL;
    VPIArray scores       = NULL;
    VPIStream stream      = NULL;
    VPIPayload harris     = NULL;
 
    int retval = 0;
 
    try
    {
        // =============================
        // Parse command line parameters
 
        if (argc != 3)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|pva|cuda> <input image>");
        }
 
        std::string strBackend       = argv[1];
        std::string strInputFileName = argv[2];
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cpu")
        {
            backend = VPI_BACKEND_CPU;
        }
        else if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else if (strBackend == "pva")
        {
            backend = VPI_BACKEND_PVA;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend +
                                     "' not recognized, it must be either cpu, cuda or pva.");
        }
 
        // =====================
        // Load the input image
 
        cvImage = cv::imread(strInputFileName);
        if (cvImage.empty())
        {
            throw std::runtime_error("Can't open '" + strInputFileName + "'");
        }
 
        // =================================
        // Allocate all VPI resources needed
 
        // Create adjusted backend type that includes CPU and CUDA
        VPIBackend adjBackend = static_cast<VPIBackend>(backend | VPI_BACKEND_CUDA | VPI_BACKEND_CPU);
 
        // Create the stream where processing will happen
        CHECK_STATUS(vpiStreamCreate(adjBackend, &stream));
 
        // We now wrap the loaded image into a VPIImage object to be used by VPI.
        // VPI won't make a copy of it, so the original
        // image must be in scope at all times.
        CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvImage, adjBackend, &imgInput));
 
        CHECK_STATUS(vpiImageCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_FORMAT_S16, adjBackend, &imgGrayscale));
 
        // Create the output keypoint array. Currently for PVA backend it must have 8192 elements.
        CHECK_STATUS(vpiArrayCreate(8192, VPI_ARRAY_TYPE_KEYPOINT_F32, adjBackend, &keypoints));
 
        // Create the output scores array. It also must have 8192 elements and elements must be uint32_t.
        CHECK_STATUS(vpiArrayCreate(8192, VPI_ARRAY_TYPE_U32, adjBackend, &scores));
 
        // Create the payload for Harris Corners Detector algorithm
        CHECK_STATUS(vpiCreateHarrisCornerDetector(backend, cvImage.cols, cvImage.rows, &harris));
 
        // Define the algorithm parameters. We'll use defaults, expect for sensitivity.
        VPIHarrisCornerDetectorParams harrisParams;
        CHECK_STATUS(vpiInitHarrisCornerDetectorParams(&harrisParams));
        harrisParams.sensitivity = 0.01;
 
        // ================
        // Processing stage
 
        // First convert input to grayscale
        CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, imgInput, imgGrayscale, NULL));
 
        // Then get Harris corners
        CHECK_STATUS(
            vpiSubmitHarrisCornerDetector(stream, backend, harris, imgGrayscale, keypoints, scores, &harrisParams));
 
        // Wait until the algorithm finishes processing
        CHECK_STATUS(vpiStreamSync(stream));
 
        // =======================================
        // Output processing and saving it to disk
 
        // Lock output keypoints and scores to retrieve its data on cpu memory
        VPIArrayData outKeypointsData;
        VPIArrayData outScoresData;
        VPIImageData imgData;
        CHECK_STATUS(vpiArrayLockData(keypoints, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &outKeypointsData));
        CHECK_STATUS(vpiArrayLockData(scores, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &outScoresData));
        CHECK_STATUS(vpiImageLockData(imgGrayscale, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &imgData));
 
        VPIKeypointF32 *outKeypoints = (VPIKeypointF32 *)outKeypointsData.buffer.aos.data;
        uint32_t *outScores          = (uint32_t *)outScoresData.buffer.aos.data;
 
        printf("\n%d keypoints found\n", *outKeypointsData.buffer.aos.sizePointer);
 
        cv::Mat img;
        CHECK_STATUS(vpiImageDataExportOpenCVMat(imgData, &img));
 
        cv::Mat outImage = DrawKeypoints(img, outKeypoints, outScores, *outKeypointsData.buffer.aos.sizePointer);
 
        imwrite("harris_corners_" + strBackend + ".png", outImage);
 
        // Done handling outputs, don't forget to unlock them.
        CHECK_STATUS(vpiImageUnlock(imgGrayscale));
        CHECK_STATUS(vpiArrayUnlock(scores));
        CHECK_STATUS(vpiArrayUnlock(keypoints));
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // ========
    // Clean up
 
    // Make sure stream is synchronized before destroying the objects
    // that might still be in use.
    if (stream != NULL)
    {
        vpiStreamSync(stream);
    }
 
    vpiImageDestroy(imgInput);
    vpiImageDestroy(imgGrayscale);
    vpiArrayDestroy(keypoints);
    vpiArrayDestroy(scores);
    vpiPayloadDestroy(harris);
    vpiStreamDestroy(stream);
 
    return retval;
}