ORB Feature Detector

Overview

Orientated FAST and rBRIEF (ORB) is a feature detection and description algorithm. It detects features across an input pyramid as well as a descriptor for each feature, returning the coordinates for each feature as well as its associated bitstring descriptor. This sample application detects ORB features from the given input image, prints the keypoints and descriptors to stdout, and draws the keypoints on the output image file. Green keypoints have valid descriptors, blue keypoints (descriptors with all values zero) don't. The cuda backend does not compute descriptors close to the border of the image, whereas the cpu backend does. Due to implementation differences, cuda and cpu do not return the same results.

Instructions

The command line parameters are:

<backend> <input image>

where

• backend: either cpu or cuda; it defines the backend that will perform the processing.
• input image: input image file name to be used as source image, it accepts png, jpeg and possibly others.

Here's one example:

• C++

  ./vpi_sample_18_orb_feature_detector cuda ../assets/kodim08.png

This is using the CUDA backend and one of the provided sample images. You can try with other images, respecting the constraints imposed by the algorithm.

Results

Input image	Matched featurs between input image and flipped image image

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++

#include <opencv2/core.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Array.h>
#include <vpi/Image.h>
#include <vpi/Pyramid.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/GaussianPyramid.h>
#include <vpi/algo/ImageFlip.h>
#include <vpi/algo/ORB.h>
 
#include <bitset>
#include <cstdio>
#include <cstring> // for memset
#include <iostream>
#include <sstream>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
static cv::Mat DrawKeypoints(cv::Mat img, VPIKeypointF32 *kpts, VPIBriefDescriptor *outDescriptors, int numKeypoints)
{
    cv::Mat out;
    img.convertTo(out, CV_8UC1);
    cvtColor(out, out, cv::COLOR_GRAY2BGR);
 
    if (numKeypoints == 0)
    {
        return out;
    }
 
    for (int i = 0; i < numKeypoints; ++i)
      {
    auto de=outDescriptors[i];
    cv::Scalar col(255,0,0);
    for(int j=0; j<32;j++){
      if(de.data[j]>0) { col=cv::Scalar(0,255, 0); break;}
    }
    circle(out, cv::Point(kpts[i].x, kpts[i].y), 3, col, -1);
      }
    return out;
}
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvImage;
 
    // VPI objects that will be used
    VPIImage imgInput     = NULL;
    VPIImage imgGrayScale = NULL;
 
    VPIPyramid pyrInput   = NULL;
    VPIArray keypoints    = NULL;
    VPIArray descriptors  = NULL;
    VPIPayload orbPayload = NULL;
    VPIStream stream      = NULL;
 
    int retval = 0;
 
    try
    {
        // =============================
        // Parse command line parameters
 
        if (argc != 3)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|cuda> <input image>");
        }
 
        std::string strBackend       = argv[1];
        std::string strInputFileName = argv[2];
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cpu")
        {
            backend = VPI_BACKEND_CPU;
        }
        else if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend + "' not recognized, it must be either cpu or cuda.");
        }
 
        // =====================
        // Load the input image
 
        cvImage = cv::imread(strInputFileName);
        if (cvImage.empty())
        {
            throw std::runtime_error("Can't open first image: '" + strInputFileName + "'");
        }
 
        // =================================
        // Allocate all VPI resources needed
 
        // Create the stream where processing will happen
        CHECK_STATUS(vpiStreamCreate(0, &stream));
 
        // Define the algorithm parameters.
        VPIORBParams orbParams;
        CHECK_STATUS(vpiInitORBParams(&orbParams));
    //default params
    //params.fastParams.circleRadius       = 3;
    //params.fastParams.arcLength          = 9;
    //params.fastParams.intensityThreshold = 20;
    //params.fastParams.nonMaxSuppression  = 1;
    //params.maxFeatures                   = 100;
    //params.pyramidLevels                 = 4;
    //params.enableRBRIEF                  = true;
    //params.scoreType                     = VPI_CORNER_SCORE_HARRIS;
 
 
    orbParams.scoreType=VPI_CORNER_SCORE_HARRIS;
    //orbParams.scoreType=VPI_CORNER_SCORE_FAST;
    orbParams.maxFeatures=10000;
    orbParams.pyramidLevels=6;
 
    VPIBorderExtension be=VPI_BORDER_CLAMP; //VPI_BORDER_ZERO
    
        // We now wrap the loaded image into a VPIImage object to be used by VPI.
        // VPI won't make a copy of it, so the original
        // image must be in scope at all times.
        CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvImage, 0, &imgInput));
        CHECK_STATUS(vpiImageCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_FORMAT_U8, 0, &imgGrayScale));
 
        // Create the output keypoint array.
        CHECK_STATUS( vpiArrayCreate(orbParams.maxFeatures, VPI_ARRAY_TYPE_KEYPOINT_F32, backend | VPI_BACKEND_CPU, &keypoints));
 
        // Create the output descriptors array.
        CHECK_STATUS(vpiArrayCreate(orbParams.maxFeatures, VPI_ARRAY_TYPE_BRIEF_DESCRIPTOR, backend | VPI_BACKEND_CPU, &descriptors));
 
        // Create the payload for ORB Feature Detector algorithm
        CHECK_STATUS( vpiCreateORBFeatureDetector(backend, 20000, &orbPayload));
 
        // ================
        // Processing stage
 
        // First convert input to grayscale
        CHECK_STATUS(vpiSubmitConvertImageFormat(stream, backend, imgInput, imgGrayScale, NULL));
    
        // Then, create the Gaussian Pyramid for the image and wait for the execution to finish
        CHECK_STATUS(vpiPyramidCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_FORMAT_U8, orbParams.pyramidLevels, 0.5, backend, &pyrInput));
        CHECK_STATUS(vpiSubmitGaussianPyramidGenerator(stream, backend, imgGrayScale, pyrInput, be));
 
        // Then get ORB features and wait for the execution to finish
        CHECK_STATUS(vpiSubmitORBFeatureDetector(stream, backend, orbPayload, pyrInput, keypoints, descriptors,  &orbParams, be));
 
        CHECK_STATUS(vpiStreamSync(stream));
 
        // =======================================
        // Output processing and saving it to disk
 
        // Lock output keypoints and scores to retrieve its data on cpu memory
        VPIArrayData outKeypointsData;
    VPIArrayData outDescriptorsData;
        VPIImageData imgData;
        CHECK_STATUS(vpiArrayLockData(keypoints, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &outKeypointsData));
    CHECK_STATUS(vpiArrayLockData(descriptors, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &outDescriptorsData));
        CHECK_STATUS(vpiImageLockData(imgGrayScale, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &imgData));
 
        VPIKeypointF32 *outKeypoints = (VPIKeypointF32 *)outKeypointsData.buffer.aos.data;
    VPIBriefDescriptor *outDescriptors = (VPIBriefDescriptor *) outDescriptorsData.buffer.aos.data;
 
    int nkp = *outKeypointsData.buffer.aos.sizePointer;
    int nde = *outDescriptorsData.buffer.aos.sizePointer;
 
    printf("%d keypoints found\n", nkp);
    for(int i=0; i<nkp;i++)
    {
      printf("%d %f %f\n",i, (outKeypoints[i]).x,(outKeypoints[i]).y);
    }
 
    printf("%d descriptors found\n", nde);
    for(int i=0; i<nde;i++)
       {
      printf("%d: ",i);
      for(int j=0; j<VPI_BRIEF_DESCRIPTOR_ARRAY_LENGTH; j++)
      {
        printf("%hhu ", (outDescriptors[i]).data[j]);
      }
      printf("\n");
    }
 
        cv::Mat img;
        CHECK_STATUS(vpiImageDataExportOpenCVMat(imgData, &img));
 
        cv::Mat outImage = DrawKeypoints(img, outKeypoints, outDescriptors, *outKeypointsData.buffer.aos.sizePointer);
 
        imwrite("orb_feature_detector_" + strBackend + ".png", outImage);
 
        // Done handling outputs, don't forget to unlock them.
        CHECK_STATUS(vpiImageUnlock(imgGrayScale));
        CHECK_STATUS(vpiArrayUnlock(keypoints));
    CHECK_STATUS(vpiArrayUnlock(descriptors));
    
    
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // ========
    // Clean up
 
    // Make sure stream is synchronized before destroying the objects
    // that might still be in use.
    vpiStreamSync(stream);
 
    vpiImageDestroy(imgInput);
    vpiImageDestroy(imgGrayScale);
    vpiArrayDestroy(keypoints);
    vpiArrayDestroy(descriptors);
    vpiPayloadDestroy(orbPayload);
    vpiStreamDestroy(stream);
 
    return retval;
}