ORB feature detector

Overview

Oriented FAST and rBRIEF (ORB) is a feature detection and description algorithm. It detects features across an input pyramid as well as a descriptor for each feature, returning the coordinates for each feature as well as its associated bitstring descriptor. This sample application does: (1) read an input image; (2) create a Gaussian pyramid from the image; (3) run ORB on the pyramid; (4) draw the ORB features as key points with colors representing the ORB descriptor; (5) write an output image with the colored key points. Each feature is drawn on top of the input image as a circle, whose color maps from blue to red the descriptor associated with the feature. The map uses the Hamming distance of each descriptor to the first descriptor, hence the blue key point is the first descriptor and shades of yellow to red are progressively distant.

Instructions

The command line parameters are:

<backend> <input image>

where

• backend: either cpu or cuda; it defines the backend that will perform the processing.
• input image: input image file name to be used as source image, it accepts png, jpeg and others.

Here's one example:

• C++

  ./vpi_sample_18_orb_feature_detector cuda ../assets/kodim08.png

• Python

  python3 main.py cuda ../assets/kodim08.png

This is using the CUDA backend and one of the provided sample images. You can try with other images, respecting the constraints imposed by the algorithm.

Results

Input image	Matched featurs between input image and flipped image image

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++ Python

import sys
import vpi
import numpy as np
from PIL import Image, ImageOps
from argparse import ArgumentParser
import cv2
 
 
# Parse command line arguments
parser = ArgumentParser()
parser.add_argument('backend', choices=['cpu','cuda'],
                    help='Backend to be used for processing')
 
parser.add_argument('s', metavar='filename',
                    help='Image to be used as source')
 
args = parser.parse_args()
 
if args.backend == 'cpu':
    backend = vpi.Backend.CPU
elif args.backend == 'cuda':
    backend = vpi.Backend.CUDA
else:
    sys.exit("Un-supported backend")
 
# Load input image into a vpi.Image
try:
    srcData = np.asarray(ImageOps.grayscale(Image.open(args.s)))
except IOError:
    sys.exit("Source file not found")
except:
    sys.exit("Error with source file")
 
src = vpi.asimage(srcData)
 
# Using the chosen backend to build the input pyramid and run ORB
with backend:
    pyr = src.gaussian_pyramid(3)
    corners, descriptors = pyr.orb(intensity_threshold=142, max_features_per_level=88, max_pyr_levels=3)
 
# Draw the keypoints in the output image
 
out = src.convert(vpi.Format.BGR8, backend=vpi.Backend.CUDA)
 
if corners.size > 0:
    distances = []
    with descriptors.rlock_cpu() as descriptors_data:
        first_desc = descriptors_data[0][0]
        for i in range(descriptors.size):
            curr_desc = descriptors_data[i][0]
            hamm_dist = sum([bin(c ^ f).count('1') for c, f in zip(curr_desc, first_desc)])
            distances.append(hamm_dist)
 
    max_dist = max(distances)
 
    cmap = cv2.applyColorMap(np.arange(0, 256, dtype=np.uint8), cv2.COLORMAP_JET)
    cmap_idx = lambda i: int(round((distances[i] / max_dist) * 255))
 
    with out.lock_cpu() as out_data, corners.rlock_cpu() as corners_data:
        for i in range(corners.size):
            color = tuple([int(x) for x in cmap[cmap_idx(i), 0]])
            kpt = tuple(corners_data[i].astype(np.int16))
            x = kpt[0] * (2 ** kpt[2])
            y = kpt[1] * (2 ** kpt[2])
            cv2.circle(out_data, (x, y), 3, color, -1)
 
# Save the output image to disk
cv2.imwrite('orb_feature_python'+str(sys.version_info[0])+'_'+args.backend+'.png', out.cpu())

#include <opencv2/core.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Array.h>
#include <vpi/Image.h>
#include <vpi/Pyramid.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/GaussianPyramid.h>
#include <vpi/algo/ImageFlip.h>
#include <vpi/algo/ORB.h>
 
#include <bitset>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <numeric>
#include <sstream>
#include <vector>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
static cv::Mat DrawKeypoints(cv::Mat img, VPIPyramidalKeypointF32 *kpts, VPIBriefDescriptor *descs, int numKeypoints)
{
    cv::Mat out;
    img.convertTo(out, CV_8UC1);
    cvtColor(out, out, cv::COLOR_GRAY2BGR);
 
    if (numKeypoints == 0)
    {
        return out;
    }
 
    std::vector<int> distances(numKeypoints, 0);
    float maxDist = 0.f;
 
    for (int i = 0; i < numKeypoints; i++)
    {
        for (int j = 0; j < VPI_BRIEF_DESCRIPTOR_ARRAY_LENGTH; j++)
        {
            distances[i] += std::bitset<8 * sizeof(uint8_t)>(descs[i].data[j] ^ descs[0].data[j]).count();
        }
        if (distances[i] > maxDist)
        {
            maxDist = distances[i];
        }
    }
 
    uint8_t ids[256];
    std::iota(&ids[0], &ids[0] + 256, 0);
    cv::Mat idsMat(256, 1, CV_8UC1, ids);
 
    cv::Mat cmap;
    applyColorMap(idsMat, cmap, cv::COLORMAP_JET);
 
    for (int i = 0; i < numKeypoints; i++)
    {
        int cmapIdx = static_cast<int>(std::round((distances[i] / maxDist) * 255));
 
        float rescale = std::pow(2, kpts[i].octave);
        float x       = kpts[i].x * rescale;
        float y       = kpts[i].y * rescale;
 
        circle(out, cv::Point(x, y), 3, cmap.at<cv::Vec3b>(cmapIdx, 0), -1);
    }
 
    return out;
}
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvImage;
 
    // VPI objects that will be used
    VPIImage imgInput     = NULL;
    VPIImage imgGrayScale = NULL;
 
    VPIPyramid pyrInput   = NULL;
    VPIArray keypoints    = NULL;
    VPIArray descriptors  = NULL;
    VPIPayload orbPayload = NULL;
    VPIStream stream      = NULL;
 
    int retval = 0;
 
    try
    {
        // =============================
        // Parse command line parameters
 
        if (argc != 3)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|cuda|pva> <input image>");
        }
 
        std::string strBackend       = argv[1];
        std::string strInputFileName = argv[2];
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cpu")
        {
            backend = VPI_BACKEND_CPU;
        }
        else if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else if (strBackend == "pva")
        {
            backend = VPI_BACKEND_PVA;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend +
                                     "' not recognized, it must be either cpu, cuda or pva.");
        }
 
        // Use the selected backend with CPU to be able to read data back from CUDA to CPU for example.
        const VPIBackend backendWithCPU = static_cast<VPIBackend>(backend | VPI_BACKEND_CPU);
 
        // =====================
        // Load the input image
 
        cvImage = cv::imread(strInputFileName);
        if (cvImage.empty())
        {
            throw std::runtime_error("Can't open first image: '" + strInputFileName + "'");
        }
 
        // =================================
        // Allocate all VPI resources needed
 
        // Create the stream where processing will happen
        CHECK_STATUS(vpiStreamCreate(0, &stream));
 
        // Define the algorithm parameters.
        VPIORBParams orbParams;
        CHECK_STATUS(vpiInitORBParams(&orbParams));
 
        orbParams.fastParams.intensityThreshold = 142;
        orbParams.maxFeaturesPerLevel           = 88;
        orbParams.maxPyramidLevels              = 3;
 
        // We now wrap the loaded image into a VPIImage object to be used by VPI.
        // VPI won't make a copy of it, so the original image must be in scope at all times.
        CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvImage, 0, &imgInput));
        CHECK_STATUS(vpiImageCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_FORMAT_U8, 0, &imgGrayScale));
 
        // For the output arrays capacity we can use the maximum number of features per level multiplied by the
        // maximum number of pyramid levels, this will be the de factor maximum for all levels of the input.
        int outCapacity = orbParams.maxFeaturesPerLevel * orbParams.maxPyramidLevels;
 
        // Create the output keypoint array.
        CHECK_STATUS(vpiArrayCreate(outCapacity, VPI_ARRAY_TYPE_PYRAMIDAL_KEYPOINT_F32, backendWithCPU, &keypoints));
 
        // Create the output descriptors array.  To output corners only use NULL instead.
        CHECK_STATUS(vpiArrayCreate(outCapacity, VPI_ARRAY_TYPE_BRIEF_DESCRIPTOR, backendWithCPU, &descriptors));
 
        // For the internal buffers capacity we can use the maximum number of features per level multiplied by 20.
        // This will make FAST find a large number of corners so then ORB can select the top N corners in
        // accordance to Harris score of each corner, where N = maximum number of features per level.
        int bufCapacity = orbParams.maxFeaturesPerLevel * 20;
 
        // Create the payload for ORB Feature Detector algorithm
        CHECK_STATUS(vpiCreateORBFeatureDetector(backend, bufCapacity, &orbPayload));
 
        // ================
        // Processing stage
 
        // First convert input to grayscale
        CHECK_STATUS(vpiSubmitConvertImageFormat(stream, backend, imgInput, imgGrayScale, NULL));
 
        // Then, create the Gaussian Pyramid for the image and wait for the execution to finish
        CHECK_STATUS(vpiPyramidCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_FORMAT_U8, orbParams.maxPyramidLevels, 0.5,
                                      backend, &pyrInput));
        CHECK_STATUS(vpiSubmitGaussianPyramidGenerator(stream, backend, imgGrayScale, pyrInput, VPI_BORDER_CLAMP));
 
        // Then get ORB features and wait for the execution to finish
        CHECK_STATUS(vpiSubmitORBFeatureDetector(stream, backend, orbPayload, pyrInput, keypoints, descriptors,
                                                 &orbParams, VPI_BORDER_LIMITED));
 
        CHECK_STATUS(vpiStreamSync(stream));
 
        // =======================================
        // Output processing and saving it to disk
 
        // Lock output keypoints and scores to retrieve its data on cpu memory
        VPIArrayData outKeypointsData;
        VPIArrayData outDescriptorsData;
        VPIImageData imgData;
        CHECK_STATUS(vpiArrayLockData(keypoints, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &outKeypointsData));
        CHECK_STATUS(vpiArrayLockData(descriptors, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &outDescriptorsData));
        CHECK_STATUS(vpiImageLockData(imgGrayScale, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &imgData));
 
        VPIPyramidalKeypointF32 *outKeypoints = (VPIPyramidalKeypointF32 *)outKeypointsData.buffer.aos.data;
        VPIBriefDescriptor *outDescriptors    = (VPIBriefDescriptor *)outDescriptorsData.buffer.aos.data;
 
        cv::Mat img;
        CHECK_STATUS(vpiImageDataExportOpenCVMat(imgData, &img));
 
        // Draw the keypoints in the output image
        cv::Mat outImage = DrawKeypoints(img, outKeypoints, outDescriptors, *outKeypointsData.buffer.aos.sizePointer);
 
        // Save the output image to disk
        imwrite("orb_feature_detector_" + strBackend + ".png", outImage);
 
        // Done handling outputs, don't forget to unlock them.
        CHECK_STATUS(vpiImageUnlock(imgGrayScale));
        CHECK_STATUS(vpiArrayUnlock(keypoints));
        CHECK_STATUS(vpiArrayUnlock(descriptors));
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // ========
    // Clean up
 
    // Make sure stream is synchronized before destroying the objects
    // that might still be in use.
    vpiStreamSync(stream);
 
    vpiImageDestroy(imgInput);
    vpiImageDestroy(imgGrayScale);
    vpiArrayDestroy(keypoints);
    vpiArrayDestroy(descriptors);
    vpiPayloadDestroy(orbPayload);
    vpiStreamDestroy(stream);
 
    return retval;
}