Template Matching

Overview

Template Matching is a method for searching and finding the location of a template image in the larger source image. The output is the image holding the matching score of the template image in each location. The maximum location in the output corresponds to the best match location.

Instructions

The command line parameters are:

<backend> <input image> <bounding box>

where

• backend: either cpu or cuda; it defines the backend that will perform the processing.
• input image: input image file name to be used as source image, it accepts png, jpeg and possibly others.
• bounding box: extract the defined bounding box from the input image and use that as template image.

Here's one example:

• C++

  ./vpi_sample_17_template_matching cuda ../assets/kodim08.png 100,200,100,100

• Python

  python3 main.py cuda ../assets/kodim08.png 100,200,100,100

This is using the CUDA backend and one of the provided sample images. You can try with other images, respecting the constraints imposed by the algorithm.

Results

Input image	Output image converted to U8

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++ Python

import sys
import vpi
import numpy as np
from PIL import Image, ImageOps
from argparse import ArgumentParser
import cv2
 
# Parse command line arguments
parser = ArgumentParser()
parser.add_argument('backend', choices=['cpu','cuda'],
                    help='Backend to be used for processing')
 
parser.add_argument('s', metavar='filename',
                    help='Image to be used as source')
 
parser.add_argument('t', metavar='X,Y,W,H',
                    help='Use specified bounding box of source as template image')
 
args = parser.parse_args()
 
if args.backend == 'cpu':
    backend = vpi.Backend.CPU
elif args.backend == 'cuda':
    backend = vpi.Backend.CUDA
else:
    sys.exit("Un-supported backend")
 
try:
    templBbox = np.array([int(x) for x in args.t.split(',')])
except ValueError:
    exit("Error parsing template bounding box")
 
# Load input into a vpi.Image
try:
    srcData = np.asarray(ImageOps.grayscale(Image.open(args.s)))
except IOError:
    sys.exit("Source file not found")
except:
    sys.exit("Error with source file")
 
src = vpi.asimage(srcData)
templData = srcData[templBbox[1]:templBbox[1]+templBbox[3], templBbox[0]:templBbox[0]+templBbox[2]]
templ = vpi.asimage(templData)
 
# Using the chosen backend,
with backend:
    output = vpi.templateMatching(src, templ)
 
    temp = output.convert(vpi.Format.F32, backend=vpi.Backend.CUDA, scale = 255)
 
    min_coords, max_coords = temp.minmaxloc(min_capacity=10000, max_capacity=10000)
 
    output = temp.convert(vpi.Format.U8, backend=vpi.Backend.CUDA)
 
# Lock output
with max_coords.rlock_cpu() as max_data:
    max_loc = tuple(max_data[0].astype(int)[::-1])
 
# Save result to disk
print('Provided coord of bounding box for the template is [{}, {}] with w={} and h={}'.format(templBbox[1], templBbox[0], templBbox[2], templBbox[3]))
print('Template matching location coord is [{}, {}]'.format(max_loc[0], max_loc[1]))
outputData = output.cpu()
cv2.rectangle(outputData, (max_loc[1]-20, max_loc[0]-20), (max_loc[1]+20, max_loc[0]+20), 255, 2)
Image.fromarray(outputData).save('template_matching_score_python'+str(sys.version_info[0])+'_'+args.backend+'.png')

#include <opencv2/core/version.hpp>
#if CV_MAJOR_VERSION >= 3
#    include <opencv2/imgcodecs.hpp>
#else
#    include <opencv2/highgui/highgui.hpp>
#endif
#include <opencv2/imgproc/imgproc.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Array.h>
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/MinMaxLoc.h>
#include <vpi/algo/TemplateMatching.h>
 
#include <cassert>
#include <cstring> // for memset
#include <iostream>
#include <sstream>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvImage;
    cv::Mat cvImageU8;
 
    cv::Mat cvTempl;
 
    VPIStream stream = NULL;
 
    // VPI objects that will be used
    VPIImage input = NULL;
    VPIImage templ = NULL;
    int originX, originY, templWidth, templHeight;
 
    int outWidth, outHeight;
    VPIImage output       = NULL;
    VPIImage outputScaled = NULL;
    VPIImage outputU8     = NULL;
 
    VPIArray minCoords = NULL;
    VPIArray maxCoords = NULL;
 
    VPIPayload payload       = NULL;
    VPIPayload payloadMinMax = NULL;
 
    int retval = 0;
 
    try
    {
        if (argc != 4)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|cuda> <input image> <x,y,w,h> ");
        }
 
        std::string strBackend       = argv[1];
        std::string strInputFileName = argv[2];
 
        if (sscanf(argv[3], "%d,%d,%d,%d", &originX, &originY, &templWidth, &templHeight) != 4)
        {
            throw std::runtime_error(
                "Invalid data format. Specify the bounding box of the input image as template image.");
        }
 
        // Load the input image
        cvImage = cv::imread(strInputFileName);
        if (cvImage.empty())
        {
            throw std::runtime_error("Can't open '" + strInputFileName + "'");
        }
 
        assert(cvImage.type() == CV_8UC3);
 
        // convert image to gray scale
        cvtColor(cvImage, cvImageU8, cv::COLOR_BGR2GRAY);
 
        if (originX + templWidth > cvImage.cols || originY + templHeight > cvImage.rows)
        {
            throw std::runtime_error("Bounding box is out of range of input image size");
        }
 
        cv::Rect templROI(originX, originY, templWidth, templHeight);
        cv::Mat croppedRef(cvImageU8, templROI);
        croppedRef.copyTo(cvTempl);
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cpu")
        {
            backend = VPI_BACKEND_CPU;
        }
        else if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend + "' not recognized, it must be either cpu, cuda");
        }
 
        // 1. Initialization phase ---------------------------------------
 
        CHECK_STATUS(vpiStreamCreate(backend, &stream));
 
        // We now wrap the loaded image into a VPIImage object to be used by VPI.
        // VPI won't make a copy of it, so the original
        // image must be in scope at all times.
        CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvImageU8, 0, &input));
 
        // Create template iamge
        CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvTempl, 0, &templ));
 
        // Now create the output image.
        outWidth  = cvImage.cols - templWidth + 1;
        outHeight = cvImage.rows - templHeight + 1;
        CHECK_STATUS(vpiImageCreate(outWidth, outHeight, VPI_IMAGE_FORMAT_F32, 0, &output));
        CHECK_STATUS(vpiImageCreate(outWidth, outHeight, VPI_IMAGE_FORMAT_F32, 0, &outputScaled));
        CHECK_STATUS(vpiImageCreate(outWidth, outHeight, VPI_IMAGE_FORMAT_U8, 0, &outputU8));
 
        CHECK_STATUS(vpiArrayCreate(10000, VPI_ARRAY_TYPE_KEYPOINT_F32, 0, &minCoords));
        CHECK_STATUS(vpiArrayCreate(10000, VPI_ARRAY_TYPE_KEYPOINT_F32, 0, &maxCoords));
 
        // Create payload
        CHECK_STATUS(vpiCreateTemplateMatching(backend, cvImage.cols, cvImage.rows, &payload));
 
        CHECK_STATUS(vpiCreateMinMaxLoc(backend, outWidth, outHeight, VPI_IMAGE_FORMAT_F32, &payloadMinMax));
 
        // 2. Computation phase ---------------------------------------
 
        // Set source image
        CHECK_STATUS(vpiTemplateMatchingSetSourceImage(stream, backend, payload, input));
 
        // Set source image
        CHECK_STATUS(vpiTemplateMatchingSetTemplateImage(stream, backend, payload, templ, NULL));
 
        // Submit
        CHECK_STATUS(vpiSubmitTemplateMatching(stream, backend, payload, output, VPI_TEMPLATE_MATCHING_NCC));
 
        CHECK_STATUS(vpiSubmitMinMaxLoc(stream, backend, payloadMinMax, output, minCoords, maxCoords));
 
        // Convert output from F32 to U8
        VPIConvertImageFormatParams params;
        CHECK_STATUS(vpiInitConvertImageFormatParams(&params));
        params.scale = 255;
        CHECK_STATUS(vpiSubmitConvertImageFormat(stream, backend, output, outputScaled, &params));
 
        CHECK_STATUS(vpiSubmitConvertImageFormat(stream, backend, outputScaled, outputU8, NULL));
 
        // Wait until the algorithm finishes processing
        CHECK_STATUS(vpiStreamSync(stream));
 
        // Now let's retrieve the output image contents and output it to disk
        {
            // Lock output image to retrieve its data on cpu memory
            VPIArrayData maxCoordsData;
            CHECK_STATUS(vpiArrayLockData(maxCoords, VPI_LOCK_READ, VPI_ARRAY_BUFFER_HOST_AOS, &maxCoordsData));
 
            VPIKeypointF32 *max_coords = (VPIKeypointF32 *)maxCoordsData.buffer.aos.data;
            int max_i                  = max_coords[0].y;
            int max_j                  = max_coords[0].x;
 
            VPIImageData outData;
            CHECK_STATUS(vpiImageLockData(outputU8, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &outData));
 
            // Returned data consists of host-accessible memory buffers in pitch-linear layout.
            assert(outData.bufferType == VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR);
 
            cv::Mat cvOut;
            CHECK_STATUS(vpiImageDataExportOpenCVMat(outData, &cvOut));
 
            cv::Rect rect(max_i - 20, max_j - 20, 40, 40);
            rectangle(cvOut, rect, 255, 2);
            imwrite("template_matching_score_" + strBackend + ".png", cvOut);
 
            printf("Provided coord of bounding box for the template is [%d, %d] with w=%d and h=%d \n", originY,
                   originX, templWidth, templHeight);
            printf("Template matching location coord is [%d, %d] \n", max_j, max_i);
 
            // Done handling output image, don't forget to unlock it.
            CHECK_STATUS(vpiArrayUnlock(maxCoords));
            CHECK_STATUS(vpiImageUnlock(outputU8));
        }
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // Clean up
 
    // Make sure stream is synchronized before destroying the objects
    // that might still be in use.
    vpiStreamSync(stream);
 
    vpiPayloadDestroy(payload);
    vpiPayloadDestroy(payloadMinMax);
 
    vpiImageDestroy(input);
    vpiImageDestroy(templ);
    vpiImageDestroy(output);
    vpiImageDestroy(outputU8);
    vpiArrayDestroy(minCoords);
    vpiArrayDestroy(maxCoords);
    vpiStreamDestroy(stream);
 
    return retval;
}