Image FFT

Overview

The Image FFT application outputs a spectrum representation of an input image, saving it into spectrum.png. The user can define what backend will be used for processing.

This sample shows the following VPI features:

• Creating and destroying a VPI device.
• Wrapping an image hosted on CPU (the input) to be used by VPI.
• Convert an image to another format
• Creating a VPI-managed 2D image where output will be written to.
• Create a ImageFFT algorithm and call it.
• Simple device synchronization.
• Image locking to access its contents from CPU side.
• Interpret and process the resulting spectrum information.
• Error handling.
• Environment clean up.

Instructions

The usage is:

./vpi_sample_07_image_fft <backend> <input image>

where

• backend: either cpu or cuda; it defines the backend that will perform the processing.
• input image: input image file name; it accepts png, jpeg and possibly others.

Note

Currently ImageFFT isn't available on PVA backend.

Here's one example:

./vpi_sample_07_image_fft cpu ../assets/kodim8.png

This is using the CUDA backend and one of the provided sample images. You can try with other images, respecting the constraints imposed by the algorithm.

Source code

For convenience, here's the code that is also installed in the samples directory.

/*
* Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*  * Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
*  * Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*  * Neither the name of NVIDIA CORPORATION nor the names of its
*    contributors may be used to endorse or promote products derived
*    from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
 
#include <opencv2/core/version.hpp>
#if CV_MAJOR_VERSION >= 3
#    include <opencv2/imgcodecs.hpp>
#else
#    include <opencv2/highgui/highgui.hpp>
#endif
 
#include <vpi/Image.h>
#include <vpi/Stream.h>
#include <vpi/algo/ImageFFT.h>
#include <vpi/algo/ImageFormatConverter.h>
 
#include <cstring> // for memset
#include <iostream>
 
#define CHECK_STATUS(STMT)                                      \
    do                                                          \
    {                                                           \
        VPIStatus status = (STMT);                              \
        if (status != VPI_SUCCESS)                              \
        {                                                       \
            throw std::runtime_error(vpiStatusGetName(status)); \
        }                                                       \
    } while (0);
 
// Auxiliary functions to process spectrum before saving it to disk.
cv::Mat LogMagnitude(cv::Mat cpx);
cv::Mat CompleteFullHermitian(cv::Mat in, cv::Size fullSize);
cv::Mat InplaceFFTShift(cv::Mat mag);
 
int main(int argc, char *argv[])
{
    VPIImage image    = NULL;
    VPIImage imageF32 = NULL;
    VPIImage spectrum = NULL;
    VPIStream stream  = NULL;
    VPIPayload fft    = NULL;
 
    int retval = 0;
 
    try
    {
        if (argc != 3)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|pva|cuda> <input image>");
        }
 
        std::string strDevType       = argv[1];
        std::string strInputFileName = argv[2];
 
        // Load the input image
        cv::Mat cvImage = cv::imread(strInputFileName, cv::IMREAD_GRAYSCALE);
        if (cvImage.empty())
        {
            throw std::runtime_error("Can't open '" + strInputFileName + "'");
        }
 
        assert(cvImage.type() == CV_8UC1);
 
        // Now process the device type
        VPIDeviceType devType;
 
        if (strDevType == "cpu")
        {
            devType = VPI_DEVICE_TYPE_CPU;
        }
        else if (strDevType == "cuda")
        {
            devType = VPI_DEVICE_TYPE_CUDA;
        }
        else if (strDevType == "pva")
        {
            devType = VPI_DEVICE_TYPE_PVA;
        }
        else
        {
            throw std::runtime_error("Backend '" + strDevType +
                                     "' not recognized, it must be either cpu, cuda or pva.");
        }
 
        // Create the stream for the given backend.
        CHECK_STATUS(vpiStreamCreate(devType, &stream));
 
        // We now wrap the loaded image into a VPIImage object to be used by VPI.
        {
            // First fill VPIImageData with the, well, image data...
            VPIImageData imgData;
            memset(&imgData, 0, sizeof(imgData));
            imgData.type                = VPI_IMAGE_TYPE_U8;
            imgData.numPlanes           = 1;
            imgData.planes[0].width     = cvImage.cols;
            imgData.planes[0].height    = cvImage.rows;
            imgData.planes[0].rowStride = cvImage.step[0];
            imgData.planes[0].data      = cvImage.data;
 
            // Wrap it into a VPIImage. VPI won't make a copy of it, so the original
            // image must be in scope at all times.
            CHECK_STATUS(vpiImageWrapHostMem(&imgData, 0, &image));
        }
 
        // Temporary image that holds the float version of input
        CHECK_STATUS(vpiImageCreate(cvImage.cols, cvImage.rows, VPI_IMAGE_TYPE_F32, 0, &imageF32));
 
        // Now create the output image. Note that for real inputs, the output spectrum is a Hermitian
        // matrix (conjugate-symmetric), so only the non-redundant components are output, basically the
        // left half. We adjust the output width accordingly.
        CHECK_STATUS(vpiImageCreate(cvImage.cols / 2 + 1, cvImage.rows, VPI_IMAGE_TYPE_2F32, 0, &spectrum));
 
        // Create the FFT payload that does real (space) to complex (frequency) transformation
        CHECK_STATUS(
            vpiCreateImageFFT(stream, cvImage.cols, cvImage.rows, VPI_IMAGE_TYPE_F32, VPI_IMAGE_TYPE_2F32, &fft));
 
        // Now our processing pipeline:
 
        // Convert image to float
        CHECK_STATUS(vpiSubmitImageFormatConverter(stream, image, imageF32, VPI_CONVERSION_CAST, 1, 0));
 
        // Submit it for processing passing the image to be gradient and the result image
        CHECK_STATUS(vpiSubmitImageFFT(fft, imageF32, spectrum, 0));
 
        // Wait until the algorithm finishes processing
        CHECK_STATUS(vpiStreamSync(stream));
 
        // Now let's retrieve the result spectrum
        {
            // Lock output image to retrieve its data on cpu memory
            VPIImageData outData;
            CHECK_STATUS(vpiImageLock(spectrum, VPI_LOCK_READ, &outData));
 
            assert(outData.type == VPI_IMAGE_TYPE_2F32);
 
            // Wrap spectrum to be used by OpenCV
            cv::Mat cvSpectrum(outData.planes[0].height, outData.planes[0].width, CV_32FC2, outData.planes[0].data,
                               outData.planes[0].rowStride);
 
            // Process it
            cv::Mat mag = InplaceFFTShift(LogMagnitude(CompleteFullHermitian(cvSpectrum, cvImage.size())));
 
            // Normalize the result to fit in 8-bits
            normalize(mag, mag, 0, 255, cv::NORM_MINMAX);
 
            // Write to disk
            imwrite("spectrum_" + strDevType + ".png", mag);
 
            // Done handling output image, don't forget to unlock it.
            CHECK_STATUS(vpiImageUnlock(spectrum));
        }
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // Clean up
    vpiStreamDestroy(stream);
    vpiImageDestroy(image);
    vpiImageDestroy(imageF32);
    vpiImageDestroy(spectrum);
 
    // Payload is owned by the stream, so it's already destroyed
    // since the stream is now destroyed.
 
    return retval;
}
 
// Auxiliary functions --------------------------------
 
cv::Mat LogMagnitude(cv::Mat cpx)
{
    // Split spectrum into real and imaginary parts
    cv::Mat reim[2];
    assert(cpx.channels() == 2);
    split(cpx, reim);
 
    // Calculate the magnitude
    cv::Mat mag;
    magnitude(reim[0], reim[1], mag);
 
    // Convert to logarithm scale
    mag += cv::Scalar::all(1);
    log(mag, mag);
    mag = mag(cv::Rect(0, 0, mag.cols & -2, mag.rows & -2));
 
    return mag;
}
 
cv::Mat CompleteFullHermitian(cv::Mat in, cv::Size fullSize)
{
    assert(in.type() == CV_32FC2);
 
    cv::Mat out(fullSize, CV_32FC2);
    for (int i = 0; i < out.rows; ++i)
    {
        for (int j = 0; j < out.cols; ++j)
        {
            cv::Vec2f p;
            if (j < in.cols)
            {
                p = in.at<cv::Vec2f>(i, j);
            }
            else
            {
                p    = in.at<cv::Vec2f>((out.rows - i) % out.rows, (out.cols - j) % out.cols);
                p[1] = -p[1];
            }
            out.at<cv::Vec2f>(i, j) = p;
        }
    }
 
    return out;
}
 
cv::Mat InplaceFFTShift(cv::Mat mag)
{
    // Rearrange the quadrants of the fourier spectrum
    // so that the origin is at the image center.
 
    // Create a ROI for each 4 quadrants.
    int cx = mag.cols / 2;
    int cy = mag.rows / 2;
    cv::Mat qTL(mag, cv::Rect(0, 0, cx, cy));   // top-left
    cv::Mat qTR(mag, cv::Rect(cx, 0, cx, cy));  // top-right
    cv::Mat qBL(mag, cv::Rect(0, cy, cx, cy));  // bottom-left
    cv::Mat qBR(mag, cv::Rect(cx, cy, cx, cy)); // bottom-right
 
    // swap top-left with bottom-right quadrants
    cv::Mat tmp;
    qTL.copyTo(tmp);
    qBR.copyTo(qTL);
    tmp.copyTo(qBR);
 
    // swap top-right with bottom-left quadrants
    qTR.copyTo(tmp);
    qBL.copyTo(qTR);
    tmp.copyTo(qBL);
 
    return mag;
}

Results

Input image	Output image, spectrum