Temporal Noise Reduction

Overview

The Temporal Noise Reduction sample application takes a noisy input video, reduces its noise and writes the result to disk. The user can define what backend will be used for processing.

This sample shows the following:

• Creating and destroying a VPI stream.
• Wrapping an OpenCV image to be used by VPI.
• Wrapping a VPI-managed image into an OpenCV's cv::Mat.
• Use OpenCV to fetch frames from a video file.
• Use OpenCV save frames into a video file.
• Create pipeline that does Convert Image Format to convert from/to NV12 format and run Temporal Noise Reduction.
• Shows how to properly use Temporal Noise Reduction to process a video sequence.

Instructions

The usage is:

./vpi_sample_09_tnr <backend> <input video> <output video>

where

• backend: either cuda or pva, as cpu isn't available yet. It defines the backend that will perform the processing.
• input video: video file to have noise reduced; it accepts .mp4, .avi and possibly others, depending on OpenCV's support.
• output video: file to write the de-noised result. It'll use the same codec and fps as input video.

VPI samples installer includes a sample video with noise added artificially, found in /opt/nvidia/vpi-0.4/samples/assets/noisy.mp4.

Here's one invocation example:

./vpi_sample_09_tnr cuda ../assets/noisy.mp4 denoised.mp4

Results

Input video	De-noised video

Source code

For convenience, here's the code that is also installed in the samples directory.

/*
* Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*  * Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
*  * Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*  * Neither the name of NVIDIA CORPORATION nor the names of its
*    contributors may be used to endorse or promote products derived
*    from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
 
#include <opencv2/core/version.hpp>
#if CV_MAJOR_VERSION >= 3
#    include <opencv2/imgcodecs.hpp>
#    include <opencv2/videoio.hpp>
#else
#    include <opencv2/highgui/highgui.hpp>
#endif
 
#include <opencv2/imgproc/imgproc.hpp>
 
#include <vpi/Context.h>
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/TemporalNoiseReduction.h>
 
#include <algorithm>
#include <cstring> // for memset
#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
#include <vector>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
// Utility function to wrap a cv::Mat into a VPIImage
static VPIImage ToVPIImage(VPIImage image, const cv::Mat &frame)
{
    VPIImageData imgData;
    memset(&imgData, 0, sizeof(imgData));
 
    switch (frame.type())
    {
    case CV_8U:
        imgData.type = VPI_IMAGE_FORMAT_U8;
        break;
    case CV_8UC3:
        imgData.type = VPI_IMAGE_FORMAT_BGR8;
        break;
    case CV_8UC4:
        imgData.type = VPI_IMAGE_FORMAT_BGRA8;
        break;
    default:
        throw std::runtime_error("Frame type not supported");
    }
 
    // First fill VPIImageData with the, well, image data...
    imgData.numPlanes            = 1;
    imgData.planes[0].width      = frame.cols;
    imgData.planes[0].height     = frame.rows;
    imgData.planes[0].pitchBytes = frame.step[0];
    imgData.planes[0].data       = frame.data;
 
    if (image == nullptr)
    {
        // Now create a VPIImage that wraps it.
        CHECK_STATUS(vpiImageCreateHostMemWrapper(&imgData, 0, &image));
    }
    else
    {
        // image is already created, we only have to update the wrapped memory,
        // this is done without allocating memory.
        CHECK_STATUS(vpiImageSetWrappedHostMem(image, &imgData));
    }
    return image;
};
 
// Utility function to wrap a VPIImageData into a cv::Mat
static cv::Mat ToCV(const VPIImageData &imgData)
{
    cv::Mat out;
 
    switch (imgData.type)
    {
    case VPI_IMAGE_FORMAT_BGR8:
        out = cv::Mat(imgData.planes[0].height, imgData.planes[0].width, CV_8UC3, imgData.planes[0].data,
                      imgData.planes[0].pitchBytes);
        break;
 
    case VPI_IMAGE_FORMAT_BGRA8:
        out = cv::Mat(imgData.planes[0].height, imgData.planes[0].width, CV_8UC4, imgData.planes[0].data,
                      imgData.planes[0].pitchBytes);
        break;
 
    case VPI_IMAGE_FORMAT_U8:
        out = cv::Mat(imgData.planes[0].height, imgData.planes[0].width, CV_8UC1, imgData.planes[0].data,
                      imgData.planes[0].pitchBytes);
        break;
 
    default:
        throw std::runtime_error("Frame type not supported");
    }
 
    return out;
}
 
int main(int argc, char *argv[])
{
    // We'll create all our objects under this context, so that
    // we don't have to track what objects to destroy. Just destroying
    // the context will destroy all objects.
    VPIContext ctx = nullptr;
 
    int retval = 0;
 
    try
    {
        if (argc != 4)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|vic|cuda> <input_video> <output>");
        }
 
        std::string strBackend     = argv[1];
        std::string strInputVideo  = argv[2];
        std::string strOutputVideo = argv[3];
 
        // Load the input video
        cv::VideoCapture invid;
        if (!invid.open(strInputVideo))
        {
            throw std::runtime_error("Can't open '" + strInputVideo + "'");
        }
 
        // Create our context.
        CHECK_STATUS(vpiContextCreate(0, &ctx));
 
        // Activate it. From now on all created objects will be owned by it.
        CHECK_STATUS(vpiContextSetCurrent(ctx));
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cpu")
        {
            backend = VPI_BACKEND_CPU;
        }
        else if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else if (strBackend == "vic")
        {
            backend = VPI_BACKEND_VIC;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend +
                                     "' not recognized, it must be either cpu, cuda or vic.");
        }
 
        // Create the stream for the given backend.
        VPIStream streamTNR;
        CHECK_STATUS(vpiStreamCreate(backend, &streamTNR));
 
        VPIStream stream;
        // PVA backend doesn't have currently Convert Image Format algorithm. We'll use CUDA
        // backend to do that.
        CHECK_STATUS(vpiStreamCreate(VPI_BACKEND_CUDA | backend, &stream));
 
#if CV_MAJOR_VERSION >= 3
        int w      = invid.get(cv::CAP_PROP_FRAME_WIDTH);
        int h      = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
        int fourcc = invid.get(cv::CAP_PROP_FOURCC);
        double fps = invid.get(cv::CAP_PROP_FPS);
#else
        int w      = invid.get(CV_CAP_PROP_FRAME_WIDTH);
        int h      = invid.get(CV_CAP_PROP_FRAME_HEIGHT);
        int fourcc = invid.get(CV_CAP_PROP_FOURCC);
        double fps = invid.get(CV_CAP_PROP_FPS);
#endif
 
        cv::VideoWriter outVideo(strOutputVideo, fourcc, fps, cv::Size(w, h));
 
        VPIImage imgPrevious, imgCurrent, imgOutput;
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12, 0, &imgPrevious));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12, 0, &imgCurrent));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12, 0, &imgOutput));
 
        // Create a Temporal Noise Reduction payload configured to process NV12
        // frames under outdoor low light
        VPIPayload tnr;
        CHECK_STATUS(vpiCreateTemporalNoiseReduction(backend, w, h, VPI_IMAGE_FORMAT_NV12, VPI_TNR_DEFAULT,
                                                     VPI_TNR_PRESET_INDOOR_LOW_LIGHT, 1, &tnr));
 
        VPIImage frameBGR = nullptr;
 
        int curFrame = 0;
        cv::Mat cvFrame;
        while (invid.read(cvFrame))
        {
            printf("Frame: %d\n", ++curFrame);
 
            frameBGR = ToVPIImage(frameBGR, cvFrame);
 
            // First convert it to NV12
            CHECK_STATUS(
                vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, frameBGR, imgCurrent, VPI_CONVERSION_CAST, 1, 0));
 
            // Apply temporal noise reduction
            // For first frame, we have to pass nullptr as previous frame, this will reset internal
            // state.
            CHECK_STATUS(vpiSubmitTemporalNoiseReduction(stream, tnr, curFrame == 1 ? nullptr : imgPrevious, imgCurrent,
                                                         imgOutput));
 
            // Convert output back to BGR
            CHECK_STATUS(
                vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, imgOutput, frameBGR, VPI_CONVERSION_CAST, 1, 0));
            CHECK_STATUS(vpiStreamSync(stream));
 
            // Now add it to the output video stream
            VPIImageData imgdata;
            CHECK_STATUS(vpiImageLock(frameBGR, VPI_LOCK_READ, &imgdata));
            outVideo << ToCV(imgdata);
            CHECK_STATUS(vpiImageUnlock(frameBGR));
 
            // this iteration's output will be next's previous. Previous, which would be discarded, will be reused
            // to store next frame.
            std::swap(imgPrevious, imgOutput);
        };
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // Clean up
    vpiContextDestroy(ctx);
 
    return retval;
}