Temporal Noise Reduction

Overview

The Temporal Noise Reduction sample application takes a noisy input video, reduces its noise and writes the result to disk. The user can define what backend will be used for processing.

This sample shows the following:

• Creating and destroying a VPI stream.
• Wrapping an OpenCV image to be used by VPI.
• Wrapping a VPI-managed image into an OpenCV's cv::Mat.
• Use OpenCV to fetch frames from a video file.
• Use OpenCV save frames into a video file.
• Create pipeline that does Image Format Converter to convert from/to NV12 format and run Temporal Noise Reduction.
• Shows how to properly use Temporal Noise Reduction to process a video sequence.

Instructions

The usage is:

./vpi_sample_09_tnr <backend> <input video> <output video>

where

• backend: either cuda or pva, as cpu isn't available yet. It defines the backend that will perform the processing.
• input video: video file to have noise reduced; it accepts .mp4, .avi and possibly others, depending on OpenCV's support.
• output video: file to write the de-noised result. It'll use the same codec and fps as input video.

VPI samples installer includes a sample video with noise added artificially, found in /opt/nvidia/vpi-0.3/samples/assets/noisy.mp4.

Here's one invocation example:

./vpi_sample_09_tnr cuda ../assets/noisy.mp4 denoised.mp4

Results

Input video	De-noised video

Source code

For convenience, here's the code that is also installed in the samples directory.

/*
* Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*  * Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
*  * Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*  * Neither the name of NVIDIA CORPORATION nor the names of its
*    contributors may be used to endorse or promote products derived
*    from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
 
#include <opencv2/core/version.hpp>
#if CV_MAJOR_VERSION >= 3
#    include <opencv2/imgcodecs.hpp>
#    include <opencv2/videoio.hpp>
#else
#    include <opencv2/highgui/highgui.hpp>
#endif
 
#include <opencv2/imgproc/imgproc.hpp>
 
#include <vpi/Context.h>
#include <vpi/Image.h>
#include <vpi/Stream.h>
#include <vpi/algo/ImageFormatConverter.h>
#include <vpi/algo/TemporalNoiseReduction.h>
 
#include <algorithm>
#include <cstring> // for memset
#include <fstream>
#include <iostream>
#include <map>
#include <vector>
 
#define CHECK_STATUS(STMT)                                      \
    do                                                          \
    {                                                           \
        VPIStatus status = (STMT);                              \
        if (status != VPI_SUCCESS)                              \
        {                                                       \
            throw std::runtime_error(vpiStatusGetName(status)); \
        }                                                       \
    } while (0);
 
// Utility function to wrap a cv::Mat into a VPIImage
static VPIImage ToVPIImage(const cv::Mat &frame)
{
    VPIImageData imgData;
    memset(&imgData, 0, sizeof(imgData));
 
    switch (frame.type())
    {
    case CV_8U:
        imgData.type = VPI_IMAGE_TYPE_U8;
        break;
    case CV_8UC3:
        imgData.type = VPI_IMAGE_TYPE_BGR8;
        break;
    case CV_8UC4:
        imgData.type = VPI_IMAGE_TYPE_BGRA8;
        break;
    default:
        throw std::runtime_error("Frame type not supported");
    }
 
    // First fill VPIImageData with the, well, image data...
    imgData.numPlanes           = 1;
    imgData.planes[0].width     = frame.cols;
    imgData.planes[0].height    = frame.rows;
    imgData.planes[0].rowStride = frame.step[0];
    imgData.planes[0].data      = frame.data;
 
    // Now create a VPIImage that wraps it.
    VPIImage img;
    CHECK_STATUS(vpiImageWrapHostMem(&imgData, 0, &img));
    return img;
};
 
// Utility function to wrap a VPIImageData into a cv::Mat
static cv::Mat ToCV(const VPIImageData &imgData)
{
    cv::Mat out;
 
    switch (imgData.type)
    {
    case VPI_IMAGE_TYPE_BGR8:
        out = cv::Mat(imgData.planes[0].height, imgData.planes[0].width, CV_8UC3, imgData.planes[0].data,
                      imgData.planes[0].rowStride);
        break;
 
    case VPI_IMAGE_TYPE_BGRA8:
        out = cv::Mat(imgData.planes[0].height, imgData.planes[0].width, CV_8UC4, imgData.planes[0].data,
                      imgData.planes[0].rowStride);
        break;
 
    case VPI_IMAGE_TYPE_U8:
        out = cv::Mat(imgData.planes[0].height, imgData.planes[0].width, CV_8UC1, imgData.planes[0].data,
                      imgData.planes[0].rowStride);
        break;
 
    default:
        throw std::runtime_error("Frame type not supported");
    }
 
    return out;
}
 
int main(int argc, char *argv[])
{
    // We'll create all our objects under this context, so that
    // we don't have to track what objects to destroy. Just destroying
    // the context will destroy all objects.
    VPIContext ctx = nullptr;
 
    int retval = 0;
 
    try
    {
        if (argc != 4)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|pva|cuda> <input_video> <output>");
        }
 
        std::string strDevType     = argv[1];
        std::string strInputVideo  = argv[2];
        std::string strOutputVideo = argv[3];
 
        // Load the input video
        cv::VideoCapture invid;
        if (!invid.open(strInputVideo))
        {
            throw std::runtime_error("Can't open '" + strInputVideo + "'");
        }
 
        // Create our context.
        CHECK_STATUS(vpiContextCreate(0, &ctx));
 
        // Activate it. From now on all created objects will be owned by it.
        CHECK_STATUS(vpiContextSetCurrent(ctx));
 
        // Now process the device type
        VPIDeviceType devType;
 
        if (strDevType == "cpu")
        {
            devType = VPI_DEVICE_TYPE_CPU;
        }
        else if (strDevType == "cuda")
        {
            devType = VPI_DEVICE_TYPE_CUDA;
        }
        else if (strDevType == "pva")
        {
            devType = VPI_DEVICE_TYPE_PVA;
        }
        else
        {
            throw std::runtime_error("Backend '" + strDevType +
                                     "' not recognized, it must be either cpu, cuda or pva.");
        }
 
        // Create the stream for the given backend.
        VPIStream streamTNR;
        CHECK_STATUS(vpiStreamCreate(devType, &streamTNR));
 
        VPIStream streamConv;
        if (devType == VPI_DEVICE_TYPE_PVA)
        {
            // PVA backend doesn't have currently Image Format Converter algorithm. We'll use CUDA
            // backend to do that.
            CHECK_STATUS(vpiStreamCreate(VPI_DEVICE_TYPE_CUDA, &streamConv));
        }
        else
        {
            // On other cases, we can use the same stream to do format conversion.
            streamConv = streamTNR;
        }
 
        int w = invid.get(cv::CAP_PROP_FRAME_WIDTH);
        int h = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
 
        cv::VideoWriter outVideo(strOutputVideo, invid.get(cv::CAP_PROP_FOURCC), invid.get(cv::CAP_PROP_FPS),
                                 cv::Size(w, h));
 
        VPIImage imgPrevious, imgCurrent, imgOutput;
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_TYPE_NV12, 0, &imgPrevious));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_TYPE_NV12, 0, &imgCurrent));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_TYPE_NV12, 0, &imgOutput));
 
        // Create a Temporal Noise Reduction payload configured to process NV12
        // frames under outdoor low light
        VPIPayload tnr;
        CHECK_STATUS(vpiCreateTemporalNoiseReduction(streamTNR, w, h, VPI_IMAGE_TYPE_NV12, VPI_TNR_DEFAULT,
                                                     VPI_TNR_PRESET_INDOOR_LOW_LIGHT, 1, &tnr));
 
        int curFrame = 1;
        cv::Mat cvFrame;
        while (invid.read(cvFrame))
        {
            printf("Frame: %d\n", curFrame++);
 
            VPIImage frameBGR = ToVPIImage(cvFrame);
 
            // First convert it to NV12
            CHECK_STATUS(vpiSubmitImageFormatConverter(streamConv, frameBGR, imgCurrent, VPI_CONVERSION_CAST, 1, 0));
 
            if (streamConv != streamTNR)
            {
                // Ideally we'd use VPIEvents to synchronize streamTNR and streamConv
                // (vpiEventRecord and vpiStreamWaitFor) but currently this doesn't work.
                // We have to synchronize the whole stream.
                CHECK_STATUS(vpiStreamSync(streamConv));
            }
 
            // Apply temporal noise reduction
            // For first frame, we have to pass nullptr as previous frame, this will reset internal
            // state.
            CHECK_STATUS(
                vpiSubmitTemporalNoiseReduction(tnr, curFrame == 1 ? nullptr : imgPrevious, imgCurrent, imgOutput));
            if (streamConv != streamTNR)
            {
                CHECK_STATUS(vpiStreamSync(streamTNR));
            }
 
            // Convert output back to BGR
            CHECK_STATUS(vpiSubmitImageFormatConverter(streamConv, imgOutput, frameBGR, VPI_CONVERSION_CAST, 1, 0));
            CHECK_STATUS(vpiStreamSync(streamConv));
 
            // Now add it to the output video stream
            VPIImageData imgdata;
            CHECK_STATUS(vpiImageLock(frameBGR, VPI_LOCK_READ, &imgdata));
            outVideo << ToCV(imgdata);
            CHECK_STATUS(vpiImageUnlock(frameBGR));
 
            // Not needed anymore
            vpiImageDestroy(frameBGR);
 
            // this iteration's output will be next's previous. Previous, which would be discarded, will be reused
            // to store next frame.
            std::swap(imgPrevious, imgOutput);
        };
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // Clean up
    vpiContextDestroy(ctx);
 
    return retval;
}