Temporal Noise Reduction

Overview

The Temporal Noise Reduction sample application takes a noisy input video, reduces its noise and writes the result to disk. The user can define what backend will be used for processing.

Instructions

The command line parameters are:

<backend> <input video>

where

• backend: either cuda or vic (on Jetson devices only). It defines the backend that will perform the processing.
• input video: video file to have noise reduced; it accepts .mp4, .avi and possibly others, depending on OpenCV's support.

VPI samples installer includes a sample video with noise added artificially, found in /opt/nvidia/vpi2/samples/assets/noisy.mp4.

Here's one invocation example:

• C++

  ./vpi_sample_09_tnr cuda ../assets/noisy.mp4

• Python

  python3 main.py cuda ../assets/noisy.mp4

The application will process noisy.mp4 and create denoised_cuda.mp4 with the denoised version of the input.

Results

Input video	De-noised video

Source Code

For convenience, here's the code that is also installed in the samples directory.

Language: C++ Python

import cv2
import sys
import vpi
import numpy as np
from argparse import ArgumentParser
 
# ----------------------------
# Parse command line arguments
 
parser = ArgumentParser()
parser.add_argument('backend', choices=['cuda','vic'],
                    help='Backend to be used for processing')
 
parser.add_argument('input',
                    help='Input video to be denoised')
 
args = parser.parse_args();
 
if args.backend == 'cuda':
    backend = vpi.Backend.CUDA
else:
    assert args.backend == 'vic'
    backend = vpi.Backend.VIC
 
# -----------------------------
# Open input and output videos
 
inVideo = cv2.VideoCapture(args.input)
 
fourcc = cv2.VideoWriter_fourcc(*'MPEG')
inSize = (int(inVideo.get(cv2.CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = inVideo.get(cv2.CAP_PROP_FPS)
 
outVideo = cv2.VideoWriter('denoised_python'+str(sys.version_info[0])+'_'+args.backend+'.mp4',
                            fourcc, fps, inSize)
 
#--------------------------------------------------------------
# Create the TNR object using the backend specified by the user
with backend:
    tnr = vpi.TemporalNoiseReduction(inSize, vpi.Format.NV12_ER)
 
#--------------------------------------------------------------
# Main processing loop
curFrame = 0
while True:
    curFrame+=1
    print("Frame: {}".format(curFrame))
 
    # Read one input frame
    ret, cvFrame = inVideo.read()
    if not ret:
        break
 
    # Convert it to NV12_ER format to be used by VPI
    with vpi.Backend.CUDA:
        frame = vpi.asimage(cvFrame).convert(vpi.Format.NV12_ER)
 
    # Retrieve the corresponding denoised frame
    denoised = tnr(frame, preset=vpi.TNRPreset.INDOOR_MEDIUM_LIGHT, strength=1)
 
    # Convert it to RGB8 for output using the CUDA backend
    with vpi.Backend.CUDA:
        denoised = denoised.convert(vpi.Format.RGB8)
 
    # Write the denoised frame to the output video
    with denoised.rlock_cpu() as data:
        outVideo.write(data)

#include <opencv2/core/version.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/videoio.hpp>
#include <vpi/OpenCVInterop.hpp>
 
#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/TemporalNoiseReduction.h>
 
#include <algorithm>
#include <cstring> // for memset
#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
#include <vector>
 
#define CHECK_STATUS(STMT)                                    \
    do                                                        \
    {                                                         \
        VPIStatus status = (STMT);                            \
        if (status != VPI_SUCCESS)                            \
        {                                                     \
            char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH];       \
            vpiGetLastStatusMessage(buffer, sizeof(buffer));  \
            std::ostringstream ss;                            \
            ss << vpiStatusGetName(status) << ": " << buffer; \
            throw std::runtime_error(ss.str());               \
        }                                                     \
    } while (0);
 
int main(int argc, char *argv[])
{
    // OpenCV image that will be wrapped by a VPIImage.
    // Define it here so that it's destroyed *after* wrapper is destroyed
    cv::Mat cvFrame;
 
    // Declare all VPI objects we'll need here so that we
    // can destroy them at the end.
    VPIStream stream     = NULL;
    VPIImage imgPrevious = NULL, imgCurrent = NULL, imgOutput = NULL;
    VPIImage frameBGR = NULL;
    VPIPayload tnr    = NULL;
 
    // main return value
    int retval = 0;
 
    try
    {
        // =============================
        // Parse command line parameters
 
        if (argc != 3)
        {
            throw std::runtime_error(std::string("Usage: ") + argv[0] + " <vic|cuda> <input_video>");
        }
 
        std::string strBackend    = argv[1];
        std::string strInputVideo = argv[2];
 
        // Now parse the backend
        VPIBackend backend;
 
        if (strBackend == "cuda")
        {
            backend = VPI_BACKEND_CUDA;
        }
        else if (strBackend == "vic")
        {
            backend = VPI_BACKEND_VIC;
        }
        else
        {
            throw std::runtime_error("Backend '" + strBackend + "' not recognized, it must be either cuda or vic.");
        }
 
        // ===============================
        // Prepare input and output videos
 
        // Load the input video
        cv::VideoCapture invid;
        if (!invid.open(strInputVideo))
        {
            throw std::runtime_error("Can't open '" + strInputVideo + "'");
        }
 
        // Open the output video for writing using input's characteristics
        int w      = invid.get(cv::CAP_PROP_FRAME_WIDTH);
        int h      = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
        int fourcc = cv::VideoWriter::fourcc('M', 'P', 'E', 'G');
        double fps = invid.get(cv::CAP_PROP_FPS);
 
        // Create the output video
        cv::VideoWriter outVideo("denoised_" + strBackend + ".mp4", fourcc, fps, cv::Size(w, h));
        if (!outVideo.isOpened())
        {
            throw std::runtime_error("Can't create output video");
        }
 
        // =================================
        // Allocate all VPI resources needed
 
        // We'll use the backend passed to run remap algorithm and the CUDA to do image format
        // conversions, therefore we have to force enabling of CUDA backend, along with the
        // desired backend.
        CHECK_STATUS(vpiStreamCreate(VPI_BACKEND_CUDA | backend, &stream));
 
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgPrevious));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgCurrent));
        CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgOutput));
 
        // Create a Temporal Noise Reduction payload configured to process NV12_ER
        // frames under indoor medium light
        CHECK_STATUS(vpiCreateTemporalNoiseReduction(backend, w, h, VPI_IMAGE_FORMAT_NV12_ER, VPI_TNR_DEFAULT, &tnr));
 
        // ====================
        // Main processing loop
 
        int curFrame = 0;
        while (invid.read(cvFrame))
        {
            printf("Frame: %d\n", ++curFrame);
 
            // frameBGR isn't allocated yet?
            if (frameBGR == NULL)
            {
                // Create a VPIImage that wraps the frame
                CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvFrame, 0, &frameBGR));
            }
            else
            {
                // reuse existing VPIImage wrapper to wrap the new frame.
                CHECK_STATUS(vpiImageSetWrappedOpenCVMat(frameBGR, cvFrame));
            }
 
            // First convert it to NV12_ER
            CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, frameBGR, imgCurrent, NULL));
 
            // Apply temporal noise reduction
            // For first frame, we have to pass NULL as previous frame,
            // this will reset internal state.
            VPITNRParams params;
            CHECK_STATUS(vpiInitTemporalNoiseReductionParams(&params));
 
            params.preset   = VPI_TNR_PRESET_INDOOR_MEDIUM_LIGHT;
            params.strength = 1.0f;
 
            CHECK_STATUS(vpiSubmitTemporalNoiseReduction(stream, 0, tnr, curFrame == 1 ? NULL : imgPrevious, imgCurrent,
                                                         imgOutput, &params));
 
            // Convert output back to BGR
            CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, imgOutput, frameBGR, NULL));
            CHECK_STATUS(vpiStreamSync(stream));
 
            // Now add it to the output video stream
            VPIImageData imgdata;
            CHECK_STATUS(vpiImageLockData(frameBGR, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &imgdata));
 
            cv::Mat outFrame;
            CHECK_STATUS(vpiImageDataExportOpenCVMat(imgdata, &outFrame));
            outVideo << outFrame;
 
            CHECK_STATUS(vpiImageUnlock(frameBGR));
 
            // this iteration's output will be next's previous. Previous, which would be discarded, will be reused
            // to store next frame.
            std::swap(imgPrevious, imgOutput);
        };
    }
    catch (std::exception &e)
    {
        std::cerr << e.what() << std::endl;
        retval = 1;
    }
 
    // =========================
    // Destroy all VPI resources
    vpiStreamDestroy(stream);
    vpiPayloadDestroy(tnr);
    vpiImageDestroy(imgPrevious);
    vpiImageDestroy(imgCurrent);
    vpiImageDestroy(imgOutput);
    vpiImageDestroy(frameBGR);
 
    return retval;
}