GazeNet.h

Go to the documentation of this file.

/*###############################################################################
#
# Copyright(c) 2021 NVIDIA CORPORATION. All Rights Reserved.
#
# NVIDIA CORPORATION and its licensors retain all intellectual property
# and proprietary rights in and to this software, related documentation
# and any modifications thereto. Any use, reproduction, disclosure or
# distribution of this software and related documentation without an express
# license agreement from NVIDIA CORPORATION is strictly prohibited.
#
###############################################################################*/
 
#ifndef CVCORE_GAZENET_H_
#define CVCORE_GAZENET_H_
 
#include <memory>
 
#include <cuda_runtime.h>
 
#include <cv/core/Array.h>
#include <cv/core/BBox.h>
#include <cv/core/CameraModel.h>
#include <cv/core/Core.h>
#include <cv/core/Image.h>
#include <cv/core/MathTypes.h>
#include <cv/core/Model.h>
#include <cv/core/Tensor.h>
 
namespace cvcore { namespace gazenet {
 
struct GazeNetVisualizerParams
{
    std::vector<double> input3DLandmarks;
    size_t outputLength;
};
 
CVCORE_API extern const ImagePreProcessingParams defaultPreProcessorParams;
 
CVCORE_API extern const ModelInputParams defaultModelInputParams;
 
CVCORE_API extern const ModelInferenceParams defaultInferenceParams;
 
enum class FeatureType
{
    FACEGRID,
    LANDMARKS
};
 
class CVCORE_API GazeNetPreProcessor
{
public:
    static constexpr size_t NUM_LANDMARKS = 68;
 
    static constexpr float BBOX_FACE_SCALE = 1.3f;
 
    static constexpr float BBOX_EYE_SCALE = 1.8f;
 
    static const float DEFAULT_LANDMARKS_MEAN[];
 
    static const float DEFAULT_LANDMARKS_STD[];
 
    GazeNetPreProcessor() = delete;
 
    GazeNetPreProcessor(const ImagePreProcessingParams &preProcessorParams, const ModelInputParams &modelInputParams);
 
    ~GazeNetPreProcessor();
 
    void setLandmarksMeanAndVariance(const Array<Vector2f> &landmarksMean, const Array<Vector2f> &landmarksStd);
 
    void execute(Tensor<NHWC, C1, F32> &outputFace, Tensor<NHWC, C1, F32> &outputLeft,
                 Tensor<NHWC, C1, F32> &outputRight, Tensor<CL, CX, F32> &outputFeatures,
                 const Tensor<NHWC, C3, U8> &inputImage, const Array<BBox> &inputBBox,
                 const Array<ArrayN<Vector2f, GazeNetPreProcessor::NUM_LANDMARKS>> &inputLandmarks,
                 FeatureType type = FeatureType::FACEGRID, cudaStream_t stream = 0);
 
private:
    struct GazeNetPreProcessorImpl;
    std::unique_ptr<GazeNetPreProcessorImpl> m_pImpl;
};
 
class CVCORE_API GazeNet
{
public:
    static constexpr size_t OUTPUT_SIZE = 5;
 
    static constexpr size_t FACEGRID_WIDTH = 25;
 
    static constexpr size_t FACEGRID_HEIGHT = 25;
 
    static constexpr size_t LANDMARKS_EYE_LEFT_BEGIN = 42;
 
    static constexpr size_t LANDMARKS_EYE_LEFT_END = 48;
 
    static constexpr size_t LANDMARKS_EYE_RIGHT_BEGIN = 36;
 
    static constexpr size_t LANDMARKS_EYE_RIGHT_END = 42;
 
    GazeNet() = delete;
 
    GazeNet(const ImagePreProcessingParams &preProcessorParams, const ModelInputParams &modelInputParams,
            const ModelInferenceParams &inferenceParams);
 
    ~GazeNet();
 
    void setLandmarksMeanAndVariance(const Array<Vector2f> &landmarksMean, const Array<Vector2f> &landmarksStd);
 
    void execute(Array<ArrayN<float, GazeNet::OUTPUT_SIZE>> &outputGaze, const Tensor<NHWC, C3, U8> &inputImage,
                 const Array<BBox> &inputBBox,
                 const Array<ArrayN<Vector2f, GazeNetPreProcessor::NUM_LANDMARKS>> &inputLandmarks,
                 FeatureType type = FeatureType::FACEGRID, cudaStream_t stream = 0);
 
private:
    struct GazeNetImpl;
    std::unique_ptr<GazeNetImpl> m_pImpl;
};
 
class CVCORE_API GazeNetVisualizer
{
public:
    GazeNetVisualizer(const CameraIntrinsics &cameraParams);
 
    GazeNetVisualizer(const GazeNetVisualizerParams &params, const CameraIntrinsics &cameraParams);
 
    ~GazeNetVisualizer();
 
    void execute(Array<Vector2i> &outputGazeLeftEndPt, Array<Vector2i> &outputGazeLeftStartPt,
                 Array<Vector2i> &outputGazeRightEndPt, Array<Vector2i> &outputGazeRightStartPt,
                 const Array<ArrayN<Vector2f, GazeNetPreProcessor::NUM_LANDMARKS>> &input2DLandmarks,
                 const Array<ArrayN<float, GazeNet::OUTPUT_SIZE>> &inputGaze);
 
private:
    struct GazeNetVisualizerImpl;
    std::unique_ptr<GazeNetVisualizerImpl> m_pImpl;
};
 
}} // namespace cvcore::gazenet
 
#endif