pvaAplNmsVpu.hpp#

Fully qualified name: public/src/primitive/pvaAplNmsVpu.hpp

File members: public/src/primitive/pvaAplNmsVpu.hpp

/*
 * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef PVA_APL_NMS_VPU_HPP
#define PVA_APL_NMS_VPU_HPP

#include <cupva_device.h>

#include <type_traits>

namespace pvaApl {

// Define a traits struct to map DataType to specific vector types
template<typename DataType>
struct NmsVectorTypes;

template<>
struct NmsVectorTypes<int32_t>
{
    using DVTYPE = dvint;
    using DVTYPEX = dvintx;
};

template<int32_t NmsSize, typename DataType>
class NmsVpu
{
private:
    static_assert((NmsSize == 3) || (NmsSize == 5), "NmsVpu supports 3x3 and 5x5 windows only.");
    static_assert(std::is_same_v<DataType, int32_t>, "NmsVpu supports int32_t only.");

    using DVTYPEX = typename NmsVectorTypes<DataType>::DVTYPEX;

    static constexpr int32_t kRadius = NmsSize >> 1;
    static constexpr int32_t kCenterBias = (kRadius * NmsSize) + kRadius;
    static constexpr int32_t kHorUnroll = (NmsSize == 3) ? 4 : 2;

    template<typename AgenType>
    auto dvload(AgenType &ag) -> decltype(auto)
    {
        return static_cast<dvintx>(dvuint_load(ag));
    }

    DVTYPEX dvzero()
    {
        DVTYPEX v;
        v.lo = replicatew(0);
        v.hi = replicatew(0);
        return v;
    }

    AgenCFG m_cfgs[5];
    int32_t m_niter;
    int32_t m_niterFirstPass;
    int32_t m_verScratchOffset;
    DataType *m_cbStart;
    int32_t m_cbLen;

    DataType *WrapToCb(DataType *p) const
    {
        if ((m_cbLen > 0) && (p >= (m_cbStart + m_cbLen)))
        {
            p -= m_cbLen;
        }
        return p;
    }

    void HorExec()
    {
        int32_t niterCeil = m_niterFirstPass;
        agen_A a_in       = init_agen_A_from_cfg(m_cfgs[0]);
        agen_C a_out      = init_agen_C_from_cfg(m_cfgs[1]);

        for (int32_t i = 0; i < niterCeil; i++) chess_prepare_for_pipelining
        chess_loop_range(12, )
        chess_unroll_loop(kHorUnroll)
        {
            DVTYPEX v0 = dvload(a_in);
            DVTYPEX v1 = dvload(a_in);
            DVTYPEX v2 = dvload(a_in);

            v1 = v1 + 1;
            v2 = v2 + 2;

            DVTYPEX vmax;
            vmax.lo = vmax3(v0.lo, v1.lo, v2.lo);
            vmax.hi = vmax3(v0.hi, v1.hi, v2.hi);
            chess_separator();

            if constexpr (NmsSize > 3)
            {
                DVTYPEX v3 = dvload(a_in);
                DVTYPEX v4 = dvload(a_in);

                v3 = v3 + 3;
                v4 = v4 + 4;

                vmax.lo = vmax3(vmax.lo, v3.lo, v4.lo);
                vmax.hi = vmax3(vmax.hi, v3.hi, v4.hi);
                chess_separator();
            }

            vstore(vmax, a_out);
        }
    }

    void VerExec()
    {
        int32_t niterCeil = m_niter;
        agen_A a_in       = init_agen_A_from_cfg(m_cfgs[2]);
        agen_C a_buf      = init_agen_C_from_cfg(m_cfgs[3]);
        agen_B a_out      = init_agen_B_from_cfg(m_cfgs[4]);

        DVTYPEX vZero = dvzero();

        for (int32_t i = 0; i < niterCeil; i++) chess_prepare_for_pipelining
        chess_loop_range(12, )
        chess_unroll_loop(2)
        {
            DVTYPEX v0 = dvload(a_buf);
            DVTYPEX v1 = dvload(a_buf);
            DVTYPEX v2 = dvload(a_buf);

            DVTYPEX vcenter = dvload(a_in);

            v1 = v1 + NmsSize;
            v2 = v2 + (2 * NmsSize);

            DVTYPEX vmax;
            vmax.lo = vmax3(v0.lo, v1.lo, v2.lo);
            vmax.hi = vmax3(v0.hi, v1.hi, v2.hi);
            chess_separator();

            if constexpr (NmsSize > 3)
            {
                DVTYPEX v3 = dvload(a_buf);
                DVTYPEX v4 = dvload(a_buf);

                v3 = v3 + (3 * NmsSize);
                v4 = v4 + (4 * NmsSize);

                vmax.lo = vmax3(vmax.lo, v3.lo, v4.lo);
                vmax.hi = vmax3(vmax.hi, v3.hi, v4.hi);
                chess_separator();
            }

            DVTYPEX vpred  = ((vcenter + kCenterBias) == vmax);
            DVTYPEX output = dvmux(vpred, vcenter, vZero);
            vstore(output, a_out);
        }
    }

public:
    static constexpr int32_t Radius = kRadius;

    static constexpr int32_t ScratchElements(int32_t width, int32_t height)
    {
        return (((width * height) + 15) / 16) * 16;
    }

    void Update(DataType *input, DataType *restrict output)
    {
        cupvaModifyAgenCfgBase(&m_cfgs[0], static_cast<void *>(input));
        cupvaModifyAgenCfgBase(&m_cfgs[2], static_cast<void *>(WrapToCb(input + m_verScratchOffset)));
        cupvaModifyAgenCfgBase(&m_cfgs[4], static_cast<void *>(output));
    }

    void Init(DataType *input, int32_t width, int32_t height, int32_t inputLinePitch, int32_t outputLinePitch,
              DataType *restrict output, DataType *cbStart, int32_t cbLen, DataType *scratch)
    {
        m_cbStart = cbStart;
        m_cbLen   = cbLen;

        int32_t const vecw = chess_elementsof(DVTYPEX);
        int32_t const halo = kRadius * 2;

        int32_t const outWidth  = width - halo;
        int32_t const outHeight = height - halo;

        int32_t const niter1 = (outWidth + vecw - 1) / vecw;
        int32_t const niter2 = outHeight;

        // The horizontal pass also has to produce the vertical halo rows, hence height rather than
        // outHeight here.
        int32_t const niterFirstPass = niter1 * height;
        int32_t const niter          = niter1 * niter2;

        int32_t const scratchLp = outWidth;

        // horizontal pass input
        agen a0 = init(static_cast<void *>(input));
        AgenWrapper inputWrapper;
        inputWrapper.size = sizeof(DataType);
        inputWrapper.n1   = static_cast<uint16_t>(NmsSize);
        inputWrapper.s1   = 1;
        inputWrapper.n2   = static_cast<uint16_t>(niter1);
        inputWrapper.s2   = vecw;
        inputWrapper.n3   = static_cast<uint16_t>(height);
        inputWrapper.s3   = inputLinePitch;
        INIT_AGEN3(a0, inputWrapper);
        a0        = update_agen_cb_size(a0, cbLen * static_cast<int32_t>(sizeof(DataType)));
        a0        = update_agen_cb_start(a0, reinterpret_cast<intptr_t>(cbStart));
        m_cfgs[0] = extract_agen_cfg(a0);

        // horizontal pass output
        agen a1 = init(static_cast<void *>(scratch));
        AgenWrapper horizOutWrapper;
        horizOutWrapper.size = sizeof(DataType);
        horizOutWrapper.n1   = static_cast<uint16_t>(niter1);
        horizOutWrapper.s1   = vecw;
        horizOutWrapper.n2   = static_cast<uint16_t>(height);
        horizOutWrapper.s2   = scratchLp;
        INIT_AGEN2(a1, horizOutWrapper);
        m_cfgs[1] = extract_agen_cfg(a1);

        // vertical pass center value, read back from the original input rather than lane-extracted
        // from the scratch row
        int32_t const verScratchOffset = (kRadius * inputLinePitch) + kRadius;
        m_verScratchOffset             = verScratchOffset;
        agen a2                        = init(static_cast<void *>(WrapToCb(input + verScratchOffset)));
        AgenWrapper inputWrapperCenter;
        inputWrapperCenter.size = sizeof(DataType);
        inputWrapperCenter.n1   = static_cast<uint16_t>(niter1);
        inputWrapperCenter.s1   = vecw;
        inputWrapperCenter.n2   = static_cast<uint16_t>(niter2);
        inputWrapperCenter.s2   = inputLinePitch;
        INIT_AGEN2(a2, inputWrapperCenter);
        // The center tap walks the same circular input buffer as the horizontal pass, so it needs the
        // same wrap.
        a2        = update_agen_cb_size(a2, cbLen * static_cast<int32_t>(sizeof(DataType)));
        a2        = update_agen_cb_start(a2, reinterpret_cast<intptr_t>(cbStart));
        m_cfgs[2] = extract_agen_cfg(a2);

        // vertical pass scratch taps
        agen a3 = init(static_cast<void *>(scratch));
        AgenWrapper verIn;
        verIn.size = sizeof(DataType);
        verIn.n1   = static_cast<uint16_t>(NmsSize);
        verIn.s1   = scratchLp;
        verIn.n2   = static_cast<uint16_t>(niter1);
        verIn.s2   = vecw;
        verIn.n3   = static_cast<uint16_t>(niter2);
        verIn.s3   = scratchLp;
        INIT_AGEN3(a3, verIn);
        m_cfgs[3] = extract_agen_cfg(a3);

        // output
        agen a4 = init(static_cast<void *>(output));
        AgenWrapper outputWrapper;
        outputWrapper.size = sizeof(DataType);
        outputWrapper.n1   = static_cast<uint16_t>(niter1);
        outputWrapper.s1   = vecw;
        outputWrapper.n2   = static_cast<uint16_t>(niter2);
        outputWrapper.s2   = outputLinePitch;
        INIT_AGEN2(a4, outputWrapper);
        m_cfgs[4] = extract_agen_cfg(a4);

        // Both loops over-run to their unroll factor. That is safe because every agen of a given
        // pass shares the same total iteration product, so they wrap back to base together and the
        // surplus iterations simply recompute iteration 0.
        m_niter          = (niter + 1) / 2 * 2;
        m_niterFirstPass = (niterFirstPass + kHorUnroll - 1) / kHorUnroll * kHorUnroll;
    }

    void Execute()
    {
        HorExec();
        VerExec();
    }
};
} // namespace pvaApl

#endif /* PVA_APL_NMS_VPU_HPP */