/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at https://www.aomedia.org/license/software-license. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at https://www.aomedia.org/license/patent-license.
 */

#include "cdef.h"
#include "common_dsp_rtcd.h"
#include "bitstream_unit.h"

static INLINE int32_t sign(int32_t i) {
    return i < 0 ? -1 : 1;
}

static INLINE int32_t constrain(int32_t diff, int32_t threshold, int32_t damping) {
    if (!threshold) {
        return 0;
    }

    const int32_t shift = AOMMAX(0, damping - get_msb(threshold));
    return sign(diff) * AOMMIN(abs(diff), AOMMAX(0, threshold - (abs(diff) >> shift)));
}

/*
This is Cdef_Directions (section 7.15.3) with 2 padding entries at the
beginning and end of the table. The cdef direction range is [0, 7] and the
first index is offset +/-2. This removes the need to constrain the first
index to the same range using e.g., & 7.
*/
DECLARE_ALIGNED(16, const int, eb_cdef_directions_padded[12][2]) = {
    /* Padding: svt_aom_eb_cdef_directions[6] */
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0},
    /* Padding: svt_aom_eb_cdef_directions[7] */
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1},

    /* Begin svt_aom_eb_cdef_directions */
    {-1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2},
    {0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2},
    {0 * CDEF_BSTRIDE + 1, 0 * CDEF_BSTRIDE + 2},
    {0 * CDEF_BSTRIDE + 1, 1 * CDEF_BSTRIDE + 2},
    {1 * CDEF_BSTRIDE + 1, 2 * CDEF_BSTRIDE + 2},
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 1},
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0},
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1},
    /* End svt_aom_eb_cdef_directions */

    /* Padding: svt_aom_eb_cdef_directions[0] */
    {-1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2},
    /* Padding: svt_aom_eb_cdef_directions[1] */
    {0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2},
};

const int (*const svt_aom_eb_cdef_directions)[2] = eb_cdef_directions_padded + 2;

/* Compute the primary filter strength for an 8x8 block based on the
directional variance difference. A high variance difference means
that we have a highly directional pattern (e.g. a high contrast
edge), so we can apply more deringing. A low variance means that we
either have a low contrast edge, or a non-directional texture, so
we want to be careful not to blur. */
static INLINE int32_t adjust_strength(int32_t strength, int32_t var) {
    const int32_t i = (var >> 6) ? AOMMIN(get_msb(var >> 6), 12) : 0;
    /* We use the variance of 8x8 blocks to adjust the strength. */
    return var ? (strength * (4 + i) + 8) >> 4 : 0;
}

void svt_aom_copy_rect8_8bit_to_16bit_c(uint16_t* dst, int32_t dstride, const uint8_t* src, int32_t sstride, int32_t v,
                                        int32_t h) {
    for (int32_t i = 0; i < v; i++) {
        for (int32_t j = 0; j < h; j++) {
            dst[i * dstride + j] = src[i * sstride + j];
        }
    }
}

/* Detect direction. 0 means 45-degree up-right, 2 is horizontal, and so on.
The search minimizes the weighted variance along all the lines in a
particular direction, i.e. the squared error between the input and a
"predicted" block where each pixel is replaced by the average along a line
in a particular direction. Since each direction have the same sum(x^2) term,
that term is never computed. See Section 2, step 2, of:
http://jmvalin.ca/notes/intra_paint.pdf */
uint8_t svt_aom_cdef_find_dir_c(const uint16_t* img, int32_t stride, int32_t* var, int32_t coeff_shift) {
    int32_t cost[8]        = {0};
    int32_t partial[8][15] = {{0}};
    int32_t best_cost      = 0;
    uint8_t i;
    uint8_t best_dir = 0;
    /* Instead of dividing by n between 2 and 8, we multiply by 3*5*7*8/n.
    The output is then 840 times larger, but we don't care for finding
    the max. */
    static const int32_t div_table[] = {0, 840, 420, 280, 210, 168, 140, 120, 105};
    for (i = 0; i < 8; i++) {
        int32_t j;
        for (j = 0; j < 8; j++) {
            int32_t x;
            /* We subtract 128 here to reduce the maximum range of the squared
            partial sums. */
            x = (img[i * stride + j] >> coeff_shift) - 128;
            partial[0][i + j] += x;
            partial[1][i + j / 2] += x;
            partial[2][i] += x;
            partial[3][3 + i - j / 2] += x;
            partial[4][7 + i - j] += x;
            partial[5][3 - i / 2 + j] += x;
            partial[6][j] += x;
            partial[7][i / 2 + j] += x;
        }
    }
    for (i = 0; i < 8; i++) {
        cost[2] += partial[2][i] * partial[2][i];
        cost[6] += partial[6][i] * partial[6][i];
    }
    cost[2] *= div_table[8];
    cost[6] *= div_table[8];
    for (i = 0; i < 7; i++) {
        cost[0] += (partial[0][i] * partial[0][i] + partial[0][14 - i] * partial[0][14 - i]) * div_table[i + 1];
        cost[4] += (partial[4][i] * partial[4][i] + partial[4][14 - i] * partial[4][14 - i]) * div_table[i + 1];
    }
    cost[0] += partial[0][7] * partial[0][7] * div_table[8];
    cost[4] += partial[4][7] * partial[4][7] * div_table[8];
    for (i = 1; i < 8; i += 2) {
        int32_t j;
        for (j = 0; j < 4 + 1; j++) {
            cost[i] += partial[i][3 + j] * partial[i][3 + j];
        }
        cost[i] *= div_table[8];
        for (j = 0; j < 4 - 1; j++) {
            cost[i] += (partial[i][j] * partial[i][j] + partial[i][10 - j] * partial[i][10 - j]) * div_table[2 * j + 2];
        }
    }
    for (i = 0; i < 8; i++) {
        if (cost[i] > best_cost) {
            best_cost = cost[i];
            best_dir  = i;
        }
    }
    /* Difference between the optimal variance and the variance along the
    orthogonal direction. Again, the sum(x^2) terms cancel out. */
    *var = best_cost - cost[(best_dir + 4) & 7];
    /* We'd normally divide by 840, but dividing by 1024 is close enough
    for what we're going to do with this. */
    *var >>= 10;
    return best_dir;
}

void svt_aom_cdef_find_dir_dual_c(const uint16_t* img1, const uint16_t* img2, int stride, int32_t* var1, int32_t* var2,
                                  int32_t coeff_shift, uint8_t* out1, uint8_t* out2) {
    *out1 = svt_aom_cdef_find_dir_c(img1, stride, var1, coeff_shift);
    *out2 = svt_aom_cdef_find_dir_c(img2, stride, var2, coeff_shift);
}

static AOM_INLINE void cdef_find_dir(uint16_t* in, CdefList* dlist, int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS],
                                     int32_t cdef_count, int32_t coeff_shift, uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS]) {
    int bi;

    // Find direction of two 8x8 blocks together.
    for (bi = 0; bi < cdef_count - 1; bi += 2) {
        const uint8_t by   = dlist[bi].by;
        const uint8_t bx   = dlist[bi].bx;
        const uint8_t by2  = dlist[bi + 1].by;
        const uint8_t bx2  = dlist[bi + 1].bx;
        const int     pos1 = 8 * by * CDEF_BSTRIDE + 8 * bx;
        const int     pos2 = 8 * by2 * CDEF_BSTRIDE + 8 * bx2;
        svt_aom_cdef_find_dir_dual(&in[pos1],
                                   &in[pos2],
                                   CDEF_BSTRIDE,
                                   &var[by][bx],
                                   &var[by2][bx2],
                                   coeff_shift,
                                   &dir[by][bx],
                                   &dir[by2][bx2]);
    }

    // Process remaining 8x8 blocks here. One 8x8 at a time.
    if (cdef_count % 2) {
        const uint8_t by = dlist[bi].by;
        const uint8_t bx = dlist[bi].bx;
        dir[by][bx]      = svt_aom_cdef_find_dir(
            &in[8 * by * CDEF_BSTRIDE + 8 * bx], CDEF_BSTRIDE, &var[by][bx], coeff_shift);
    }
}

const int32_t svt_aom_eb_cdef_pri_taps[2][2] = {{4, 2}, {3, 3}};
const int32_t svt_aom_eb_cdef_sec_taps[2][2] = {{2, 1}, {2, 1}};

/* Smooth in the direction detected. */
void svt_cdef_filter_block_c(uint8_t* dst8, uint16_t* dst16, int32_t dstride, const uint16_t* in, int32_t pri_strength,
                             int32_t sec_strength, int32_t dir, int32_t pri_damping, int32_t sec_damping, int32_t bsize,
                             int32_t coeff_shift, uint8_t subsampling_factor) {
    int32_t        i, j, k;
    const int32_t  s        = CDEF_BSTRIDE;
    const int32_t* pri_taps = svt_aom_eb_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
    const int32_t* sec_taps = svt_aom_eb_cdef_sec_taps[(pri_strength >> coeff_shift) & 1];

    for (i = 0; i < (4 << (int32_t)(bsize == BLOCK_8X8 || bsize == BLOCK_4X8)); i += subsampling_factor) {
        for (j = 0; j < (4 << (int32_t)(bsize == BLOCK_8X8 || bsize == BLOCK_8X4)); j++) {
            int16_t sum = 0;
            int16_t y;
            int16_t x   = in[i * s + j];
            int32_t max = x;
            int32_t min = x;
            for (k = 0; k < 2; k++) {
                int16_t p0 = in[i * s + j + svt_aom_eb_cdef_directions[dir][k]];
                int16_t p1 = in[i * s + j - svt_aom_eb_cdef_directions[dir][k]];
                sum += (int16_t)(pri_taps[k] * constrain(p0 - x, pri_strength, pri_damping));
                sum += (int16_t)(pri_taps[k] * constrain(p1 - x, pri_strength, pri_damping));
                if (p0 != CDEF_VERY_LARGE) {
                    max = AOMMAX(p0, max);
                }
                if (p1 != CDEF_VERY_LARGE) {
                    max = AOMMAX(p1, max);
                }
                min        = AOMMIN(p0, min);
                min        = AOMMIN(p1, min);
                int16_t s0 = in[i * s + j + svt_aom_eb_cdef_directions[(dir + 2)][k]];
                int16_t s1 = in[i * s + j - svt_aom_eb_cdef_directions[(dir + 2)][k]];
                int16_t s2 = in[i * s + j + svt_aom_eb_cdef_directions[(dir - 2)][k]];
                int16_t s3 = in[i * s + j - svt_aom_eb_cdef_directions[(dir - 2)][k]];
                if (s0 != CDEF_VERY_LARGE) {
                    max = AOMMAX(s0, max);
                }
                if (s1 != CDEF_VERY_LARGE) {
                    max = AOMMAX(s1, max);
                }
                if (s2 != CDEF_VERY_LARGE) {
                    max = AOMMAX(s2, max);
                }
                if (s3 != CDEF_VERY_LARGE) {
                    max = AOMMAX(s3, max);
                }
                min = AOMMIN(s0, min);
                min = AOMMIN(s1, min);
                min = AOMMIN(s2, min);
                min = AOMMIN(s3, min);
                sum += (int16_t)(sec_taps[k] * constrain(s0 - x, sec_strength, sec_damping));
                sum += (int16_t)(sec_taps[k] * constrain(s1 - x, sec_strength, sec_damping));
                sum += (int16_t)(sec_taps[k] * constrain(s2 - x, sec_strength, sec_damping));
                sum += (int16_t)(sec_taps[k] * constrain(s3 - x, sec_strength, sec_damping));
            }
            y = (int16_t)clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
            if (dst8) {
                dst8[i * dstride + j] = (uint8_t)y;
            } else {
                dst16[i * dstride + j] = (uint16_t)y;
            }
        }
    }
}

// C reference for the native 8-bit interior filter (no off-frame sentinel).
void svt_cdef_filter_block_8bit_c(uint8_t* dst, int32_t dstride, const uint8_t* in, int32_t pri_strength,
                                  int32_t sec_strength, int32_t dir, int32_t damping, int32_t bsize,
                                  int32_t coeff_shift, uint8_t subsampling_factor) {
    const int32_t  s        = CDEF_BSTRIDE;
    const int32_t* pri_taps = svt_aom_eb_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
    const int32_t* sec_taps = svt_aom_eb_cdef_sec_taps[(pri_strength >> coeff_shift) & 1];
    const int32_t  rows     = (bsize == BLOCK_8X8 || bsize == BLOCK_4X8) ? 8 : 4;
    const int32_t  cols     = (bsize == BLOCK_8X8 || bsize == BLOCK_8X4) ? 8 : 4;
    const int32_t  sub      = (bsize == BLOCK_4X4) ? 1 : subsampling_factor;
    for (int32_t i = 0; i < rows; i += sub) {
        for (int32_t j = 0; j < cols; j++) {
            const int16_t x   = in[i * s + j];
            int16_t       sum = 0;
            int32_t       max = x, min = x;
            for (int32_t k = 0; k < 2; k++) {
                const int16_t p0 = in[i * s + j + svt_aom_eb_cdef_directions[dir][k]];
                const int16_t p1 = in[i * s + j - svt_aom_eb_cdef_directions[dir][k]];
                sum += (int16_t)(pri_taps[k] * constrain(p0 - x, pri_strength, damping));
                sum += (int16_t)(pri_taps[k] * constrain(p1 - x, pri_strength, damping));
                max              = AOMMAX(p0, max);
                max              = AOMMAX(p1, max);
                min              = AOMMIN(p0, min);
                min              = AOMMIN(p1, min);
                const int16_t s0 = in[i * s + j + svt_aom_eb_cdef_directions[dir + 2][k]];
                const int16_t s1 = in[i * s + j - svt_aom_eb_cdef_directions[dir + 2][k]];
                const int16_t s2 = in[i * s + j + svt_aom_eb_cdef_directions[dir - 2][k]];
                const int16_t s3 = in[i * s + j - svt_aom_eb_cdef_directions[dir - 2][k]];
                sum += (int16_t)(sec_taps[k] * constrain(s0 - x, sec_strength, damping));
                sum += (int16_t)(sec_taps[k] * constrain(s1 - x, sec_strength, damping));
                sum += (int16_t)(sec_taps[k] * constrain(s2 - x, sec_strength, damping));
                sum += (int16_t)(sec_taps[k] * constrain(s3 - x, sec_strength, damping));
                max = AOMMAX(s0, max);
                max = AOMMAX(s1, max);
                max = AOMMAX(s2, max);
                max = AOMMAX(s3, max);
                min = AOMMIN(s0, min);
                min = AOMMIN(s1, min);
                min = AOMMIN(s2, min);
                min = AOMMIN(s3, min);
            }
            dst[i * dstride + j] = (uint8_t)clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
        }
    }
}

// C reference for the 8-bit find_dir (widen to 16-bit, delegate to the spec).
uint8_t svt_aom_cdef_find_dir_8bit_c(const uint8_t* img, int32_t stride, int32_t* var, int32_t coeff_shift) {
    uint16_t img16[8 * 8];
    for (int32_t i = 0; i < 8; i++) {
        for (int32_t j = 0; j < 8; j++) {
            img16[i * 8 + j] = img[i * stride + j];
        }
    }
    return svt_aom_cdef_find_dir_c(img16, 8, var, coeff_shift);
}

void svt_aom_cdef_find_dir_dual_8bit_c(const uint8_t* img1, const uint8_t* img2, int stride, int32_t* var1,
                                       int32_t* var2, int32_t coeff_shift, uint8_t* out1, uint8_t* out2) {
    *out1 = svt_aom_cdef_find_dir_8bit_c(img1, stride, var1, coeff_shift);
    *out2 = svt_aom_cdef_find_dir_8bit_c(img2, stride, var2, coeff_shift);
}

/*
 * Loop over the non-skip 8x8 blocks.  For each block, find the CDEF direction, then apply the specified filter.
*/
void svt_cdef_filter_fb(uint8_t* dst8, uint16_t* dst16, int32_t dstride, uint16_t* in, int32_t xdec, int32_t ydec,
                        uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
                        int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t pli, CdefList* dlist, int32_t cdef_count,
                        int32_t cdef_strength, int32_t damping, int32_t coeff_shift, uint8_t subsampling_factor) {
    int32_t bi;
    int32_t pri_strength = (cdef_strength / CDEF_SEC_STRENGTHS) << coeff_shift;
    int32_t sec          = cdef_strength % CDEF_SEC_STRENGTHS;
    int32_t sec_strength = (sec + (sec == 3)) << coeff_shift;
    damping += coeff_shift - (pli != PLANE_Y);

    int32_t bsize  = ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
    int32_t bsizex = 3 - xdec;
    int32_t bsizey = 3 - ydec;

    if (!dstride && cdef_strength == 0) {
        // If we're here, both primary and secondary strengths are 0, and
        // we still haven't written anything to y[] yet, so we just copy
        // the input to y[]. This is necessary only for svt_av1_cdef_search()
        // and only svt_av1_cdef_search() sets dirinit.
        for (bi = 0; bi < cdef_count; bi++) {
            int32_t   by = dlist[bi].by << bsizey;
            int32_t   bx = dlist[bi].bx << bsizex;
            int32_t   iy;
            uint16_t* src_16 = in + (by * CDEF_BSTRIDE + bx);
            if (dst8) {
                uint8_t* dst_8 = dst8 + (bi << (bsizex + bsizey));
                //size 2x2 and 3x3, no gain to use SIMD
                for (iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
                    for (int32_t ix = 0; ix < 1 << bsizex; ix++) {
                        dst_8[(iy << bsizex) + ix] = (uint8_t)src_16[iy * CDEF_BSTRIDE + ix];
                    }
                }
            } else {
                uint16_t* dst_16 = dst16 + (bi << (bsizex + bsizey));
                for (iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
                    memcpy(dst_16 + (iy << bsizex),
                           src_16 + iy * CDEF_BSTRIDE,
                           (uint32_t)(1 << bsizex) * sizeof(uint16_t));
                }
            }
        }
        return;
    }

    if (pli == 0) {
        if (!dirinit || !*dirinit) {
            cdef_find_dir(in, dlist, var, cdef_count, coeff_shift, dir);
            if (dirinit) {
                *dirinit = 1;
            }
        }
    } else if (pli == 1 && xdec != ydec) {
        for (bi = 0; bi < cdef_count; bi++) {
            static const uint8_t conv422[8] = {7, 0, 2, 4, 5, 6, 6, 6};
            static const uint8_t conv440[8] = {1, 2, 2, 2, 3, 4, 6, 0};

            int32_t by  = dlist[bi].by;
            int32_t bx  = dlist[bi].bx;
            dir[by][bx] = (xdec ? conv422 : conv440)[dir[by][bx]];
        }
    }

    for (bi = 0; bi < cdef_count; bi++) {
        int32_t by = dlist[bi].by;
        int32_t bx = dlist[bi].bx;
        int32_t t  = pli ? pri_strength : adjust_strength(pri_strength, var[by][bx]);
        int32_t k  = dstride ? (by << bsizey) * dstride + (bx << bsizex) : bi << (bsizex + bsizey);
        svt_cdef_filter_block(dst8 ? &dst8[k] : NULL,
                              dst8 ? NULL : &dst16[k],
                              dstride ? dstride : 1 << bsizex,
                              &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)],
                              t,
                              sec_strength,
                              pri_strength ? dir[by][bx] : 0,
                              damping,
                              damping,
                              bsize,
                              coeff_shift,
                              subsampling_factor);
    }
}

// Per-tap (drow,dcol) decode of eb_cdef_directions_padded, with the same +2-offset indexing as
// svt_aom_eb_cdef_directions. The boundary-aware kernels (C and NEON) use these to test, per tap,
// whether the tap lands off-frame (geometry) instead of reading an in-band 16-bit sentinel. Decoded
// directly from the flat-offset literals: each {r*CDEF_BSTRIDE + c} entry maps to {r, c}.
DECLARE_ALIGNED(16, const int8_t, eb_cdef_directions_padded_rc[12][2][2]) = {
    {{1, 0}, {2, 0}},
    {{1, 0}, {2, -1}},
    {{-1, 1}, {-2, 2}},
    {{0, 1}, {-1, 2}},
    {{0, 1}, {0, 2}},
    {{0, 1}, {1, 2}},
    {{1, 1}, {2, 2}},
    {{1, 0}, {2, 1}},
    {{1, 0}, {2, 0}},
    {{1, 0}, {2, -1}},
    {{-1, 1}, {-2, 2}},
    {{0, 1}, {-1, 2}},
};
const int8_t (*const svt_aom_eb_cdef_directions_rc)[2][2] = eb_cdef_directions_padded_rc + 2;

// True when tap (r,c), expressed in block-local coords, lands outside the frame: it crosses a
// block edge that is also a frame edge. Internal tile/SB boundaries are NOT off-frame (the
// caller only sets edge_* on true frame borders).
static INLINE int cdef_tap_off_frame(int r, int c, int rows, int cols, int edge_top, int edge_bottom, int edge_left,
                                     int edge_right) {
    return (edge_top && r < 0) || (edge_bottom && r >= rows) || (edge_left && c < 0) || (edge_right && c >= cols);
}

// Boundary-aware native 8-bit kernel: identical math to svt_cdef_filter_block_8bit_c, but excludes
// off-frame taps by geometry (the 4 per-block edge flags) instead of an in-band 16-bit sentinel.
// An off-frame tap is excluded from sum, max AND min, which exactly reproduces the 16-bit sentinel
// kernel (svt_cdef_filter_block_c): sentinel -> constrain==0 (sum), guarded for max, and large so it
// never wins min. Lets 8-bit content filter frame-perimeter ring blocks without a 16-bit buffer.
void svt_cdef_filter_block_8bit_bounded_c(uint8_t* dst, int32_t dstride, const uint8_t* in, int32_t pri_strength,
                                          int32_t sec_strength, int32_t dir, int32_t damping, int32_t bsize,
                                          int32_t coeff_shift, uint8_t subsampling_factor, int edge_top, int edge_left,
                                          int edge_bottom, int edge_right) {
    const int32_t  s        = CDEF_BSTRIDE;
    const int32_t* pri_taps = svt_aom_eb_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
    const int32_t* sec_taps = svt_aom_eb_cdef_sec_taps[(pri_strength >> coeff_shift) & 1];
    const int32_t  rows     = (bsize == BLOCK_8X8 || bsize == BLOCK_4X8) ? 8 : 4;
    const int32_t  cols     = (bsize == BLOCK_8X8 || bsize == BLOCK_8X4) ? 8 : 4;
    const int32_t  sub      = (bsize == BLOCK_4X4) ? 1 : subsampling_factor;
    for (int32_t i = 0; i < rows; i += sub) {
        for (int32_t j = 0; j < cols; j++) {
            const int16_t x   = in[i * s + j];
            int16_t       sum = 0;
            int32_t       max = x, min = x;
            for (int32_t k = 0; k < 2; k++) {
                // Primary taps p0 at (i+dr, j+dc), p1 at (i-dr, j-dc).
                const int     pdr  = svt_aom_eb_cdef_directions_rc[dir][k][0];
                const int     pdc  = svt_aom_eb_cdef_directions_rc[dir][k][1];
                const int32_t poff = svt_aom_eb_cdef_directions[dir][k];
                if (!cdef_tap_off_frame(i + pdr, j + pdc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
                    const int16_t p0 = in[i * s + j + poff];
                    sum += (int16_t)(pri_taps[k] * constrain(p0 - x, pri_strength, damping));
                    max = AOMMAX(p0, max);
                    min = AOMMIN(p0, min);
                }
                if (!cdef_tap_off_frame(i - pdr, j - pdc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
                    const int16_t p1 = in[i * s + j - poff];
                    sum += (int16_t)(pri_taps[k] * constrain(p1 - x, pri_strength, damping));
                    max = AOMMAX(p1, max);
                    min = AOMMIN(p1, min);
                }
                // Secondary taps from directions dir+2 and dir-2 (each used with +/- offset).
                const int     s0dr  = svt_aom_eb_cdef_directions_rc[dir + 2][k][0];
                const int     s0dc  = svt_aom_eb_cdef_directions_rc[dir + 2][k][1];
                const int32_t s0off = svt_aom_eb_cdef_directions[dir + 2][k];
                const int     s2dr  = svt_aom_eb_cdef_directions_rc[dir - 2][k][0];
                const int     s2dc  = svt_aom_eb_cdef_directions_rc[dir - 2][k][1];
                const int32_t s2off = svt_aom_eb_cdef_directions[dir - 2][k];
                if (!cdef_tap_off_frame(i + s0dr, j + s0dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
                    const int16_t s0 = in[i * s + j + s0off];
                    sum += (int16_t)(sec_taps[k] * constrain(s0 - x, sec_strength, damping));
                    max = AOMMAX(s0, max);
                    min = AOMMIN(s0, min);
                }
                if (!cdef_tap_off_frame(i - s0dr, j - s0dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
                    const int16_t s1 = in[i * s + j - s0off];
                    sum += (int16_t)(sec_taps[k] * constrain(s1 - x, sec_strength, damping));
                    max = AOMMAX(s1, max);
                    min = AOMMIN(s1, min);
                }
                if (!cdef_tap_off_frame(i + s2dr, j + s2dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
                    const int16_t s2 = in[i * s + j + s2off];
                    sum += (int16_t)(sec_taps[k] * constrain(s2 - x, sec_strength, damping));
                    max = AOMMAX(s2, max);
                    min = AOMMIN(s2, min);
                }
                if (!cdef_tap_off_frame(i - s2dr, j - s2dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
                    const int16_t s3 = in[i * s + j - s2off];
                    sum += (int16_t)(sec_taps[k] * constrain(s3 - x, sec_strength, damping));
                    max = AOMMAX(s3, max);
                    min = AOMMIN(s3, min);
                }
            }
            dst[i * dstride + j] = (uint8_t)clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
        }
    }
}

#if CDEF_8BITS_PATH
#include "cdef_copy.h"

static AOM_INLINE void cdef_find_dir_8bit(const uint8_t* in, CdefList* dlist, int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS],
                                          int32_t cdef_count, int32_t coeff_shift,
                                          uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS]) {
    int bi;
    for (bi = 0; bi < cdef_count - 1; bi += 2) {
        const uint8_t by   = dlist[bi].by;
        const uint8_t bx   = dlist[bi].bx;
        const uint8_t by2  = dlist[bi + 1].by;
        const uint8_t bx2  = dlist[bi + 1].bx;
        const int     pos1 = 8 * by * CDEF_BSTRIDE + 8 * bx;
        const int     pos2 = 8 * by2 * CDEF_BSTRIDE + 8 * bx2;
        svt_aom_cdef_find_dir_dual_8bit(&in[pos1],
                                        &in[pos2],
                                        CDEF_BSTRIDE,
                                        &var[by][bx],
                                        &var[by2][bx2],
                                        coeff_shift,
                                        &dir[by][bx],
                                        &dir[by2][bx2]);
    }
    if (cdef_count % 2) {
        const uint8_t by = dlist[bi].by;
        const uint8_t bx = dlist[bi].bx;
        dir[by][bx]      = svt_aom_cdef_find_dir_8bit(
            &in[8 * by * CDEF_BSTRIDE + 8 * bx], CDEF_BSTRIDE, &var[by][bx], coeff_shift);
    }
}

// in8: 8-bit padded buffer for all blocks. Interior-vs-frame-edge dispatch is decided from the
// frame_* geometry flags; frame-perimeter ring blocks use the boundary-aware kernel on in8.
// Body of svt_cdef_filter_fb_lbd, force-inlined so the per-plane call sites below specialize it with
// compile-time-constant bsize/bsizex/bsizey/is_luma. That constant block size then propagates into the
// (PGO/LTO-inlined) per-block kernels, which otherwise only see a runtime bsize.
static AOM_FORCE_INLINE void cdef_filter_fb_lbd_impl(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top,
                                                     int frame_left, int frame_bottom, int frame_right, int vsize,
                                                     int hsize, int32_t xdec, int32_t ydec,
                                                     uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
                                                     int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t pli,
                                                     CdefList* dlist, int32_t cdef_count, int32_t cdef_strength,
                                                     int32_t damping, int32_t coeff_shift, uint8_t subsampling_factor,
                                                     int32_t bsize, int32_t bsizex, int32_t bsizey, int is_luma) {
    int32_t bi;
    int32_t pri_strength = (cdef_strength / CDEF_SEC_STRENGTHS) << coeff_shift;
    int32_t sec          = cdef_strength % CDEF_SEC_STRENGTHS;
    int32_t sec_strength = (sec + (sec == 3)) << coeff_shift;
    damping += coeff_shift - (is_luma ? 0 : 1);

    // Last block row/col index within the fb (frame-edge ring detection).
    const int32_t by_last = (vsize >> bsizey) - 1;
    const int32_t bx_last = (hsize >> bsizex) - 1;

    if (!dstride && cdef_strength == 0) {
        // Zero-strength search candidate: copy input straight to packed output (4/8-byte rows).
        const int w = 1 << bsizex;
        if (w == 8) {
            for (bi = 0; bi < cdef_count; bi++) {
                const uint8_t* src_8 = in8 + ((dlist[bi].by << bsizey) * CDEF_BSTRIDE + (dlist[bi].bx << bsizex));
                uint8_t*       dst_b = dst8 + (bi << (bsizex + bsizey));
                for (int32_t iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
                    memcpy(dst_b + (iy << bsizex), src_8 + iy * CDEF_BSTRIDE, 8);
                }
            }
        } else {
            for (bi = 0; bi < cdef_count; bi++) {
                const uint8_t* src_8 = in8 + ((dlist[bi].by << bsizey) * CDEF_BSTRIDE + (dlist[bi].bx << bsizex));
                uint8_t*       dst_b = dst8 + (bi << (bsizex + bsizey));
                for (int32_t iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
                    memcpy(dst_b + (iy << bsizex), src_8 + iy * CDEF_BSTRIDE, 4);
                }
            }
        }
        return;
    }

    if (is_luma) {
        if (!dirinit || !*dirinit) {
            cdef_find_dir_8bit(in8, dlist, var, cdef_count, coeff_shift, dir);
            if (dirinit) {
                *dirinit = 1;
            }
        }
    } else if (pli == 1 && xdec != ydec) {
        for (bi = 0; bi < cdef_count; bi++) {
            static const uint8_t conv422[8] = {7, 0, 2, 4, 5, 6, 6, 6};
            static const uint8_t conv440[8] = {1, 2, 2, 2, 3, 4, 6, 0};
            int32_t              by         = dlist[bi].by;
            int32_t              bx         = dlist[bi].bx;
            dir[by][bx]                     = (xdec ? conv422 : conv440)[dir[by][bx]];
        }
    }

    // Strength 0 reaches here only as the "compute dir for later planes" pass; identity passthrough.
    if (cdef_strength == 0) {
        return;
    }

    bool edge_fb = frame_top || frame_left || frame_bottom || frame_right;
    for (bi = 0; bi < cdef_count; bi++) {
        int32_t   by          = dlist[bi].by;
        int32_t   bx          = dlist[bi].bx;
        int32_t   t           = is_luma ? adjust_strength(pri_strength, var[by][bx]) : pri_strength;
        int32_t   k           = dstride ? (by << bsizey) * dstride + (bx << bsizex) : bi << (bsizex + bsizey);
        int32_t   ds          = dstride ? dstride : 1 << bsizex;
        int32_t   off         = (by * CDEF_BSTRIDE << bsizey) + (bx << bsizex);
        int32_t   bdir        = pri_strength ? dir[by][bx] : 0;
        const int edge_top    = frame_top && by == 0;
        const int edge_bottom = frame_bottom && by == by_last;
        const int edge_left   = frame_left && bx == 0;
        const int edge_right  = frame_right && bx == bx_last;
        if (edge_fb && (edge_top || edge_bottom || edge_left || edge_right)) {
            svt_cdef_filter_block_8bit_bounded(&dst8[k],
                                               ds,
                                               &in8[off],
                                               t,
                                               sec_strength,
                                               bdir,
                                               damping,
                                               bsize,
                                               coeff_shift,
                                               subsampling_factor,
                                               edge_top,
                                               edge_left,
                                               edge_bottom,
                                               edge_right);
        } else {
            svt_cdef_filter_block_8bit(
                &dst8[k], ds, &in8[off], t, sec_strength, bdir, damping, bsize, coeff_shift, subsampling_factor);
        }
    }
}

static void cdef_filter_fb_lbd_luma(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top, int frame_left,
                                    int frame_bottom, int frame_right, int vsize, int hsize,
                                    uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
                                    int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], CdefList* dlist, int32_t cdef_count,
                                    int32_t cdef_strength, int32_t damping, int32_t coeff_shift,
                                    uint8_t subsampling_factor) {
    cdef_filter_fb_lbd_impl(dst8,
                            dstride,
                            in8,
                            frame_top,
                            frame_left,
                            frame_bottom,
                            frame_right,
                            vsize,
                            hsize,
                            0,
                            0,
                            dir,
                            dirinit,
                            var,
                            /*pli*/ 0,
                            dlist,
                            cdef_count,
                            cdef_strength,
                            damping,
                            coeff_shift,
                            subsampling_factor,
                            BLOCK_8X8,
                            3,
                            3,
                            /*is_luma*/ 1);
}

static void cdef_filter_fb_lbd_chroma(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top, int frame_left,
                                      int frame_bottom, int frame_right, int vsize, int hsize, int32_t pli,
                                      uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
                                      int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], CdefList* dlist, int32_t cdef_count,
                                      int32_t cdef_strength, int32_t damping, int32_t coeff_shift,
                                      uint8_t subsampling_factor) {
    cdef_filter_fb_lbd_impl(dst8,
                            dstride,
                            in8,
                            frame_top,
                            frame_left,
                            frame_bottom,
                            frame_right,
                            vsize,
                            hsize,
                            1,
                            1,
                            dir,
                            dirinit,
                            var,
                            pli,
                            dlist,
                            cdef_count,
                            cdef_strength,
                            damping,
                            coeff_shift,
                            subsampling_factor,
                            BLOCK_4X4,
                            2,
                            2,
                            /*is_luma*/ 0);
}

void svt_cdef_filter_fb_lbd(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top, int frame_left,
                            int frame_bottom, int frame_right, int vsize, int hsize, int32_t xdec, int32_t ydec,
                            uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
                            int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t pli, CdefList* dlist, int32_t cdef_count,
                            int32_t cdef_strength, int32_t damping, int32_t coeff_shift, uint8_t subsampling_factor) {
    if (xdec == 0 && ydec == 0) { // luma: BLOCK_8X8
        cdef_filter_fb_lbd_luma(dst8,
                                dstride,
                                in8,
                                frame_top,
                                frame_left,
                                frame_bottom,
                                frame_right,
                                vsize,
                                hsize,
                                dir,
                                dirinit,
                                var,
                                dlist,
                                cdef_count,
                                cdef_strength,
                                damping,
                                coeff_shift,
                                subsampling_factor);
    } else if (xdec == 1 && ydec == 1) { // chroma 4:2:0: BLOCK_4X4
        cdef_filter_fb_lbd_chroma(dst8,
                                  dstride,
                                  in8,
                                  frame_top,
                                  frame_left,
                                  frame_bottom,
                                  frame_right,
                                  vsize,
                                  hsize,
                                  pli,
                                  dir,
                                  dirinit,
                                  var,
                                  dlist,
                                  cdef_count,
                                  cdef_strength,
                                  damping,
                                  coeff_shift,
                                  subsampling_factor);
    } else { // 4:2:2 / 4:4:4: runtime block size
        const int32_t bsize = ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
        cdef_filter_fb_lbd_impl(dst8,
                                dstride,
                                in8,
                                frame_top,
                                frame_left,
                                frame_bottom,
                                frame_right,
                                vsize,
                                hsize,
                                xdec,
                                ydec,
                                dir,
                                dirinit,
                                var,
                                pli,
                                dlist,
                                cdef_count,
                                cdef_strength,
                                damping,
                                coeff_shift,
                                subsampling_factor,
                                bsize,
                                3 - xdec,
                                3 - ydec,
                                pli == 0);
    }
}
#endif // CDEF_8BITS_PATH
