minih264e.h

#ifndef MINIH264_H
#define MINIH264_H
/*
    https://github.com/lieff/minih264
    To the extent possible under law, the author(s) have dedicated all copyright and related and neighboring rights to this software to the public domain worldwide.
    This software is distributed without any warranty.
    See <http://creativecommons.org/publicdomain/zero/1.0/>.
*/

#ifdef __cplusplus
extern "C" {
#endif

#ifndef H264E_SVC_API
#   define H264E_SVC_API 1
#endif

#ifndef H264E_MAX_THREADS
#   define H264E_MAX_THREADS 4
#endif

/**
*   API return error codes
*/
#define H264E_STATUS_SUCCESS                0
#define H264E_STATUS_BAD_ARGUMENT           1
#define H264E_STATUS_BAD_PARAMETER          2
#define H264E_STATUS_BAD_FRAME_TYPE         3
#define H264E_STATUS_SIZE_NOT_MULTIPLE_16   4
#define H264E_STATUS_SIZE_NOT_MULTIPLE_2    5
#define H264E_STATUS_BAD_LUMA_ALIGN         6
#define H264E_STATUS_BAD_LUMA_STRIDE        7
#define H264E_STATUS_BAD_CHROMA_ALIGN       8
#define H264E_STATUS_BAD_CHROMA_STRIDE      9

/**
*   Frame type definitions
*   - Sequence must start with key (IDR) frame.
*   - P (Predicted) frames are most efficiently coded
*   - Dropable frames may be safely removed from bitstream, and used
*     for frame rate scalability
*   - Golden and Recovery frames used for error recovery. These
*     frames uses "long-term reference" for prediction, and
*     can be decoded if P frames sequence is interrupted.
*     They acts similarly to key frame, but coded more efficiently.
*
*   Type        Refers to   Saved as long-term  Saved as short-term
*   ---------------------------------------------------------------
*   Key (IDR) : N/A         Yes                 Yes                |
*   Golden    : long-term   Yes                 Yes                |
*   Recovery  : long-term   No                  Yes                |
*   P         : short-term  No                  Yes                |
*   Droppable : short-term  No                  No                 |
*                                                                  |
*   Example sequence:        K   P   P   G   D   P   R   D   K     |
*   long-term reference       1K  1K  1K  4G  4G  4G  4G  4G  9K   |
*                             /         \ /         \         /    |
*   coded frame             1K  2P  3P  4G  5D  6P  7R  8D  9K     |
*                             \ / \ / \   \ /   / \   \ /     \    |
*   short-term reference      1K  2P  3P  4G  4G  6P  7R  7R  9K   |
*
*/
#define H264E_FRAME_TYPE_DEFAULT    0       // Frame type set according to GOP size
#define H264E_FRAME_TYPE_KEY        6       // Random access point: SPS+PPS+Intra frame
#define H264E_FRAME_TYPE_I          5       // Intra frame: updates long & short-term reference
#define H264E_FRAME_TYPE_GOLDEN     4       // Use and update long-term reference
#define H264E_FRAME_TYPE_RECOVERY   3       // Use long-term reference, updates short-term reference
#define H264E_FRAME_TYPE_P          2       // Use and update short-term reference
#define H264E_FRAME_TYPE_DROPPABLE  1       // Use short-term reference, don't update anything
#define H264E_FRAME_TYPE_CUSTOM     99      // Application specifies reference frame

/**
*   Speed preset index.
*   Currently used values are 0, 1, 8 and 9
*/
#define H264E_SPEED_SLOWEST         0       // All coding tools enabled, including denoise filter
#define H264E_SPEED_BALANCED        5
#define H264E_SPEED_FASTEST         10      // Minimum tools enabled

/**
*   Creations parameters
*/
typedef struct H264E_create_param_tag
{
    // Frame width: must be multiple of 16
    int width;

    // Frame height: must be multiple of 16
    int height;

    // GOP size == key frame period
    // If 0: no key frames generated except 1st frame (infinite GOP)
    // If 1: Only intra-frames produced
    int gop;

    // Video Buffer Verifier size, bits
    // If 0: VBV model would be disabled
    // Note, that this value defines Level,
    int vbv_size_bytes;

    // If set: transparent frames produced on VBV overflow
    // If not set: VBV overflow ignored, produce bitrate bigger than specified
    int vbv_overflow_empty_frame_flag;

    // If set: keep minimum bitrate using stuffing, prevent VBV underflow
    // If not set: ignore VBV underflow, produce bitrate smaller than specified
    int vbv_underflow_stuffing_flag;

    // If set: control bitrate at macroblock-level (better bitrate precision)
    // If not set: control bitrate at frame-level (better quality)
    int fine_rate_control_flag;

    // If set: don't change input, but allocate additional frame buffer
    // If not set: use input as a scratch
    int const_input_flag;

    // If 0: golden, recovery, and custom frames are disabled
    // If >0: Specifies number of persistent frame buffer's used
    int max_long_term_reference_frames;

    int enableNEON;

    // If set: enable temporal noise suppression
    int temporal_denoise_flag;

    int sps_id;

#if H264E_SVC_API
    //          SVC extension
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    // Number of SVC layers:
    // 1 = AVC
    // 2 = SVC with 2-layers of spatial scalability
    int num_layers;

    // If set, SVC extension layer will use predictors from base layer
    // (sometimes can slightly increase efficiency)
    int inter_layer_pred_flag;
#endif

#if H264E_MAX_THREADS
    //           Multi-thread extension
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    // Maximum threads, supported by the callback
    int max_threads;

    // Opaque token, passed to callback
    void *token;

    // Application-supplied callback function.
    // This callback runs given jobs, by calling provided job_func(), passing
    // job_data[i] to each one.
    //
    // The h264e_thread_pool_run() can be used here, example:
    //
    //      int max_threads = 4;
    //      void *thread_pool = h264e_thread_pool_init(max_threads);
    //
    //      H264E_create_param_t par;
    //      par.max_threads = max_threads;
    //      par.token = thread_pool;
    //      par.run_func_in_thread = h264e_thread_pool_run;
    //
    // The reason to use double callbacks is to avoid mixing portable and
    // system-dependent code, and to avoid close() function in the encoder API.
    //
    void (*run_func_in_thread)(void *token, void (*job_func)(void*), void *job_data[], int njobs);
#endif

} H264E_create_param_t;

/**
*   Run-time parameters
*/
typedef struct H264E_run_param_tag
{
    // Variable, indicating speed/quality tradeoff
    // 0 means best quality
    int encode_speed;

    // Frame type override: one of H264E_FRAME_TYPE_* values
    // if 0: GOP pattern defined by create_param::gop value
    int frame_type;

    // Used only if frame_type == H264E_FRAME_TYPE_CUSTOM
    // Reference long-term frame index [1..max_long_term_reference_frames]
    // 0 = use previous frame (short-term)
    // -1 = IDR frame, kill all long-term frames
    int long_term_idx_use;

    // Used only if frame_type == H264E_FRAME_TYPE_CUSTOM
    // Store decoded frame in long-term buffer with given index in the
    // range [1..max_long_term_reference_frames]
    // 0 = save to short-term buffer
    // -1 = Don't save frame (dropable)
    int long_term_idx_update;

    // Target frame size. Typically = bitrate/framerate
    int desired_frame_bytes;

    // Minimum quantizer value, 10 indicates good quality
    // range: [10; qp_max]
    int qp_min;

    // Maximum quantizer value, 51 indicates very bad quality
    // range: [qp_min; 51]
    int qp_max;

    // Desired NALU size. NALU produced as soon as it's size exceed this value
    // if 0: frame would be coded with a single NALU
    int desired_nalu_bytes;

    // Optional NALU notification callback, called by the encoder
    // as soon as NALU encoding complete.
    void (*nalu_callback)(
        const unsigned char *nalu_data, // Coded NALU data, w/o start code
        int sizeof_nalu_data,           // Size of NALU data
        void *token                     // optional transparent token
        );

    // token to pass to NALU callback
    void *nalu_callback_token;

} H264E_run_param_t;

/**
*    Planar YUV420 descriptor
*/
typedef struct H264E_io_yuv_tag
{
    // Pointers to 3 pixel planes of YUV image
    unsigned char *yuv[3];
    // Stride for each image plane
    int stride[3];
} H264E_io_yuv_t;

typedef struct H264E_persist_tag H264E_persist_t;
typedef struct H264E_scratch_tag H264E_scratch_t;

/**
*   Return persistent and scratch memory requirements
*   for given encoding options.
*
*   Return value:
*       -zero in case of success
*       -error code (H264E_STATUS_*), if fails
*
*   example:
*
*   int sizeof_persist, sizeof_scratch, error;
*   H264E_persist_t * enc;
*   H264E_scratch_t * scratch;
*
*   error = H264E_sizeof(param, &sizeof_persist, &sizeof_scratch);
*   if (!error)
*   {
*       enc     = malloc(sizeof_persist);
*       scratch = malloc(sizeof_scratch);
*       error = H264E_init(enc, param);
*   }
*/
int H264E_sizeof(
    const H264E_create_param_t *param,  ///< Encoder creation parameters
    int *sizeof_persist,                ///< [OUT] Size of persistent RAM
    int *sizeof_scratch                 ///< [OUT] Size of scratch RAM
);

/**
*   Initialize encoding session
*
*   Return value:
*       -zero in case of success
*       -error code (H264E_STATUS_*), if fails
*/
int H264E_init(
    H264E_persist_t *enc,               ///< Encoder object
    const H264E_create_param_t *param   ///< Encoder creation parameters
);

/**
*   Encode single video frame
*
*   Output buffer is in the scratch RAM
*
*   Return value:
*       -zero in case of success
*       -error code (H264E_STATUS_*), if fails
*/
int H264E_encode(
    H264E_persist_t *enc,               ///< Encoder object
    H264E_scratch_t *scratch,           ///< Scratch memory
    const H264E_run_param_t *run_param, ///< run-time parameters
    H264E_io_yuv_t *frame,              ///< Input video frame
    unsigned char **coded_data,         ///< [OUT] Pointer to coded data
    int *sizeof_coded_data              ///< [OUT] Size of coded data
);

/**
*   This is a "hack" function to set internal rate-control state
*   Note that encoder allows application to completely override rate-control,
*   so this function should be used only by lazy coders, who just want to change
*   VBV size, without implementing custom rate-control.
*
*   Note that H.264 level defined by VBV size on initialization.
*/
void H264E_set_vbv_state(
    H264E_persist_t *enc,               ///< Encoder object
    int vbv_size_bytes,                 ///< New VBV size
    int vbv_fullness_bytes              ///< New VBV fulness, -1 = no change
);

#ifdef __cplusplus
}
#endif

#endif //MINIH264_H

#if defined(MINIH264_IMPLEMENTATION) && !defined(MINIH264_IMPLEMENTATION_GUARD)
#define MINIH264_IMPLEMENTATION_GUARD

#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

/************************************************************************/
/*      Build configuration                                             */
/************************************************************************/
#ifndef H264E_ENABLE_DENOISE
#define H264E_ENABLE_DENOISE 1 // Build-in noise supressor
#endif

#ifndef MAX_LONG_TERM_FRAMES
#define MAX_LONG_TERM_FRAMES 8 // Max long-term frames count
#endif

#if !defined(MINIH264_ONLY_SIMD) && (defined(_M_X64) || defined(_M_ARM64) || defined(__x86_64__) || defined(__aarch64__))
/* x64 always have SSE2, arm64 always have neon, no need for generic code */
#define MINIH264_ONLY_SIMD
#endif /* SIMD checks... */

#if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))) || ((defined(__i386__) || defined(__x86_64__)) && defined(__SSE2__))
#define H264E_ENABLE_SSE2 1
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <emmintrin.h>
#endif
#elif defined(__ARM_NEON) || defined(__aarch64__)
#define H264E_ENABLE_NEON 1
#include <arm_neon.h>
#else
#ifdef MINIH264_ONLY_SIMD
#error MINIH264_ONLY_SIMD used, but SSE/NEON not enabled
#endif
#endif

#ifndef MINIH264_ONLY_SIMD
#define H264E_ENABLE_PLAIN_C 1
#endif

#define H264E_CONFIGS_COUNT ((H264E_ENABLE_SSE2) + (H264E_ENABLE_PLAIN_C) + (H264E_ENABLE_NEON))

#if defined(__ARMCC_VERSION) || defined(_WIN32) || defined(__EMSCRIPTEN__)
#define __BYTE_ORDER 0
#define __BIG_ENDIAN 1
#elif defined(__linux__) || defined(__CYGWIN__)
#include <endian.h>
#elif defined(__APPLE__)
#include <libkern/OSByteOrder.h>
#define __BYTE_ORDER BYTE_ORDER
#define __BIG_ENDIAN BIG_ENDIAN
#elif defined(__OpenBSD__) || defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
#include <sys/endian.h>
#else
#error platform not supported
#endif

#if defined(__aarch64__) && defined(__clang__)
// uintptr_t broken with aarch64 clang on ubuntu 18
#define uintptr_t unsigned long
#endif
#if defined(__arm__) && defined(__clang__)
#include <arm_acle.h>
#elif defined(__arm__) && defined(__GNUC__) && !defined(__ARMCC_VERSION)
static inline unsigned int __usad8(unsigned int val1, unsigned int val2)
{
    unsigned int result;
    __asm__ volatile ("usad8 %0, %1, %2\n\t"
                      : "=r" (result)
                      : "r" (val1), "r" (val2));
    return result;
}

static inline unsigned int __usada8(unsigned int val1, unsigned int val2, unsigned int val3)
{
    unsigned int result;
    __asm__ volatile ("usada8 %0, %1, %2, %3\n\t"
                      : "=r" (result)
                      : "r" (val1), "r" (val2), "r" (val3));
    return result;
}

static inline unsigned int __sadd16(unsigned int val1, unsigned int val2)
{
    unsigned int result;
    __asm__ volatile ("sadd16 %0, %1, %2\n\t"
                      : "=r" (result)
                      : "r" (val1), "r" (val2));
    return result;
}

static inline unsigned int __ssub16(unsigned int val1, unsigned int val2)
{
    unsigned int result;
    __asm__ volatile ("ssub16 %0, %1, %2\n\t"
                      : "=r" (result)
                      : "r" (val1), "r" (val2));
    return result;
}

static inline unsigned int __clz(unsigned int val1)
{
    unsigned int result;
    __asm__ volatile ("clz %0, %1\n\t"
                      : "=r" (result)
                      : "r" (val1));
    return result;
}
#endif

#ifdef __cplusplus
extern "C" {
#endif  //__cplusplus

#if defined(_MSC_VER) && _MSC_VER >= 1400
#   define h264e_restrict __restrict
#elif defined(__arm__)
#   define h264e_restrict __restrict
#else
#   define h264e_restrict
#endif
#if defined(_MSC_VER)
#   define ALIGN(n) __declspec(align(n))
#   define ALIGN2(n)
#else
#   define ALIGN(n)
#   define ALIGN2(n) __attribute__((aligned(n)))
#endif

#if __GNUC__ || __clang__
typedef int int_u __attribute__ ((__aligned__ (1)));
#else
typedef int int_u;
#endif

#ifndef MAX
#   define MAX(x, y) ((x) > (y) ? (x) : (y))
#endif

#ifndef MIN
#   define MIN(x, y) ((x) < (y) ? (x) : (y))
#endif

#ifndef ABS
#   define ABS(x)    ((x) >= 0 ? (x) : -(x))
#endif

#define IS_ALIGNED(p, n) (!((uintptr_t)(p) & (uintptr_t)((n) - 1)))

// bit-stream
#if __BYTE_ORDER == __BIG_ENDIAN
#   define SWAP32(x) (uint32_t)(x)
#else
#ifdef _MSC_VER
#   define SWAP32(x) _byteswap_ulong(x)
#elif defined(__GNUC__) || defined(__clang__)
#   define SWAP32(x) __builtin_bswap32(x)
#else
#   define SWAP32(x) (uint32_t)((((x) >> 24) & 0xFF) | (((x) >> 8) & 0xFF00) | (((x) << 8) & 0xFF0000) | ((x & 0xFF) << 24))
#endif
#endif

#define BS_OPEN(bs) uint32_t cache = bs->cache; int shift = bs->shift; uint32_t *buf = bs->buf;
#define BS_CLOSE(bs) bs->cache = cache; bs->shift = shift; bs->buf = buf;
#define BS_PUT(n, val)      \
if ((shift -= n) < 0)       \
{                           \
    cache |= val >> -shift; \
    *buf++ = SWAP32(cache); \
    shift += 32;            \
    cache = 0;              \
}                           \
cache |= (uint32_t)val << shift;

// Quantizer-dequantizer modes
#define QDQ_MODE_INTRA_4   2       // intra 4x4
#define QDQ_MODE_INTER     8       // inter
#define QDQ_MODE_INTRA_16  (8 + 1) // intra 16x61
#define QDQ_MODE_CHROMA    (4 + 1) // chroma

// put most frequently used bits to lsb, to use these as look-up tables
#define AVAIL_TR    8
#define AVAIL_TL    4
#define AVAIL_L     2
#define AVAIL_T     1

typedef uint8_t     pix_t;
typedef uint32_t    bs_item_t;

/**
*   Output bitstream
*/
typedef struct
{
    int         shift;  // bit position in the cache
    uint32_t    cache;  // bit cache
    bs_item_t    *buf;  // current position
    bs_item_t  *origin; // initial position
} bs_t;

/**
*   Tuple for motion vector, or height/width representation
*/
typedef union
{
    struct
    {
        int16_t x;      // horizontal or width
        int16_t y;      // vertical or height
    } s;
    int32_t u32;        // packed representation
} point_t;

/**
*   Rectangle
*/
typedef struct
{
    point_t tl;         // top-left corner
    point_t br;         // bottom-right corner
} rectangle_t;

/**
*   Quantized/dequantized representation for 4x4 block
*/
typedef struct
{
    int16_t qv[16];     // quantized coefficient
    int16_t dq[16];     // dequantized
} quant_t;

/**
*   Scratch RAM, used only for current MB encoding
*/
typedef struct H264E_scratch_tag
{
    pix_t mb_pix_inp[256];          // Input MB (cached)
    pix_t mb_pix_store[4*256];      // Prediction variants

    // Quantized/dequantized
    int16_t dcy[16];                // Y DC
    quant_t qy[16];                 // Y 16x4x4 blocks

    int16_t dcu[16];                // U DC: 4 used + align
    quant_t qu[4];                  // U 4x4x4 blocks

    int16_t dcv[16];                // V DC: 4 used + align
    quant_t qv[4];                  // V 4x4x4 blocks

    // Quantized DC:
    int16_t quant_dc[16];           // Y
    int16_t quant_dc_u[4];          // U
    int16_t quant_dc_v[4];          // V

    uint16_t nz_mask;               // Bit flags for non-zero 4x4 blocks
} scratch_t;

/**
*   Deblock filter frame context
*/
typedef struct
{
    // Motion vectors for 4x4 MB internal sub-blocks, top and left border,
    // 5x5 array without top-left cell:
    //     T0 T1 T2 T4
    //  L0 i0 i1 i2 i3
    //  L1 ...
    //  ......
    //
    point_t df_mv[5*5 - 1];         // MV for current macroblock and neighbors
    uint8_t *df_qp;                 // QP for current row of macroblocks
    int8_t *mb_type;                // Macroblock type for current row of macroblocks
    uint32_t nzflag;                // Bit flags for non-zero 4x4 blocks (left neighbors)

    // Huffman and deblock uses different nnz...
    uint8_t *df_nzflag;             // Bit flags for non-zero 4x4 blocks (top neighbors), only 4 bits used
} deblock_filter_t;

/**
*    Deblock filter parameters for current MB
*/
typedef struct
{
    uint32_t strength32[4*2];       // Strength for 4 colums and 4 rows
    uint8_t tc0[16*2];              // TC0 parameter for 4 colums and 4 rows
    uint8_t alpha[2*2];             // alpha for border/internals
    uint8_t beta[2*2];              // beta for border/internals
} deblock_params_t;

/**
*   Persistent RAM
*/
typedef struct H264E_persist_tag
{
    H264E_create_param_t param;     // Copy of create parameters
    H264E_io_yuv_t inp;             // Input picture

    struct
    {
        int pic_init_qp;            // Initial QP
    } sps;

    struct
    {
        int num;                    // Frame number
        int nmbx;                   // Frame width, macroblocks
        int nmby;                   // Frame height, macroblocks
        int nmb;                    // Number of macroblocks in frame
        int w;                      // Frame width, pixels
        int h;                      // Frame height, pixels
        rectangle_t mv_limit;       // Frame MV limits = frame + border extension
        rectangle_t mv_qpel_limit;  // Reduced MV limits for qpel interpolation filter
        int cropping_flag;          // Cropping indicator
    } frame;

    struct
    {
        int type;                   // Current slice type (I/P)
        int start_mb_num;           // # of 1st MB in the current slice
    } slice;

    struct
    {
        int x;                      // MB x position (in MB's)
        int y;                      // MB y position (in MB's)
        int num;                    // MB number
        int skip_run;               // Skip run count

        // according to table 7-13
        // -1 = skip, 0 = P16x16, 1 = P16x8, 2=P8x16, 3 = P8x8, 5 = I4x4, >=6 = I16x16
        int type;                   // MB type

        struct
        {
            int pred_mode_luma;     // Intra 16x16 prediction mode
        } i16;

        int8_t i4x4_mode[16];       // Intra 4x4 prediction modes

        int cost;                   // Best coding cost
        int avail;                  // Neighbor availability flags
        point_t mvd[16];            // Delta-MV for each 4x4 sub-part
        point_t mv[16];             // MV for each 4x4 sub-part

        point_t mv_skip_pred;       // Skip MV predictor
    } mb;

    H264E_io_yuv_t ref;             // Current reference picture
    H264E_io_yuv_t dec;             // Reconstructed current macroblock
#if H264E_ENABLE_DENOISE
    H264E_io_yuv_t denoise;         // Noise suppression filter
#endif

    unsigned char *lt_yuv[MAX_LONG_TERM_FRAMES][3]; // Long-term reference pictures
    unsigned char lt_used[MAX_LONG_TERM_FRAMES];    // Long-term "used" flags

    struct
    {
        int qp;                     // Current QP
        int vbv_bits;               // Current VBV fullness, bits
        int qp_smooth;              // Averaged QP
        int dqp_smooth;             // Adaptive QP adjustment, account for "compressibility"
        int max_dqp;                // Worst-case DQP, for long-term reference QP adjustment

        int bit_budget;             // Frame bit budget
        int prev_qp;                // Previous MB QP
        int prev_err;               // Accumulated coded size error
        int stable_count;           // Stable/not stable state machine

        int vbv_target_level;       // Desired VBV fullness after frame encode

        // Quantizer data, passed to low-level functions
        // layout:
        // multiplier_quant0, multiplier_dequant0,
        // multiplier_quant2, multiplier_dequant2,
        // multiplier_quant1, multiplier_dequant1,
        // rounding_factor_pos,
        // zero_thr_inter
        // zero_thr_inter2
        // ... and same data for chroma
        //uint16_t qdat[2][(6 + 4)];
#define OFFS_RND_INTER 6
#define OFFS_RND_INTRA 7
#define OFFS_THR_INTER 8
#define OFFS_THR2_INTER 9
#define OFFS_THR_1_OFF 10
#define OFFS_THR_2_OFF 18
#define OFFS_QUANT_VECT 26
#define OFFS_DEQUANT_VECT 34
        //struct
        //{
        //    uint16_t qdq[6]; // Interleaved Storage Quantization and Dequantization MF
        //    uint16_t rnd[2]; // inter/intra, Offset f, improves the visual quality of the reconstructed image
        //    uint16_t thr[2]; // thresholds
        //    uint16_t zero_thr[2][8];
        //    uint16_t qfull[8];
        //    uint16_t dqfull[8];
        //} qdat[2];
        // qdat[0] - luma, qdat[1] - chroma, including quant/dequant parameters
        uint16_t qdat[2][6 + 2 + 2 + 8 + 8 + 8 + 8];
    } rc;

    deblock_filter_t df;            // Deblock filter

    // Speed/quality trade-off
    struct
    {
        int disable_deblock;        // Disable deblock filter flags
    } speed;

    int most_recent_ref_frame_idx;  // Last updated long-term reference

    // predictors contexts
    point_t *mv_pred;               // MV for left&top 4x4 blocks
    uint8_t *nnz;                   // Number of non-zero coeffs per 4x4 block for left&top, the size is enc->mb.nbx*8 + 8
    int32_t *i4x4mode;              // Intra 4x4 mode for left&top, the size is enc->mb.nbx*4 + 4
    pix_t *top_line;                // left&top neighbor pixels, the size is enc->mb.nbx*32 + 32 + 16

    // output data
    uint8_t *out;                   // Output data storage (pointer to scratch RAM!)
    unsigned int out_pos;           // Output byte position, reset to 0 after each frame
    bs_t bs[1];                     // Output bitbuffer

    scratch_t *scratch;             // Pointer to scratch RAM
#if H264E_MAX_THREADS > 1
    scratch_t *scratch_store[H264E_MAX_THREADS];   // Pointer to scratch RAM
    int sizeof_scaratch;
#endif
    H264E_run_param_t run_param;    // Copy of run-time parameters

    // Consecutive IDR's must have different idr_pic_id,
    // unless there are some P between them
    uint8_t next_idr_pic_id;

    pix_t *pbest;                   // Macroblock best predictor
    pix_t *ptest;                   // Macroblock predictor under test

    point_t mv_clusters[2];         // MV clusterization for prediction

    // Flag to track short-term reference buffer, for MMCO 1 command
    int short_term_used;

#if H264E_SVC_API
    //svc ext
    int   current_layer;
    int   adaptive_base_mode_flag;
    void *enc_next;
#endif

} h264e_enc_t;

#ifdef __cplusplus
}
#endif //__cplusplus
/************************************************************************/
/*      Constants                                                       */
/************************************************************************/

// Tunable constants can be adjusted by the "training" application
#ifndef ADJUSTABLE
#   define ADJUSTABLE static const
#endif

// Huffman encode tables
#define CODE8(val, len) (uint8_t)((val << 4) + len)
#define CODE(val, len) (uint8_t)((val << 4) + (len - 1))

const uint8_t h264e_g_run_before[57] =
{
    15, 17, 20, 24, 29, 35, 42, 42, 42, 42, 42, 42, 42, 42, 42,
    /**** Table #  0 size  2 ****/
    CODE8(1, 1), CODE8(0, 1),
    /**** Table #  1 size  3 ****/
    CODE8(1, 1), CODE8(1, 2), CODE8(0, 2),
    /**** Table #  2 size  4 ****/
    CODE8(3, 2), CODE8(2, 2), CODE8(1, 2), CODE8(0, 2),
    /**** Table #  3 size  5 ****/
    CODE8(3, 2), CODE8(2, 2), CODE8(1, 2), CODE8(1, 3), CODE8(0, 3),
    /**** Table #  4 size  6 ****/
    CODE8(3, 2), CODE8(2, 2), CODE8(3, 3), CODE8(2, 3), CODE8(1, 3), CODE8(0, 3),
    /**** Table #  5 size  7 ****/
    CODE8(3, 2), CODE8(0, 3), CODE8(1, 3), CODE8(3, 3), CODE8(2, 3), CODE8(5, 3), CODE8(4, 3),
    /**** Table #  6 size 15 ****/
    CODE8(7, 3), CODE8(6, 3), CODE8(5, 3), CODE8(4, 3), CODE8(3, 3), CODE8(2,  3), CODE8(1,  3), CODE8(1, 4),
    CODE8(1, 5), CODE8(1, 6), CODE8(1, 7), CODE8(1, 8), CODE8(1, 9), CODE8(1, 10), CODE8(1, 11),
};

const uint8_t h264e_g_total_zeros_cr_2x2[12] =
{
    3, 7, 10,
    /**** Table #  0 size  4 ****/
    CODE8(1, 1), CODE8(1, 2), CODE8(1, 3), CODE8(0, 3),
    /**** Table #  1 size  3 ****/
    CODE8(1, 1), CODE8(1, 2), CODE8(0, 2),
    /**** Table #  2 size  2 ****/
    CODE8(1, 1), CODE8(0, 1),
};

const uint8_t h264e_g_total_zeros[150] =
{
    15, 31, 46, 60, 73, 85, 96, 106, 115, 123, 130, 136, 141, 145, 148,
    /**** Table #  0 size 16 ****/
    CODE8(1, 1), CODE8(3, 3), CODE8(2, 3), CODE8(3, 4), CODE8(2, 4), CODE8(3, 5), CODE8(2, 5), CODE8(3, 6),
    CODE8(2, 6), CODE8(3, 7), CODE8(2, 7), CODE8(3, 8), CODE8(2, 8), CODE8(3, 9), CODE8(2, 9), CODE8(1, 9),
    /**** Table #  1 size 15 ****/
    CODE8(7, 3), CODE8(6, 3), CODE8(5, 3), CODE8(4, 3), CODE8(3, 3), CODE8(5, 4), CODE8(4, 4), CODE8(3, 4),
    CODE8(2, 4), CODE8(3, 5), CODE8(2, 5), CODE8(3, 6), CODE8(2, 6), CODE8(1, 6), CODE8(0, 6),
    /**** Table #  2 size 14 ****/
    CODE8(5, 4), CODE8(7, 3), CODE8(6, 3), CODE8(5, 3), CODE8(4, 4), CODE8(3, 4), CODE8(4, 3), CODE8(3, 3),
    CODE8(2, 4), CODE8(3, 5), CODE8(2, 5), CODE8(1, 6), CODE8(1, 5), CODE8(0, 6),
    /**** Table #  3 size 13 ****/
    CODE8(3, 5), CODE8(7, 3), CODE8(5, 4), CODE8(4, 4), CODE8(6, 3), CODE8(5, 3), CODE8(4, 3), CODE8(3, 4),
    CODE8(3, 3), CODE8(2, 4), CODE8(2, 5), CODE8(1, 5), CODE8(0, 5),
    /**** Table #  4 size 12 ****/
    CODE8(5, 4), CODE8(4, 4), CODE8(3, 4), CODE8(7, 3), CODE8(6, 3), CODE8(5, 3), CODE8(4, 3), CODE8(3, 3),
    CODE8(2, 4), CODE8(1, 5), CODE8(1, 4), CODE8(0, 5),
    /**** Table #  5 size 11 ****/
    CODE8(1, 6), CODE8(1, 5), CODE8(7, 3), CODE8(6, 3), CODE8(5, 3), CODE8(4, 3), CODE8(3, 3), CODE8(2, 3),
    CODE8(1, 4), CODE8(1, 3), CODE8(0, 6),
    /**** Table #  6 size 10 ****/
    CODE8(1, 6), CODE8(1, 5), CODE8(5, 3), CODE8(4, 3), CODE8(3, 3), CODE8(3, 2), CODE8(2, 3), CODE8(1, 4),
    CODE8(1, 3), CODE8(0, 6),
    /**** Table #  7 size  9 ****/
    CODE8(1, 6), CODE8(1, 4), CODE8(1, 5), CODE8(3, 3), CODE8(3, 2), CODE8(2, 2), CODE8(2, 3), CODE8(1, 3),
    CODE8(0, 6),
    /**** Table #  8 size  8 ****/
    CODE8(1, 6), CODE8(0, 6), CODE8(1, 4), CODE8(3, 2), CODE8(2, 2), CODE8(1, 3), CODE8(1, 2), CODE8(1, 5),
    /**** Table #  9 size  7 ****/
    CODE8(1, 5), CODE8(0, 5), CODE8(1, 3), CODE8(3, 2), CODE8(2, 2), CODE8(1, 2), CODE8(1, 4),
    /**** Table # 10 size  6 ****/
    CODE8(0, 4), CODE8(1, 4), CODE8(1, 3), CODE8(2, 3), CODE8(1, 1), CODE8(3, 3),
    /**** Table # 11 size  5 ****/
    CODE8(0, 4), CODE8(1, 4), CODE8(1, 2), CODE8(1, 1), CODE8(1, 3),
    /**** Table # 12 size  4 ****/
    CODE8(0, 3), CODE8(1, 3), CODE8(1, 1), CODE8(1, 2),
    /**** Table # 13 size  3 ****/
    CODE8(0, 2), CODE8(1, 2), CODE8(1, 1),
    /**** Table # 14 size  2 ****/
    CODE8(0, 1), CODE8(1, 1),
};

const uint8_t h264e_g_coeff_token[277 + 18] =
{
    17 + 18, 17 + 18, // 0<=NC<2
    82 + 18, 82 + 18, // 2<=NC<4
    147 + 18, 147 + 18, 147 + 18, 147 + 18, // 5<=NC<8
    212 + 18, 212 + 18, 212 + 18, 212 + 18, 212 + 18, 212 + 18, 212 + 18, 212 + 18, 212 + 18, // NC>=8，定长表格
    0 + 18, // NC==-1
    /**** Table #  4 size 17 ****/     // offs: 0
    CODE(1, 2), CODE(1, 1), CODE(1, 3), CODE(5, 6), CODE(7, 6), CODE(6, 6), CODE(2, 7), CODE(0, 7), CODE(4, 6),
    CODE(3, 7), CODE(2, 8), CODE(0, 0), CODE(3, 6), CODE(3, 8), CODE(0, 0), CODE(0, 0), CODE(2, 6),
    /**** Table #  0 size 65 ****/     // offs: 17
    CODE( 1,  1), CODE( 1,  2), CODE( 1,  3), CODE( 3,  5), CODE( 5,  6), CODE( 4,  6), CODE( 5,  7), CODE( 3,  6),
    CODE( 7,  8), CODE( 6,  8), CODE( 5,  8), CODE( 4,  7), CODE( 7,  9), CODE( 6,  9), CODE( 5,  9), CODE( 4,  8),
    CODE( 7, 10), CODE( 6, 10), CODE( 5, 10), CODE( 4,  9), CODE( 7, 11), CODE( 6, 11), CODE( 5, 11), CODE( 4, 10),
    CODE(15, 13), CODE(14, 13), CODE(13, 13), CODE( 4, 11), CODE(11, 13), CODE(10, 13), CODE( 9, 13), CODE(12, 13),
    CODE( 8, 13), CODE(14, 14), CODE(13, 14), CODE(12, 14), CODE(15, 14), CODE(10, 14), CODE( 9, 14), CODE( 8, 14),
    CODE(11, 14), CODE(14, 15), CODE(13, 15), CODE(12, 15), CODE(15, 15), CODE(10, 15), CODE( 9, 15), CODE( 8, 15),
    CODE(11, 15), CODE( 1, 15), CODE(13, 16), CODE(12, 16), CODE(15, 16), CODE(14, 16), CODE( 9, 16), CODE( 8, 16),
    CODE(11, 16), CODE(10, 16), CODE( 5, 16), CODE( 0,  0), CODE( 7, 16), CODE( 6, 16), CODE( 0,  0), CODE( 0,  0), CODE( 4, 16),
    /**** Table #  1 size 65 ****/     // offs: 82
    CODE( 3,  2), CODE( 2,  2), CODE( 3,  3), CODE( 5,  4), CODE(11,  6), CODE( 7,  5), CODE( 9,  6), CODE( 4,  4),
    CODE( 7,  6), CODE(10,  6), CODE( 5,  6), CODE( 6,  5), CODE( 7,  7), CODE( 6,  6), CODE( 5,  7), CODE( 8,  6),
    CODE( 7,  8), CODE( 6,  7), CODE( 5,  8), CODE( 4,  6), CODE( 4,  8), CODE( 6,  8), CODE( 5,  9), CODE( 4,  7),
    CODE( 7,  9), CODE( 6,  9), CODE(13, 11), CODE( 4,  9), CODE(15, 11), CODE(14, 11), CODE( 9, 11), CODE(12, 11),
    CODE(11, 11), CODE(10, 11), CODE(13, 12), CODE( 8, 11), CODE(15, 12), CODE(14, 12), CODE( 9, 12), CODE(12, 12),
    CODE(11, 12), CODE(10, 12), CODE(13, 13), CODE(12, 13), CODE( 8, 12), CODE(14, 13), CODE( 9, 13), CODE( 8, 13),
    CODE(15, 13), CODE(10, 13), CODE( 6, 13), CODE( 1, 13), CODE(11, 13), CODE(11, 14), CODE(10, 14), CODE( 4, 14),
    CODE( 7, 13), CODE( 8, 14), CODE( 5, 14), CODE( 0,  0), CODE( 9, 14), CODE( 6, 14), CODE( 0,  0), CODE( 0,  0), CODE( 7, 14),
    /**** Table #  2 size 65 ****/     // offs: 147
    CODE(15,  4), CODE(14,  4), CODE(13,  4), CODE(12,  4), CODE(15,  6), CODE(15,  5), CODE(14,  5), CODE(11,  4),
    CODE(11,  6), CODE(12,  5), CODE(11,  5), CODE(10,  4), CODE( 8,  6), CODE(10,  5), CODE( 9,  5), CODE( 9,  4),
    CODE(15,  7), CODE( 8,  5), CODE(13,  6), CODE( 8,  4), CODE(11,  7), CODE(14,  6), CODE( 9,  6), CODE(13,  5),
    CODE( 9,  7), CODE(10,  6), CODE(13,  7), CODE(12,  6), CODE( 8,  7), CODE(14,  7), CODE(10,  7), CODE(12,  7),
    CODE(15,  8), CODE(14,  8), CODE(13,  8), CODE(12,  8), CODE(11,  8), CODE(10,  8), CODE( 9,  8), CODE( 8,  8),
    CODE(15,  9), CODE(14,  9), CODE(13,  9), CODE(12,  9), CODE(11,  9), CODE(10,  9), CODE( 9,  9), CODE(10, 10),
    CODE( 8,  9), CODE( 7,  9), CODE(11, 10), CODE( 6, 10), CODE(13, 10), CODE(12, 10), CODE( 7, 10), CODE( 2, 10),
    CODE( 9, 10), CODE( 8, 10), CODE( 3, 10), CODE( 0,  0), CODE( 5, 10), CODE( 4, 10), CODE( 0,  0), CODE( 0,  0), CODE( 1, 10),
    /**** Table #  3 size 65 ****/     // offs: 212
     3,  1,  6, 11,  0,  5, 10, 15,  4,  9, 14, 19,  8, 13, 18, 23, 12, 17, 22, 27, 16, 21, 26, 31, 20, 25, 30, 35,
    24, 29, 34, 39, 28, 33, 38, 43, 32, 37, 42, 47, 36, 41, 46, 51, 40, 45, 50, 55, 44, 49, 54, 59, 48, 53, 58, 63,
    52, 57, 62,  0, 56, 61,  0,  0, 60
};

/*
    Block scan order
    0 1 4 5
    2 3 6 7
    8 9 C D
    A B E F
*/
static const uint8_t decode_block_scan[16] = { 0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15 };

static const uint8_t qpy2qpc[52] = {  // todo: [0 - 9] not used
    0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12,
   13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
   26, 27, 28, 29, 29, 30, 31, 32, 32, 33, 34, 34, 35,
   35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39,
};

/**
*   Rate-control LUT for intra/inter macroblocks: number of bits per macroblock for given QP
*   Estimated experimentally
*/
static const uint16_t bits_per_mb[2][42 - 1] =
{
    // 10                                                          20                                                          30                                                          40                                                          50
    { 664,  597,  530,  484,  432,  384,  341,  297,  262,  235,  198,  173,  153,  131,  114,  102,   84,   74,   64,   54,   47,   42,   35,   31,   26,   22,   20,   17,   15,   13,   12,   10,    9,    9,    7,    7,    6,    5,    4,    1,    1}, // P
    {1057,  975,  925,  868,  803,  740,  694,  630,  586,  547,  496,  457,  420,  378,  345,  318,  284,  258,  234,  210,  190,  178,  155,  141,  129,  115,  102,   95,   82,   75,   69,   60,   55,   51,   45,   41,   40,   35,   31,   28,   24}  // I
};

/**
*   Deblock filter constants:
*   <alpha> <thr[1]> <thr[2]> <thr[3]> <beta>
*/
static const uint8_t g_a_tc0_b[52 - 10][5] = {
    {  0,  0,  0,  0,  0},  // 10
    {  0,  0,  0,  0,  0},  // 11
    {  0,  0,  0,  0,  0},  // 12
    {  0,  0,  0,  0,  0},  // 13
    {  0,  0,  0,  0,  0},  // 14
    {  0,  0,  0,  0,  0},  // 15
    {  4,  0,  0,  0,  2},
    {  4,  0,  0,  1,  2},
    {  5,  0,  0,  1,  2},
    {  6,  0,  0,  1,  3},
    {  7,  0,  0,  1,  3},
    {  8,  0,  1,  1,  3},
    {  9,  0,  1,  1,  3},
    { 10,  1,  1,  1,  4},
    { 12,  1,  1,  1,  4},
    { 13,  1,  1,  1,  4},
    { 15,  1,  1,  1,  6},
    { 17,  1,  1,  2,  6},
    { 20,  1,  1,  2,  7},
    { 22,  1,  1,  2,  7},
    { 25,  1,  1,  2,  8},
    { 28,  1,  2,  3,  8},
    { 32,  1,  2,  3,  9},
    { 36,  2,  2,  3,  9},
    { 40,  2,  2,  4, 10},
    { 45,  2,  3,  4, 10},
    { 50,  2,  3,  4, 11},
    { 56,  3,  3,  5, 11},
    { 63,  3,  4,  6, 12},
    { 71,  3,  4,  6, 12},
    { 80,  4,  5,  7, 13},
    { 90,  4,  5,  8, 13},
    {101,  4,  6,  9, 14},
    {113,  5,  7, 10, 14},
    {127,  6,  8, 11, 15},
    {144,  6,  8, 13, 15},
    {162,  7, 10, 14, 16},
    {182,  8, 11, 16, 16},
    {203,  9, 12, 18, 17},
    {226, 10, 13, 20, 17},
    {255, 11, 15, 23, 18},
    {255, 13, 17, 25, 18},
};

/************************************************************************/
/*  Adjustable encoder parameters. Initial MIN_QP values never used     */
/************************************************************************/

ADJUSTABLE uint16_t g_rnd_inter[] = {
    11665, 11665, 11665, 11665, 11665, 11665, 11665, 11665, 11665, 11665,
    11665, 12868, 14071, 15273, 16476,
    17679, 17740, 17801, 17863, 17924,
    17985, 17445, 16904, 16364, 15823,
    15283, 15198, 15113, 15027, 14942,
    14857, 15667, 16478, 17288, 18099,
    18909, 19213, 19517, 19822, 20126,
    20430, 16344, 12259, 8173, 4088,
    4088, 4088, 4088, 4088, 4088,
    4088, 4088,
};

ADJUSTABLE uint16_t g_thr_inter[] = {
    31878, 31878, 31878, 31878, 31878, 31878, 31878, 31878, 31878, 31878,
    31878, 33578, 35278, 36978, 38678,
    40378, 41471, 42563, 43656, 44748,
    45841, 46432, 47024, 47615, 48207,
    48798, 49354, 49911, 50467, 51024,
    51580, 51580, 51580, 51580, 51580,
    51580, 52222, 52864, 53506, 54148,
    54790, 45955, 37120, 28286, 19451,
    10616, 9326, 8036, 6745, 5455,
    4165, 4165,
};

ADJUSTABLE uint16_t g_thr_inter2[] = {
    45352, 45352, 45352, 45352, 45352, 45352, 45352, 45352, 45352, 45352,
    45352, 41100, 36848, 32597, 28345,
    24093, 25904, 27715, 29525, 31336,
    33147, 33429, 33711, 33994, 34276,
    34558, 32902, 31246, 29590, 27934,
    26278, 26989, 27700, 28412, 29123,
    29834, 29038, 28242, 27445, 26649,
    25853, 23440, 21028, 18615, 16203,
    13790, 11137, 8484, 5832, 3179,
    526, 526,
};

ADJUSTABLE uint16_t g_skip_thr_inter[52] =
{
    45, 45, 45, 45, 45, 45, 45, 45, 45, 45,
    45, 45, 45, 44, 44,
    44, 40, 37, 33, 30,
    26, 32, 38, 45, 51,
    57, 58, 58, 59, 59,
    60, 66, 73, 79, 86,
    92, 95, 98, 100, 103,
    106, 200, 300, 400, 500,
    600, 700, 800, 900, 1000,
    1377, 1377,
};

ADJUSTABLE uint16_t g_lambda_q4[52] =
{
    14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
    14, 13, 11, 10, 8,
    7, 11, 15, 20, 24,
    28, 30, 31, 33, 34,
    36, 48, 60, 71, 83,
    95, 95, 95, 96, 96,
    96, 113, 130, 147, 164,
    181, 401, 620, 840, 1059,
    1279, 1262, 1246, 1229, 1213,
    1196, 1196,
};
ADJUSTABLE uint16_t g_lambda_mv_q4[52] =
{
    13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
    13, 14, 15, 15, 16,
    17, 18, 20, 21, 23,
    24, 28, 32, 37, 41,
    45, 53, 62, 70, 79,
    87, 105, 123, 140, 158,
    176, 195, 214, 234, 253,
    272, 406, 541, 675, 810,
    944, 895, 845, 796, 746,
    697, 697,
};

ADJUSTABLE uint16_t g_skip_thr_i4x4[52] =
{
    0,1,2,3,4,5,6,7,8,9,
    7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
    24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
    44, 44, 44, 44, 44, 44, 44, 44, 44, 44,
    68, 68, 68, 68, 68, 68, 68, 68, 68, 68,
    100, 100,
};

ADJUSTABLE uint16_t g_deadzonei[] = {
    3419, 3419, 3419, 3419, 3419, 3419, 3419, 3419, 3419, 3419,
    30550, 8845, 14271, 19698, 25124,
    30550, 29556, 28562, 27569, 26575,
    25581, 25284, 24988, 24691, 24395,
    24098, 24116, 24134, 24153, 24171,
    24189, 24010, 23832, 23653, 23475,
    23296, 23569, 23842, 24115, 24388,
    24661, 19729, 14797, 9865, 4933,
    24661, 3499, 6997, 10495, 13993,
    17491, 17491,
};

// luma 4x4 Macroblock initial cost according to QP
ADJUSTABLE uint16_t g_lambda_i4_q4[] = {
    27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
    27, 31, 34, 38, 41,
    45, 76, 106, 137, 167,
    198, 220, 243, 265, 288,
    310, 347, 384, 421, 458,
    495, 584, 673, 763, 852,
    941, 1053, 1165, 1276, 1388,
    1500, 1205, 910, 614, 319,
    5000, 1448, 2872, 4296, 5720,
    7144, 7144,
};

ADJUSTABLE uint16_t g_lambda_i16_q4[] = {
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0,
    0, 3, 7, 10, 14,
    17, 14, 10, 7, 3,
    50, 20, 39, 59, 78,
    98, 94, 89, 85, 80,
    76, 118, 161, 203, 246,
    288, 349, 410, 470, 531,
    592, 575, 558, 540, 523,
    506, 506,
};

const uint8_t g_diff_to_gainQ8[256] =
{
    0, 16, 25, 32, 37, 41, 44, 48, 50, 53, 55, 57, 59, 60, 62, 64, 65,
    66, 67, 69, 70, 71, 72, 73, 74, 75, 76, 76, 77, 78, 79, 80, 80,
    81, 82, 82, 83, 83, 84, 85, 85, 86, 86, 87, 87, 88, 88, 89, 89,
    90, 90, 91, 91, 92, 92, 92, 93, 93, 94, 94, 94, 95, 95, 96, 96,
    96, 97, 97, 97, 98, 98, 98, 99, 99, 99, 99, 100, 100, 100, 101, 101,
    101, 102, 102, 102, 102, 103, 103, 103, 103, 104, 104, 104, 104, 105, 105, 105,
    105, 106, 106, 106, 106, 106, 107, 107, 107, 107, 108, 108, 108, 108, 108, 109,
    109, 109, 109, 109, 110, 110, 110, 110, 110, 111, 111, 111, 111, 111, 112, 112,
    112, 112, 112, 112, 113, 113, 113, 113, 113, 113, 114, 114, 114, 114, 114, 114,
    115, 115, 115, 115, 115, 115, 115, 116, 116, 116, 116, 116, 116, 117, 117, 117,
    117, 117, 117, 117, 118, 118, 118, 118, 118, 118, 118, 118, 119, 119, 119, 119,
    119, 119, 119, 119, 120, 120, 120, 120, 120, 120, 120, 120, 121, 121, 121, 121,
    121, 121, 121, 121, 122, 122, 122, 122, 122, 122, 122, 122, 122, 123, 123, 123,
    123, 123, 123, 123, 123, 123, 124, 124, 124, 124, 124, 124, 124, 124, 124, 125,
    125, 125, 125, 125, 125, 125, 125, 125, 125, 126, 126, 126, 126, 126, 126, 126,
    126, 126, 126, 126, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 128,
};

#if H264E_ENABLE_SSE2 && !defined(MINIH264_ASM)
#define BS_BITS 32

static void h264e_bs_put_bits_sse2(bs_t *bs, unsigned n, unsigned val)
{
    assert(!(val >> n));
    bs->shift -= n;
    assert((unsigned)n <= 32);
    if (bs->shift < 0)
    {
        assert(-bs->shift < 32);
        bs->cache |= val >> -bs->shift;
        *bs->buf++ = SWAP32(bs->cache);
        bs->shift = 32 + bs->shift;
        bs->cache = 0;
    }
    bs->cache |= val << bs->shift;
}

static void h264e_bs_flush_sse2(bs_t *bs)
{
    *bs->buf = SWAP32(bs->cache);
}

static unsigned h264e_bs_get_pos_bits_sse2(const bs_t *bs)
{
    unsigned pos_bits = (unsigned)((bs->buf - bs->origin)*BS_BITS);
    pos_bits += BS_BITS - bs->shift;
    assert((int)pos_bits >= 0);
    return pos_bits;
}

static unsigned h264e_bs_byte_align_sse2(bs_t *bs)
{
    int pos = h264e_bs_get_pos_bits_sse2(bs);
    h264e_bs_put_bits_sse2(bs, -pos & 7, 0);
    return pos + (-pos & 7);
}

/**
*   Golomb code
*   0 => 1
*   1 => 01 0
*   2 => 01 1
*   3 => 001 00
*   4 => 001 01
*
*   [0]     => 1
*   [1..2]  => 01x
*   [3..6]  => 001xx
*   [7..14] => 0001xxx
*
*/
static void h264e_bs_put_golomb_sse2(bs_t *bs, unsigned val)
{
    int size;
#if defined(_MSC_VER)
    unsigned long nbit;
    _BitScanReverse(&nbit, val + 1);
    size = 1 + nbit;
#else
    size = 32 - __builtin_clz(val + 1);
#endif
    h264e_bs_put_bits_sse2(bs, 2*size - 1, val + 1);
}

/**
*   signed Golomb code.
*   mapping to unsigned code:
*       0 => 0
*       1 => 1
*      -1 => 2
*       2 => 3
*      -2 => 4
*       3 => 5
*      -3 => 6
*/
static void h264e_bs_put_sgolomb_sse2(bs_t *bs, int val)
{
    val = 2*val - 1;
    val ^= val >> 31;
    h264e_bs_put_golomb_sse2(bs, val);
}

static void h264e_bs_init_bits_sse2(bs_t *bs, void *data)
{
    bs->origin = data;
    bs->buf = bs->origin;
    bs->shift = BS_BITS;
    bs->cache = 0;
}

static unsigned __clz_cavlc(unsigned v)
{
#if defined(_MSC_VER)
    unsigned long nbit;
    _BitScanReverse(&nbit, v);
    return 31 - nbit;
#else
    return __builtin_clz(v);
#endif
}

static void h264e_vlc_encode_sse2(bs_t *bs, int16_t *quant, int maxNumCoeff, uint8_t *nz_ctx)
{
    int nnz_context, nlevels, nnz; // nnz = nlevels + trailing_ones
    unsigned trailing_ones = 0;
    unsigned trailing_ones_sign = 0;
    uint8_t runs[16];
    uint8_t *prun = runs;
    int16_t *levels;
    int cloop = maxNumCoeff;
    int v, drun;
    unsigned zmask;
    BS_OPEN(bs)

    ALIGN(16) int16_t zzquant[16] ALIGN2(16);
    levels = zzquant + ((maxNumCoeff == 4) ? 4 : 16);
    if (maxNumCoeff != 4)
    {
        __m128i y0, y1;
        __m128i x0 = _mm_load_si128((__m128i *)quant);
        __m128i x1 = _mm_load_si128((__m128i *)(quant + 8));
#define SWAP_XMM(x, i, j)     { int t0 = _mm_extract_epi16(x, i); int t1 = _mm_extract_epi16(x, j); x = _mm_insert_epi16(x, t0, j); x = _mm_insert_epi16(x, t1, i); }
#define SWAP_XMM2(x, y, i, j) { int t0 = _mm_extract_epi16(x, i); int t1 = _mm_extract_epi16(y, j); y = _mm_insert_epi16(y, t0, j); x = _mm_insert_epi16(x, t1, i); }
        SWAP_XMM(x0, 3, 4);
        SWAP_XMM(x1, 3, 4);
        SWAP_XMM2(x0, x1, 5, 2);
        x0 = _mm_shufflelo_epi16(x0, 0 + (3 << 2) + (1 << 4) + (2 << 6));
        x0 = _mm_shufflehi_epi16(x0, 2 + (0 << 2) + (3 << 4) + (1 << 6));
        x1 = _mm_shufflelo_epi16(x1, 2 + (0 << 2) + (3 << 4) + (1 << 6));
        x1 = _mm_shufflehi_epi16(x1, 1 + (2 << 2) + (0 << 4) + (3 << 6));
        y0 = _mm_unpacklo_epi64(x0, x1);
        y1 = _mm_unpackhi_epi64(x0, x1);
        y0 = _mm_slli_epi16(y0, 1);
        y1 = _mm_slli_epi16(y1, 1);
        zmask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_packs_epi16(y0, y1), _mm_setzero_si128()));
        _mm_store_si128((__m128i *)zzquant, y0);
        _mm_store_si128((__m128i *)(zzquant + 8), y1);

        if (maxNumCoeff == 15)
            zmask |= 1;
        zmask = (~zmask) << 16;

        v = 15;
        drun = (maxNumCoeff == 16) ? 1 : 0;
    } else
    {
        __m128i x0 = _mm_loadl_epi64((__m128i *)quant);
        x0 = _mm_slli_epi16(x0, 1);
        zmask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_packs_epi16(x0, x0), _mm_setzero_si128()));
        _mm_storel_epi64((__m128i *)zzquant, x0);
        zmask = (~zmask) << 28;
        drun = 1;
        v = 3;
    }

    if (zmask)
    {
        do
        {
            int i = __clz_cavlc(zmask);
            *--levels = zzquant[v -= i];
            *prun++ = (uint8_t)(v + drun);
            zmask <<= (i + 1);
            v--;
        } while(zmask);
        quant = zzquant + ((maxNumCoeff == 4) ? 4 : 16);
        nnz = (int)(quant - levels);

        cloop = MIN(3, nnz);
        levels = quant - 1;
        do
        {
            if ((unsigned)(*levels + 2) > 4u)
            {
                break;
            }
            trailing_ones_sign = (trailing_ones_sign << 1) | (*levels-- < 0);
            trailing_ones++;
        } while (--cloop);
    } else
    {
        nnz = trailing_ones = 0;
    }
    nlevels = nnz - trailing_ones;

    nnz_context = nz_ctx[-1] + nz_ctx[1];

    nz_ctx[0] = (uint8_t)nnz;
    if (nnz_context <= 34)
    {
        nnz_context = (nnz_context + 1) >> 1;
    }
    nnz_context &= 31;

    // 9.2.1 Parsing process for total number of transform coefficient levels and trailing ones
    {
        int off = h264e_g_coeff_token[nnz_context];
        unsigned n = 6, val = h264e_g_coeff_token[off + trailing_ones + 4*nlevels];
        if (off != 230)
        {
            n = (val & 15) + 1;
            val >>= 4;
        }
        BS_PUT(n, val);
    }

    if (nnz)
    {
        if (trailing_ones)
        {
            BS_PUT(trailing_ones, trailing_ones_sign);
        }
        if (nlevels)
        {
            int vlcnum = 1;
            int sym_len, prefix_len;

            int sym = *levels-- - 2;
            if (sym < 0) sym = -3 - sym; // 将非零系数从有符号变为无符号；
            if (sym >= 6) vlcnum++; // ?
            if (trailing_ones < 3)
            {
                sym -= 2;
                if (nnz > 10)
                {
                    sym_len = 1;
                    prefix_len = sym >> 1;
                    if (prefix_len >= 15)
                    {
                        // or vlcnum = 1;  goto escape;
                        prefix_len = 15;
                        sym_len = 12;
                    }
                    sym -= prefix_len << 1;
                    // bypass vlcnum advance due to sym -= 2; above
                    goto loop_enter;
                }
            }

            if (sym < 14)
            {
                prefix_len = sym;
                sym = 0; // to avoid side effect in bitbuf
                sym_len = 0;
            } else if (sym < 30)
            {
                prefix_len = 14;
                sym_len = 4;
                sym -= 14;
            } else
            {
                vlcnum = 1;
                goto escape;
            }
            goto loop_enter;

            for (;;)
            {
                // sym = 4; vlcnum=1;
                sym_len = vlcnum; // 1
                prefix_len = sym >> vlcnum; // 2
                if (prefix_len >= 15)
                {
escape:
                    prefix_len = 15;
                    sym_len = 12;
                }
                sym -= prefix_len << vlcnum; // 0

                if (prefix_len >= 3 && vlcnum < 6) vlcnum++; // 更新后缀长度
loop_enter:
                sym |= 1 << sym_len;
                sym_len += prefix_len+1;
                BS_PUT(sym_len, (unsigned)sym);
                if (!--nlevels) break;
                sym = *levels-- - 2;
                if (sym < 0) sym = -3 - sym;
            }
        }

        if (nnz < maxNumCoeff)
        {
            const uint8_t *vlc = (maxNumCoeff == 4) ? h264e_g_total_zeros_cr_2x2 : h264e_g_total_zeros;
            uint8_t *run = runs;
            int run_prev = *run++;
            int nzeros = run_prev - nnz;
            int zeros_left = 2*nzeros - 1;
            int ctx = nnz - 1;
            run[nnz - 1] = (uint8_t)maxNumCoeff; // terminator
            for(;;)
            {
                int t;
                //encode_huff8(bs, vlc, ctx, nzeros);

                unsigned val = vlc[vlc[ctx] + nzeros];
                unsigned n = val & 15;
                val >>= 4;
                BS_PUT(n, val);

                zeros_left -= nzeros;
                if (zeros_left < 0)
                {
                    break;
                }

                t = *run++;
                nzeros = run_prev - t - 1;
                if (nzeros < 0)
                {
                    break;
                }
                run_prev = t;
                assert(zeros_left < 14);
                vlc = h264e_g_run_before;
                ctx = zeros_left;
            }
        }
    }
    BS_CLOSE(bs);
}

#define MM_LOAD_8TO16_2(p) _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(p)), _mm_setzero_si128())
static __inline __m128i subabs128_16(__m128i a, __m128i b)
{
    return _mm_or_si128(_mm_subs_epu16(a, b), _mm_subs_epu16(b, a));
}
static __inline __m128i clone2x16(const void *p)
{
    __m128i tmp = MM_LOAD_8TO16_2(p);
    return _mm_unpacklo_epi16(tmp, tmp);
}
static __inline __m128i subabs128(__m128i a, __m128i b)
{
    return _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
}

static void transpose8x8_sse(uint8_t *dst, int dst_stride, uint8_t *src, int src_stride)
{
    __m128i a = _mm_loadl_epi64((__m128i *)(src));
    __m128i b = _mm_loadl_epi64((__m128i *)(src += src_stride));
    __m128i c = _mm_loadl_epi64((__m128i *)(src += src_stride));
    __m128i d = _mm_loadl_epi64((__m128i *)(src += src_stride));
    __m128i e = _mm_loadl_epi64((__m128i *)(src += src_stride));
    __m128i f = _mm_loadl_epi64((__m128i *)(src += src_stride));
    __m128i g = _mm_loadl_epi64((__m128i *)(src += src_stride));
    __m128i h = _mm_loadl_epi64((__m128i *)(src += src_stride));

    __m128i p0 = _mm_unpacklo_epi8(a,b);  // b7 a7 b6 a6 ... b0 a0
    __m128i p1 = _mm_unpacklo_epi8(c,d);  // d7 c7 d6 c6 ... d0 c0
    __m128i p2 = _mm_unpacklo_epi8(e,f);  // f7 e7 f6 e6 ... f0 e0
    __m128i p3 = _mm_unpacklo_epi8(g,h);  // h7 g7 h6 g6 ... h0 g0

    __m128i q0 = _mm_unpacklo_epi16(p0, p1);  // d3c3 b3a3 ... d0c0 b0a0
    __m128i q1 = _mm_unpackhi_epi16(p0, p1);  // d7c7 b7a7 ... d4c4 b4a4
    __m128i q2 = _mm_unpacklo_epi16(p2, p3);  // h3g3 f3e3 ... h0g0 f0e0
    __m128i q3 = _mm_unpackhi_epi16(p2, p3);  // h7g7 f7e7 ... h4g4 f4e4

    __m128i r0 = _mm_unpacklo_epi32(q0, q2);  // h1g1f1e1 d1c1b1a1 h0g0f0e0 d0c0b0a0
    __m128i r1 = _mm_unpackhi_epi32(q0, q2);  // h3g3f3e3 d3c3b3a3 h2g2f2e2 d2c2b2a2
    __m128i r2 = _mm_unpacklo_epi32(q1, q3);
    __m128i r3 = _mm_unpackhi_epi32(q1, q3);
    _mm_storel_epi64((__m128i *)(dst), r0); dst += dst_stride; _mm_storel_epi64((__m128i *)(dst), _mm_unpackhi_epi64(r0, r0)); dst += dst_stride;
    _mm_storel_epi64((__m128i *)(dst), r1); dst += dst_stride; _mm_storel_epi64((__m128i *)(dst), _mm_unpackhi_epi64(r1, r1)); dst += dst_stride;
    _mm_storel_epi64((__m128i *)(dst), r2); dst += dst_stride; _mm_storel_epi64((__m128i *)(dst), _mm_unpackhi_epi64(r2, r2)); dst += dst_stride;
    _mm_storel_epi64((__m128i *)(dst), r3); dst += dst_stride; _mm_storel_epi64((__m128i *)(dst), _mm_unpackhi_epi64(r3, r3)); dst += dst_stride;
}

static void deblock_chroma_h_s4_sse(uint8_t *pq0, int stride, const void* threshold, int alpha, int beta, uint32_t argstr)
{
    __m128i thr, str, d;
    __m128i p1 = MM_LOAD_8TO16_2(pq0 - 2*stride);
    __m128i p0 = MM_LOAD_8TO16_2(pq0 - stride);
    __m128i q0 = MM_LOAD_8TO16_2(pq0);
    __m128i q1 = MM_LOAD_8TO16_2(pq0 + stride);
    __m128i zero = _mm_setzero_si128();
    __m128i _alpha = _mm_set1_epi16((short)alpha);
    __m128i _beta = _mm_set1_epi16((short)beta);
    __m128i tmp;

    str =                    _mm_cmplt_epi16(subabs128_16(p0, q0), _alpha);
    str = _mm_and_si128(str, _mm_cmplt_epi16(_mm_max_epi16(subabs128_16(p1, p0), subabs128_16(q1, q0)), _beta));

    if ((uint8_t)argstr != 4)
    {
        d = _mm_srai_epi16(_mm_add_epi16(_mm_sub_epi16(_mm_add_epi16(_mm_slli_epi16(_mm_sub_epi16(q0, p0), 2), p1), q1),_mm_set1_epi16(4)), 3);
        thr = _mm_add_epi16(clone2x16(threshold), _mm_set1_epi16(1));
        d = _mm_min_epi16(_mm_max_epi16(d, _mm_sub_epi16(zero, thr)), thr);

        tmp = _mm_unpacklo_epi8(_mm_cvtsi32_si128(argstr), _mm_setzero_si128());
        tmp = _mm_unpacklo_epi16(tmp, tmp);

//        str = _mm_and_si128(str, _mm_cmpgt_epi16(clone2x16(strength), zero));
        str = _mm_and_si128(str, _mm_cmpgt_epi16(tmp, zero));
        d = _mm_and_si128(str, d);
        p0 = _mm_add_epi16(p0, d);
        q0 = _mm_sub_epi16(q0, d);
    } else
    {
        __m128i pq = _mm_add_epi16(p1, q1);
        __m128i newp = _mm_srai_epi16(_mm_add_epi16(_mm_add_epi16(pq, p1), p0), 1);
        __m128i newq = _mm_srai_epi16(_mm_add_epi16(_mm_add_epi16(pq, q1), q0), 1);
        p0 = _mm_xor_si128(_mm_and_si128(_mm_xor_si128(_mm_avg_epu16(newp,zero), p0), str), p0);
        q0 = _mm_xor_si128(_mm_and_si128(_mm_xor_si128(_mm_avg_epu16(newq,zero), q0), str), q0);
    }
    _mm_storel_epi64((__m128i*)(pq0 - stride), _mm_packus_epi16(p0, zero));
    _mm_storel_epi64((__m128i*)(pq0         ), _mm_packus_epi16(q0, zero));
}

static void deblock_chroma_v_s4_sse(uint8_t *pix, int stride, const void* threshold, int alpha, int beta, uint32_t str)
{
    uint8_t t8x4[8*4];
    int i;
    uint8_t *p = pix - 2;
    __m128i t0 =_mm_unpacklo_epi16(
        _mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int_u*)p),              _mm_cvtsi32_si128(*(int_u*)(p + stride))),
        _mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int_u*)(p + 2*stride)), _mm_cvtsi32_si128(*(int_u*)(p + 3*stride)))
        );
    __m128i t1 =_mm_unpacklo_epi16(
        _mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int_u*)(p + 4*stride)), _mm_cvtsi32_si128(*(int_u*)(p + 5*stride))),
        _mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int_u*)(p + 6*stride)), _mm_cvtsi32_si128(*(int_u*)(p + 7*stride)))
        );
    __m128i p1 = _mm_unpacklo_epi32(t0, t1);
    __m128i p0 = _mm_shuffle_epi32 (p1, 0x4E); // 01001110b
    __m128i q0 = _mm_unpackhi_epi32(t0, t1);
    __m128i q1 = _mm_shuffle_epi32 (q0, 0x4E);
    _mm_storel_epi64((__m128i*)(t8x4), p1);
    _mm_storel_epi64((__m128i*)(t8x4 + 8), p0);
    _mm_storel_epi64((__m128i*)(t8x4 + 16), q0);
    _mm_storel_epi64((__m128i*)(t8x4 + 24), q1);
    deblock_chroma_h_s4_sse(t8x4 + 16, 8, threshold, alpha, beta, str);

    for (i = 0; i < 8; i++)
    {
        pix[-1] = t8x4[8  + i];
        pix[ 0] = t8x4[16 + i];
        pix += stride;
    }
}

#define CMP_BETA(p, q, beta)   _mm_cmpeq_epi8(_mm_subs_epu8(_mm_subs_epu8(p, q), beta), _mm_subs_epu8(_mm_subs_epu8(q, p), beta))
#define CMP_1(p, q, beta)     (_mm_subs_epu8(subabs128(p, q), beta))

static void deblock_luma_h_s4_sse(uint8_t *pix, int stride, int alpha, int beta)
{
    int ccloop = 2;
    do
    {
        __m128i p3 = MM_LOAD_8TO16_2(pix - 4*stride);
        __m128i p2 = MM_LOAD_8TO16_2(pix - 3*stride);
        __m128i p1 = MM_LOAD_8TO16_2(pix - 2*stride);
        __m128i p0 = MM_LOAD_8TO16_2(pix - stride);
        __m128i q0 = MM_LOAD_8TO16_2(pix);
        __m128i q1 = MM_LOAD_8TO16_2(pix + stride);
        __m128i q2 = MM_LOAD_8TO16_2(pix + 2*stride);
        __m128i q3 = MM_LOAD_8TO16_2(pix + 3*stride);
        __m128i zero = _mm_setzero_si128();
        __m128i _alpha = _mm_set1_epi16((short)alpha);
        __m128i _quarteralpha = _mm_set1_epi16((short)((alpha >> 2) + 2));
        __m128i _beta = _mm_set1_epi16((short)beta);
        __m128i ap_less_beta;
        __m128i aq_less_beta;
        __m128i str;
        __m128i pq;
        __m128i short_p;
        __m128i short_q;
        __m128i long_p;
        __m128i long_q;
        __m128i t;
        __m128i p0q0_less__quarteralpha;

        __m128i absdif_p0_q0 = subabs128_16(p0, q0);
        __m128i p0_plus_q0 = _mm_add_epi16(_mm_add_epi16(p0, q0), _mm_set1_epi16(2));

        // if (abs_p0_q0 < alpha && abs_p1_p0 < beta && abs_q1_q0 < beta)
        str = _mm_cmplt_epi16(absdif_p0_q0, _alpha);
        //str = _mm_and_si128(str, _mm_cmplt_epi16(subabs128_16(p1, p0), _beta));
        //str = _mm_and_si128(str, _mm_cmplt_epi16(subabs128_16(q1, q0), _beta));
        str = _mm_and_si128(str, _mm_cmplt_epi16(_mm_max_epi16(subabs128_16(p1, p0), subabs128_16(q1, q0)), _beta));
        p0q0_less__quarteralpha = _mm_and_si128(_mm_cmplt_epi16(absdif_p0_q0, _quarteralpha), str);

        //int short_p = (2*p1 + p0 + q1 + 2);
        //int short_q = (2*q1 + q0 + p1 + 2);
        short_p = _mm_avg_epu8(_mm_avg_epu8(p0, q1),p1);
        pq = _mm_add_epi16(_mm_add_epi16(p1, q1), _mm_set1_epi16(2));
        short_p = _mm_add_epi16(_mm_add_epi16(pq, p1), p0);
        short_q = _mm_add_epi16(_mm_add_epi16(pq, q1), q0);

        ap_less_beta = _mm_and_si128(_mm_cmplt_epi16(subabs128_16(p2, p0), _beta), p0q0_less__quarteralpha);
        t = _mm_add_epi16(_mm_add_epi16(p2, p1), p0_plus_q0);
        // short_p += t - p1 + q0;
        long_p = _mm_srai_epi16(_mm_add_epi16(_mm_sub_epi16(_mm_add_epi16(short_p, t), p1), q0), 1);

        _mm_storel_epi64((__m128i*)(pix - 2*stride), _mm_packus_epi16(_mm_or_si128(_mm_and_si128(ap_less_beta, _mm_srai_epi16(t, 2)), _mm_andnot_si128(ap_less_beta, p1)), zero));
        t = _mm_add_epi16(_mm_add_epi16(_mm_slli_epi16(_mm_add_epi16(p3, p2), 1), t), _mm_set1_epi16(2));
        _mm_storel_epi64((__m128i*)(pix - 3*stride), _mm_packus_epi16(_mm_or_si128(_mm_and_si128(ap_less_beta, _mm_srai_epi16(t, 3)), _mm_andnot_si128(ap_less_beta, p2)), zero));

        aq_less_beta = _mm_and_si128(_mm_cmplt_epi16(subabs128_16(q2, q0), _beta), p0q0_less__quarteralpha);
        t = _mm_add_epi16(_mm_add_epi16(q2, q1), p0_plus_q0);
        long_q = _mm_srai_epi16(_mm_add_epi16(_mm_sub_epi16(_mm_add_epi16(short_q, t), q1), p0), 1);
        _mm_storel_epi64((__m128i*)(pix + 1*stride), _mm_packus_epi16(_mm_or_si128(_mm_and_si128(aq_less_beta, _mm_srai_epi16(t, 2)), _mm_andnot_si128(aq_less_beta, q1)), zero));

        t = _mm_add_epi16(_mm_add_epi16(_mm_slli_epi16(_mm_add_epi16(q3, q2), 1), t), _mm_set1_epi16(2));
        _mm_storel_epi64((__m128i*)(pix + 2*stride), _mm_packus_epi16(_mm_or_si128(_mm_and_si128(aq_less_beta, _mm_srai_epi16(t, 3)), _mm_andnot_si128(aq_less_beta, q2)), zero));

        short_p = _mm_srai_epi16(_mm_or_si128(_mm_and_si128(ap_less_beta, long_p), _mm_andnot_si128(ap_less_beta, short_p)), 2);
        short_q = _mm_srai_epi16(_mm_or_si128(_mm_and_si128(aq_less_beta, long_q), _mm_andnot_si128(aq_less_beta, short_q)), 2);

        _mm_storel_epi64((__m128i*)(pix - stride), _mm_packus_epi16(_mm_or_si128(_mm_and_si128(str, short_p), _mm_andnot_si128(str, p0)), zero));
        _mm_storel_epi64((__m128i*)(pix         ), _mm_packus_epi16(_mm_or_si128(_mm_and_si128(str, short_q), _mm_andnot_si128(str, q0)), zero));

        pix += 8;
    } while (--ccloop);
}

static void deblock_luma_v_s4_sse(uint8_t *pix, int stride, int alpha, int beta)
{
    __m128i scratch[8];
    uint8_t *s = pix - 4;
    uint8_t *dst = (uint8_t *)scratch;
    int cloop = 2;
    do
    {
        transpose8x8_sse(dst, 16, s, stride);
        s += 8*stride;
        dst += 8;
    } while(--cloop);

    deblock_luma_h_s4_sse((uint8_t *)(scratch+4), 16, alpha, beta);
    s = pix - 4;
    dst = (uint8_t *)scratch;
    cloop = 2;
    do
    {
        transpose8x8_sse(s, stride, dst, 16);
        s += 8*stride;
        dst += 8;
    } while(--cloop);
}

// (a-b) >> 1s == ((a + ~b + 1) >> 1u) - 128;
//
// delta = (((q0-p0)<<2) + (p1-q1) + 4) >> 3 =
//          (4*q0 - 4*p0 + p1 - q1 + 4) >> 3 =
//          ((p1-p0) - (q1-q0) - 3*(p0-q0) + 4) >> 3
//          ((p1-p0) - (q1-q0) - 3*p0 + 3*q0) + 4) >> 3
//          (((p1-p0)-p0)>>1 - ((q1-q0)-q0)>>1 - p0 + q0) + 2) >> 2
//          ((((p1-p0)-p0)>>1 - p0)>>1 - (((q1-q0)-q0)>>1 - q0)>>1) + 1) >> 1
static void deblock_luma_h_s3_sse(uint8_t *h264e_restrict pix, int stride, int alpha, int beta, const void* threshold, uint32_t strength)
{
    __m128i p1 = _mm_loadu_si128((__m128i *)(pix - 2*stride));
    __m128i p0 = _mm_loadu_si128((__m128i *)(pix - stride));
    __m128i q0 = _mm_loadu_si128((__m128i *)pix);
    __m128i q1 = _mm_loadu_si128((__m128i *)(pix + stride));
    __m128i maskp, maskq, zeromask, thr;
    __m128i tc0tmp, p2, q2, p0q0avg, _beta;

#define HALFSUM(x, y) _mm_sub_epi8(_mm_avg_epu8(x, y), _mm_and_si128(_mm_xor_si128(y, x), _mm_set1_epi8(1)))

    // if (ABS(p0-q0) - alpha) ...
    zeromask = _mm_subs_epu8(subabs128(p0, q0), _mm_set1_epi8((int8_t)(alpha - 1)));
    //  & (ABS(p1-p0) - beta) & (ABS(q1-q0) - beta)
    _beta = _mm_set1_epi8((int8_t)(beta - 1));
    zeromask = _mm_or_si128(zeromask, _mm_subs_epu8(_mm_max_epu8(subabs128(p1, p0), subabs128(q1, q0)), _beta));
    zeromask = _mm_cmpeq_epi8(zeromask, _mm_setzero_si128());

    {
        __m128i str_x = _mm_cvtsi32_si128(strength);
        str_x = _mm_unpacklo_epi8(str_x, str_x);
        str_x = _mm_cmpgt_epi8(_mm_unpacklo_epi8(str_x, str_x), _mm_setzero_si128());
        zeromask = _mm_and_si128(zeromask, str_x);
    }

    thr = _mm_cvtsi32_si128(*(int*)threshold);//_mm_loadl_epi64((__m128i *)(threshold));
    thr = _mm_unpacklo_epi8(thr, thr);
    thr = _mm_unpacklo_epi8(thr, thr);
    thr = _mm_and_si128(thr, zeromask);

    p2 = _mm_loadu_si128((__m128i *)(pix - 3*stride));
    maskp = CMP_BETA(p2, p0, _beta);
    tc0tmp = _mm_and_si128(thr, maskp);
    p0q0avg = _mm_avg_epu8(p0, q0);     // (p0+q0+1)>>1
    _mm_storeu_si128((__m128i *)(pix - 2*stride), _mm_min_epu8(_mm_max_epu8(HALFSUM(p2, p0q0avg), _mm_subs_epu8(p1, tc0tmp)), _mm_adds_epu8(p1, tc0tmp)));

    q2 = _mm_loadu_si128((__m128i *)(pix + 2*stride));
    maskq = CMP_BETA(q2, q0, _beta);
    tc0tmp = _mm_and_si128(thr, maskq);
    _mm_storeu_si128((__m128i *)(pix + stride),  _mm_min_epu8(_mm_max_epu8(HALFSUM(q2, p0q0avg), _mm_subs_epu8(q1, tc0tmp)), _mm_adds_epu8(q1, tc0tmp)));

    thr = _mm_sub_epi8(thr, maskp);
    thr = _mm_sub_epi8(thr, maskq);
    thr = _mm_and_si128(thr, zeromask);

    {
    __m128i ff = _mm_set1_epi8(0xff);
    __m128i part1 = _mm_avg_epu8(q0, _mm_xor_si128(p0, ff));
    __m128i part2 = _mm_avg_epu8(p1, _mm_xor_si128(q1, ff));
    __m128i carry = _mm_and_si128(_mm_xor_si128(p0, q0), _mm_set1_epi8(1));
    __m128i d = _mm_adds_epu8(part1, _mm_avg_epu8(_mm_avg_epu8(part2, _mm_set1_epi8(3)), carry));
    __m128i delta_p = _mm_subs_epu8(d, _mm_set1_epi8((char)(128 + 33)));
    __m128i delta_n = _mm_subs_epu8(_mm_set1_epi8((char)(128 + 33)), d);
    delta_p = _mm_min_epu8(delta_p, thr);
    delta_n = _mm_min_epu8(delta_n, thr);

    q0 =  _mm_adds_epu8(_mm_subs_epu8(q0, delta_p), delta_n);
    p0 =  _mm_subs_epu8(_mm_adds_epu8(p0, delta_p), delta_n);

    _mm_storeu_si128 ((__m128i *)(pix - stride), p0);
    _mm_storeu_si128 ((__m128i *)pix,            q0);
    }
}

static void deblock_luma_v_s3_sse(uint8_t *pix, int stride, int alpha, int beta, const void* thr, uint32_t strength)
{
    __m128i scratch[8];
    uint8_t *s = pix - 4;
    uint8_t *dst = (uint8_t *)scratch;
    int cloop = 2;
    do
    {
        transpose8x8_sse(dst, 16, s, stride);
        s += 8*stride;
        dst += 8;
    } while(--cloop);

    deblock_luma_h_s3_sse((uint8_t*)(scratch + 4), 16, alpha, beta, thr, strength);
    s = pix - 4;
    dst = (uint8_t *)scratch;
    cloop = 2;
    do
    {
        transpose8x8_sse(s, stride, dst, 16);
        s += 8*stride;
        dst += 8;
    } while(--cloop);
}

static void h264e_deblock_chroma_sse2(uint8_t *pix, int32_t stride, const deblock_params_t *par)
{
    const uint8_t *alpha = par->alpha;
    const uint8_t *beta = par->beta;
    const uint8_t *thr = par->tc0;
    const uint8_t *strength = (uint8_t *)par->strength32;
    int a, b, x, y;
    a = alpha[0];
    b = beta[0];
    for (x = 0; x < 16; x += 8)
    {
        uint32_t str = *(uint32_t*)&strength[x];
        if (str && a)
        {
            deblock_chroma_v_s4_sse(pix + (x >> 1), stride, thr + x, a, b, str);
        }
        a = alpha[1];
        b = beta[1];
    }
    thr += 16;
    strength += 16;
    a = alpha[2];
    b = beta[2];
    for (y = 0; y < 16; y += 8)
    {
        uint32_t str = *(uint32_t*)&strength[y];
        if (str && a)
        {
            deblock_chroma_h_s4_sse(pix, stride, thr + y, a, b, str);
        }
        pix += 4*stride;
        a = alpha[3];
        b = beta[3];
    }
}

static void h264e_deblock_luma_sse2(uint8_t *pix, int32_t stride, const deblock_params_t *par)
{
    const uint8_t *alpha = par->alpha;
    const uint8_t *beta = par->beta;
    const uint8_t *thr = par->tc0;
    const uint8_t *strength = (uint8_t *)par->strength32;
    int a, b, x, y;
    a = alpha[0];
    b = beta[0];
    for (x = 0; x < 16; x += 4)
    {
        uint32_t str = *(uint32_t*)&strength[x];
        if ((uint8_t)str == 4)
        {
            deblock_luma_v_s4_sse(pix + x, stride, a, b);
        } else if (str && a)
        {
            deblock_luma_v_s3_sse(pix + x, stride, a, b, thr + x, str);
        }
        a = alpha[1];
        b = beta[1];
    }
    thr += 16;
    strength += 16;
    a = alpha[2];
    b = beta[2];
    for (y = 0; y < 16; y += 4)
    {
        uint32_t str = *(uint32_t*)&strength[y];
        if ((uint8_t)str == 4)
        {
            deblock_luma_h_s4_sse(pix, stride, a, b);
        } else if (str && a)
        {
            deblock_luma_h_s3_sse(pix, stride, a, b, thr + y, str);
        }
        a = alpha[3];
        b = beta[3];
        pix += 4*stride;
    }
}

static void h264e_denoise_run_sse2(unsigned char *frm, unsigned char *frmprev, int w, int h_arg, int stride_frm, int stride_frmprev)
{
#define MM_LOAD_8TO16(p) _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(p)), zero)
    int cloop, h = h_arg;
    __m128i zero = _mm_setzero_si128();
    __m128i exp  = _mm_set1_epi32(0x7F800000);

    w -= 2;
    h -= 2;
    if (w <= 2 || h <= 2)
    {
        return;
    }

    do
    {
        unsigned char *pf = frm += stride_frm;
        unsigned char *pp = frmprev += stride_frmprev;
        cloop = w >> 3;
        pp[-stride_frmprev] = *pf++;
        pp++;

        while (cloop--)
        {
            __m128 float_val;
            __m128i log_neighbour, log_d;
            __m128i log_neighbour_h, log_neighbour_l, log_d_h, log_d_l;
            __m128i a, b;
            __m128i gain;
            __m128i abs_d, abs_neighbour;
            a = MM_LOAD_8TO16(pf);
            b = MM_LOAD_8TO16(pp);
            abs_d   = _mm_or_si128(_mm_subs_epu16(a, b), _mm_subs_epu16(b, a));
            a = MM_LOAD_8TO16(pf-stride_frm);
            a = _mm_add_epi16(a, MM_LOAD_8TO16(pf - 1));
            a = _mm_add_epi16(a, MM_LOAD_8TO16(pf + 1));
            a = _mm_add_epi16(a, MM_LOAD_8TO16(pf + stride_frm));
            b = MM_LOAD_8TO16(pp-stride_frmprev);
            b = _mm_add_epi16(b, MM_LOAD_8TO16(pp - 1));
            b = _mm_add_epi16(b, MM_LOAD_8TO16(pp + 1));
            b = _mm_add_epi16(b, MM_LOAD_8TO16(pp + stride_frmprev));

            abs_neighbour = _mm_or_si128(_mm_subs_epu16(a, b), _mm_subs_epu16(b, a));

            abs_neighbour = _mm_srai_epi16(abs_neighbour, 2);

            abs_d = _mm_add_epi16(abs_d, _mm_set1_epi16(1));
            abs_neighbour = _mm_add_epi16(abs_neighbour, _mm_set1_epi16(1));

            float_val = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpacklo_epi16(abs_neighbour, zero), 16), 16));
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            log_neighbour_l  = _mm_sub_epi32(_mm_srli_epi32(_mm_and_si128(_mm_castps_si128(float_val), exp), 23), _mm_set1_epi32(127));

            float_val = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpackhi_epi16(abs_neighbour, zero), 16), 16));
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            log_neighbour_h  = _mm_sub_epi32(_mm_srli_epi32(_mm_and_si128(_mm_castps_si128(float_val), exp), 23), _mm_set1_epi32(127));

            float_val = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpacklo_epi16(abs_d, zero), 16), 16));
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            log_d_l = _mm_sub_epi32(_mm_srli_epi32(_mm_and_si128(_mm_castps_si128(float_val), exp), 23), _mm_set1_epi32(127));

            float_val = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpackhi_epi16(abs_d, zero), 16), 16));
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            float_val = _mm_mul_ps(float_val, float_val);
            log_d_h = _mm_sub_epi32(_mm_srli_epi32(_mm_and_si128(_mm_castps_si128(float_val), exp), 23), _mm_set1_epi32(127));

            log_d = _mm_packs_epi32(log_d_l, log_d_h);
            log_neighbour = _mm_packs_epi32(log_neighbour_l, log_neighbour_h);

            log_neighbour = _mm_slli_epi16(log_neighbour, 8);
            log_neighbour = _mm_adds_epu16(log_neighbour, log_neighbour);
            log_neighbour = _mm_adds_epu16(log_neighbour, log_neighbour);
            log_neighbour = _mm_srli_epi16(log_neighbour, 8);

            log_neighbour = _mm_subs_epu16(_mm_set1_epi16(255), log_neighbour);
            log_d = _mm_subs_epu16(_mm_set1_epi16(255), log_d);

            gain = _mm_mullo_epi16(log_d, log_neighbour);

            a = MM_LOAD_8TO16(pf);
            b = MM_LOAD_8TO16(pp);
{
            __m128i s;
            __m128i gain_inv;
            gain_inv = _mm_sub_epi16(_mm_set1_epi8((char)255), gain);
            s = _mm_add_epi16(_mm_mulhi_epu16(a, gain_inv), _mm_mulhi_epu16(b, gain));
            b = _mm_mullo_epi16(b, gain);
            a = _mm_mullo_epi16(a, gain_inv);
            a = _mm_sub_epi16(_mm_avg_epu16(a, b), _mm_and_si128(_mm_xor_si128(a, b), _mm_set1_epi16(1)));
            a = _mm_avg_epu16(_mm_srli_epi16(a, 14), _mm_set1_epi16(0));
            a = _mm_add_epi16(a, s);
            _mm_storel_epi64((__m128i *)(pp-stride_frmprev), _mm_packus_epi16(a,zero));
}
            pf += 8;
            pp += 8;
        }

        cloop = w & 7;
        while (cloop--)
        {
            int d, neighbourhood;
            unsigned g, gd, gn, out_val;
            d = pf[0] - pp[0];
            neighbourhood  = pf[-1]      - pp[-1];
            neighbourhood += pf[+1]      - pp[+1];
            neighbourhood += pf[-stride_frm] - pp[-stride_frmprev];
            neighbourhood += pf[+stride_frm] - pp[+stride_frmprev];

            if (d < 0)
            {
                d = -d;
            }
            if (neighbourhood < 0)
            {
                neighbourhood = -neighbourhood;
            }
            neighbourhood >>= 2;

            gd = g_diff_to_gainQ8[d];
            gn = g_diff_to_gainQ8[neighbourhood];

            gn <<= 2;
            if (gn > 255)
            {
                gn = 255;
            }

            gn = 255 - gn;
            gd = 255 - gd;
            g = gn*gd;  // Q8*Q8 = Q16;

            //out_val = ((pp[0]*g ) >> 16) + (((0xffff-g)*pf[0] ) >> 16);
            out_val = (pp[0]*g + (0xffff-g)*pf[0]  + (1<<15)) >> 16;

            assert(out_val <= 255);

            pp[-stride_frmprev] = (unsigned char)out_val;

            pf++, pp++;
        }
        pp[-stride_frmprev] = *pf++;
    } while(--h);

    memcpy(frmprev + stride_frmprev, frm + stride_frm, w+2);
    h = h_arg - 2;
    do
    {
        memcpy(frmprev, frmprev - stride_frmprev, w+2);
        frmprev -= stride_frmprev;
    } while(--h);
    memcpy(frmprev, frm - stride_frm*(h_arg-2), w+2);
}

#define IS_NULL(p) ((p) < (pix_t *)(uintptr_t)32)

static uint32_t intra_predict_dc_sse(const pix_t *left, const pix_t *top, int log_side)
{
    unsigned dc = 0, side = 1u << log_side, round = 0;
    __m128i sum = _mm_setzero_si128();
    if (!IS_NULL(left))
    {
        int cloop = side;
        round += side >> 1;
        do
        {
            sum = _mm_add_epi64(sum, _mm_sad_epu8(_mm_cvtsi32_si128(*(int*)left), _mm_setzero_si128()));
            left += 4;
        } while (cloop -= 4);
    }
    if (!IS_NULL(top))
    {
        int cloop = side;
        round += side >> 1;
        do
        {
            sum = _mm_add_epi64(sum, _mm_sad_epu8(_mm_cvtsi32_si128(*(int*)top), _mm_setzero_si128()));
            top += 4;
        } while (cloop -= 4);
    }
    dc = _mm_cvtsi128_si32(sum);
    dc += round;
    if (round == side) dc >>= 1;
    dc >>= log_side;
    if (!round) dc = 128;
    return dc * 0x01010101;
}

/*
 * Note: To make the code more readable we refer to the neighboring pixels
 * in variables named as below:
 *
 *    UL U0 U1 U2 U3 U4 U5 U6 U7
 *    L0 xx xx xx xx
 *    L1 xx xx xx xx
 *    L2 xx xx xx xx
 *    L3 xx xx xx xx
 */
#define UL edge[-1]
#define U0 edge[0]
#define T1 edge[1]
#define U2 edge[2]
#define U3 edge[3]
#define U4 edge[4]
#define U5 edge[5]
#define U6 edge[6]
#define U7 edge[7]
#define L0 edge[-2]
#define L1 edge[-3]
#define L2 edge[-4]
#define L3 edge[-5]

static void h264e_intra_predict_16x16_sse2(pix_t *predict,  const pix_t *left, const pix_t *top, int mode)
{
    int cloop = 16;
    if (mode < 1)
    {
        __m128i a = _mm_load_si128((__m128i *)top);
        do
        {
            _mm_store_si128((__m128i *)predict, a);
            predict += 16;
        } while(--cloop);
    } else if (mode == 1)
    {
        const __m128i c1111 = _mm_set1_epi8(1);
        do
        {
            _mm_store_si128((__m128i *)predict, _mm_shuffle_epi32(_mm_mul_epu32(_mm_cvtsi32_si128(*left++), c1111), 0));
            predict += 16;
        } while(--cloop);
    } else //if (mode == 2)
    {
        __m128i dc128;
        int dc = intra_predict_dc_sse(left, top, 4);
        dc128 = _mm_shuffle_epi32(_mm_cvtsi32_si128(dc), 0);
        do
        {
            _mm_store_si128((__m128i *)predict, dc128);
            predict += 16;
        } while(--cloop);
    }
}

static void h264e_intra_predict_chroma_sse2(pix_t *predict, const pix_t *left, const pix_t *top, int mode)
{
    int cloop = 8;
    if (mode < 1)
    {
        __m128i a = _mm_load_si128((__m128i *)top);
        do
        {
            _mm_store_si128((__m128i *)predict, a);
            predict += 16;
        } while(--cloop);
    } else if (mode == 1)
    {
        do
        {
            __m128i t = _mm_unpacklo_epi32(_mm_cvtsi32_si128(left[0]*0x01010101u), _mm_cvtsi32_si128(left[8]*0x01010101u));
            t = _mm_unpacklo_epi32(t, t);
            _mm_store_si128((__m128i *)predict, t);
            left++;
            predict += 16;
        } while(--cloop);
    } else //if (mode == 2)
    {
        // chroma
        uint32_t *d = (uint32_t*)predict;
        __m128i *d128 = (__m128i *)predict;
        __m128i tmp;
        cloop = 2;
        do
        {
            d[0] = d[1] = d[16] = intra_predict_dc_sse(left, top, 2);
            d[17] = intra_predict_dc_sse(left + 4, top + 4, 2);
            if (!IS_NULL(top))
            {
                d[1] = intra_predict_dc_sse(NULL, top + 4, 2);
            }
            if (!IS_NULL(left))
            {
                d[16] = intra_predict_dc_sse(NULL, left + 4, 2);
            }
            d += 2;
            left += 8;
            top += 8;
        } while(--cloop);
        tmp = _mm_load_si128(d128++);
        _mm_store_si128(d128++, tmp);
        _mm_store_si128(d128++, tmp);
        _mm_store_si128(d128++, tmp);
        tmp = _mm_load_si128(d128++);
        _mm_store_si128(d128++, tmp);
        _mm_store_si128(d128++, tmp);
        _mm_store_si128(d128++, tmp);
    }
}

static int h264e_intra_choose_4x4_sse2(const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty)
{
    int best_m = 0;
    int sad, best_sad = 0x10000;

    __m128i b0 = _mm_loadl_epi64((__m128i *)blockin);
    __m128i b1 = _mm_loadl_epi64((__m128i *)(blockin + 16));
    __m128i b2 = _mm_loadl_epi64((__m128i *)(blockin + 32));
    __m128i b3 = _mm_loadl_epi64((__m128i *)(blockin + 48));
    __m128i c  = _mm_unpacklo_epi32(b0, b1);
    __m128i d  = _mm_unpacklo_epi32(b2, b3);
    __m128i sse_blockin = _mm_unpacklo_epi64(c, d);
    __m128i t, t0, t1, t2, res, sad128, best128;

#define TEST(mode) sad128 = _mm_sad_epu8(res, sse_blockin);                 \
            sad128 = _mm_adds_epu16 (sad128, _mm_shuffle_epi32(sad128, 2)); \
            sad  = _mm_cvtsi128_si32(sad128);                               \
            if (mode != mpred) sad += penalty;                              \
            if (sad < best_sad)                                             \
            {                                                               \
                best128 = res;                                              \
                best_sad = sad;                                             \
                best_m = mode;                                              \
            }

    __m128i border = _mm_loadu_si128((__m128i *)(&L3));
    int topright = 0x01010101u*U7;

    if (!(avail & AVAIL_TR))
    {
        topright = 0x01010101u*U3;
        //border = _mm_insert_epi32 (border, topright, 2);
        border = _mm_insert_epi16 (border, topright, 4);
        border = _mm_insert_epi16 (border, topright, 5);
    }
    //border = _mm_insert_epi32 (border, topright, 3);
    border = _mm_insert_epi16 (border, topright, 6);
    border = _mm_insert_epi16 (border, topright, 7);

    // DC
    {
        unsigned dc = 0, round = 0;

        if (avail & AVAIL_L)
        {
            dc += _mm_cvtsi128_si32(_mm_sad_epu8(_mm_and_si128(border, _mm_set_epi32(0, 0, 0, ~0)), _mm_setzero_si128()));
            round += 2;
        }
        if (avail & AVAIL_T)
        {
            dc += _mm_cvtsi128_si32(_mm_sad_epu8(_mm_and_si128(_mm_srli_si128(border, 5), _mm_set_epi32(0, 0, 0, ~0)), _mm_setzero_si128()));
            round += 2;
        }
        dc += round;
        if (round == 4) dc >>= 1;
        dc >>= 2;
        if (!round) dc = 128;
        t = _mm_cvtsi32_si128(dc * 0x01010101);
        t = _mm_unpacklo_epi32(t, t);
        best128 =_mm_unpacklo_epi32(t, t);

        //TEST(2)
        sad128 = _mm_sad_epu8(best128, sse_blockin);
        sad128 = _mm_adds_epu16 (sad128, _mm_shuffle_epi32(sad128, 2));
        best_sad = _mm_cvtsi128_si32(sad128);

        if (2 != mpred) best_sad += penalty;
        best_m = 2;
    }

    if (avail & AVAIL_T)
    {
        t = _mm_srli_si128(border, 5);
        t = _mm_unpacklo_epi32(t, t);
        res =  _mm_unpacklo_epi32(t, t);
        TEST(0)

        t0 = _mm_srli_si128(border, 5);
        t1 = _mm_srli_si128(border, 6);
        t2 = _mm_srli_si128(border, 7);
        t = _mm_sub_epi8(_mm_avg_epu8(t0, t2), _mm_and_si128(_mm_xor_si128(t0, t2), _mm_set1_epi8(1)));
        t = _mm_avg_epu8(t, t1);
        t2 = _mm_unpacklo_epi32(t, _mm_srli_si128(t, 1));

        res = _mm_unpacklo_epi64(t2, _mm_unpacklo_epi32(_mm_srli_si128(t, 2), _mm_srli_si128(t, 3)));
        TEST(3)

        t0 = _mm_avg_epu8(t0,t1);
        t0  = _mm_unpacklo_epi32(t0, _mm_srli_si128(t0, 1));
        res = _mm_unpacklo_epi32(t0, t2);
        TEST(7)
    }

    if (avail & AVAIL_L)
    {
        int ext;
        t = _mm_unpacklo_epi8(border, border);
        t = _mm_shufflelo_epi16(t, 3 + (2 << 2) + (1 << 4) + (0 << 6));
        res = _mm_unpacklo_epi8(t, t);
        TEST(1)

        t0 = _mm_unpacklo_epi8(border, _mm_setzero_si128());
        t0 = _mm_shufflelo_epi16(t0, 3 + (2 << 2) + (1 << 4) + (0 << 6));
        t0 = _mm_packus_epi16(t0, t0);       // 0 1 2 3

        t1 = _mm_unpacklo_epi8(t0, t0);      // 0 0 1 1 2 2 3 3

        ext = _mm_extract_epi16(t1, 3);
        t0 = _mm_insert_epi16 (t0, ext, 2);  // 0 1 2 3 3 3
        t1 = _mm_insert_epi16 (t1, ext, 4);  // 0 0 1 1 2 2 3 3 33
        t2 = _mm_slli_si128(t0, 2);          // x x 0 1 2 3 3 3
        t = _mm_sub_epi8(_mm_avg_epu8(t0, t2), _mm_and_si128(_mm_xor_si128(t0, t2), _mm_set1_epi8(1)));
        // 0 1 2 3 3 3
        // x x 0 1 2 3
        t = _mm_unpacklo_epi8(t2, t);
        // 0   1   2   3   3   3
        // x   x   0   1   2   3
        // x   x   0   1   2   3
        t = _mm_avg_epu8(t, _mm_slli_si128(t1, 2));
        // 0 0 1 1 2 2 3 3

        res = _mm_unpacklo_epi32(_mm_srli_si128(t, 4), _mm_srli_si128(t, 6));
        //res = _mm_insert_epi32 (res, ext|(ext<<16),3);
        res = _mm_insert_epi16 (res, ext, 6);
        res = _mm_insert_epi16 (res, ext, 7);
        TEST(8)
    }

    if ((avail & (AVAIL_T | AVAIL_L | AVAIL_TL)) == (AVAIL_T | AVAIL_L | AVAIL_TL))
    {
        int t16;
        t0 = _mm_srli_si128(border, 1);
        t1 = _mm_srli_si128(border, 2);
        t = _mm_sub_epi8(_mm_avg_epu8(border, t1), _mm_and_si128(_mm_xor_si128(border, t1), _mm_set1_epi8(1)));
        t = _mm_avg_epu8(t, t0);

        res = _mm_unpacklo_epi64(_mm_unpacklo_epi32(_mm_srli_si128(t, 3), _mm_srli_si128(t, 2)), _mm_unpacklo_epi32(_mm_srli_si128(t, 1), t));
        TEST(4)

        t1 = _mm_unpacklo_epi8(t2 = _mm_avg_epu8(t0,border), t);
        t1 = _mm_unpacklo_epi32(t1, _mm_srli_si128(t1, 2));
        res = _mm_shuffle_epi32(t1, 3 | (2 << 2) | (1 << 4) | (0 << 6));
        res = _mm_insert_epi16 (res, _mm_extract_epi16 (t, 2), 1);
        TEST(6)

        t = _mm_srli_si128(t, 1);
        res = _mm_unpacklo_epi32(_mm_srli_si128(t2, 4), _mm_srli_si128(t, 2));
        t2 =  _mm_insert_epi16 (t2, t16 = _mm_extract_epi16 (t, 0), 1);
        t  =  _mm_insert_epi16 (t, (t16 << 8), 0);
        res = _mm_unpacklo_epi64(res, _mm_unpacklo_epi32(_mm_srli_si128(t2, 3), _mm_srli_si128(t, 1)));
        TEST(5)
    }

    ((uint32_t *)blockpred)[ 0] = _mm_extract_epi16(best128, 0) | ((unsigned)_mm_extract_epi16(best128, 1) << 16);
    ((uint32_t *)blockpred)[ 4] = _mm_extract_epi16(best128, 2) | ((unsigned)_mm_extract_epi16(best128, 3) << 16);
    ((uint32_t *)blockpred)[ 8] = _mm_extract_epi16(best128, 4) | ((unsigned)_mm_extract_epi16(best128, 5) << 16);
    ((uint32_t *)blockpred)[12] = _mm_extract_epi16(best128, 6) | ((unsigned)_mm_extract_epi16(best128, 7) << 16);

    return best_m + (best_sad << 4);    // pack result
}

#define MM_LOAD_8TO16(p) _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(p)), zero)
#define MM_LOAD_REG(p, sh) _mm_unpacklo_epi8(_mm_srli_si128(p, sh), zero)
#define __inline
static __inline void copy_wh_sse(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    assert(h % 4 == 0);
    if (w == 16)
    {
        do
        {
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
            _mm_store_si128((__m128i *)dst, _mm_loadu_si128((__m128i *)src)); src += src_stride; dst += 16;
        } while(h -= 8);
    } else //if (w == 8)
    {
        do
        {
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
        } while(h -= 4);
    }
}

static __inline void hpel_lpf_diag_sse(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    ALIGN(16) int16_t scratch[21 * 16] ALIGN2(16);  /* 21 rows by 16 pixels per row */

    /*
     * Intermediate values will be 1/2 pel at Horizontal direction
     * Starting at (0.5, -2) at top extending to (0.5, height + 3) at bottom
     * scratch contains a 2D array of size (w)X(h + 5)
     */
    __m128i zero = _mm_setzero_si128();
    __m128i c32,c5 = _mm_set1_epi16(5);
    int cloop = h + 5;
    int16_t *h264e_restrict dst16 = scratch;
    const int16_t *src16 = scratch + 2*16;
    src -= 2*src_stride;
    if (w == 8)
    {
        src16 = scratch + 2*8;
        do
        {
            __m128i inp = _mm_loadu_si128((__m128i*)(src - 2));
            _mm_store_si128((__m128i*)dst16, _mm_add_epi16(
                _mm_mullo_epi16(
                    _mm_sub_epi16(
                        _mm_slli_epi16(
                            _mm_add_epi16(MM_LOAD_REG(inp, 2), MM_LOAD_REG(inp, 3)),
                            2),
                        _mm_add_epi16(MM_LOAD_REG(inp, 1), MM_LOAD_REG(inp, 4))),
                    c5),
                _mm_add_epi16(_mm_unpacklo_epi8(inp, zero), MM_LOAD_REG(inp, 5))
            ));
            src += src_stride;
            dst16 += 8;
        } while (--cloop);

        c32 = _mm_set1_epi16(32);
        cloop = h;
        do
        {
            // (20*x2 - 5*x1 + x0 + 512) >> 10 =>
            // (16*x2 + 4*x2 - 4*x1 - x1 + x0 + 512) >> 10 =>
            // ((((x0 - x1) >> 2) + (x2 - x1)) >> 2) + x2 + 32 >> 6
            __m128i x1 = _mm_add_epi16(_mm_load_si128((__m128i*)(src16 - 1*8)), _mm_load_si128((__m128i*)(src16 + 2*8)));
            __m128i x2 = _mm_add_epi16(_mm_load_si128((__m128i*)(src16 - 0*8)), _mm_load_si128((__m128i*)(src16 + 1*8)));
            _mm_storel_epi64((__m128i*)dst,
                _mm_packus_epi16(
                    _mm_srai_epi16(
                        _mm_add_epi16(
                            _mm_srai_epi16(
                                _mm_sub_epi16(
                                    _mm_srai_epi16(
                                        _mm_sub_epi16(
                                            _mm_add_epi16(_mm_load_si128((__m128i*)(src16 - 2*8)), _mm_load_si128((__m128i*)(src16 + 3*8))),
                                            x1),
                                        2),
                                    _mm_sub_epi16(x1, x2)),
                                2),
                            _mm_add_epi16(x2, c32)),
                        6),
                    zero));
            src16 += 8;
            dst += 16;
        } while(--cloop);
    } else
    {
        do
        {
            _mm_store_si128((__m128i*)dst16, _mm_add_epi16(
                _mm_mullo_epi16(
                    _mm_sub_epi16(
                        _mm_slli_epi16(
                            _mm_add_epi16(MM_LOAD_8TO16(src - 0), MM_LOAD_8TO16(src + 1)),
                            2),
                        _mm_add_epi16(MM_LOAD_8TO16(src - 1), MM_LOAD_8TO16(src + 2))),
                    c5),
                _mm_add_epi16(MM_LOAD_8TO16(src - 2), MM_LOAD_8TO16(src + 3))
            ));
            _mm_store_si128((__m128i*)(dst16 + 8), _mm_add_epi16(
                _mm_mullo_epi16(
                    _mm_sub_epi16(
                        _mm_slli_epi16(
                            _mm_add_epi16(MM_LOAD_8TO16(src + 8 - 0), MM_LOAD_8TO16(src + 8 + 1)),
                            2),
                        _mm_add_epi16(MM_LOAD_8TO16(src + 8 - 1), MM_LOAD_8TO16(src + 8 + 2))),
                    c5),
                _mm_add_epi16(MM_LOAD_8TO16(src + 8 - 2), MM_LOAD_8TO16(src + 8 + 3))
            ));
            src += src_stride;
            dst16 += 8*2;
        } while (--cloop);

        c32 = _mm_set1_epi16(32);
        cloop = 2*h;
        do
        {
            // (20*x2 - 5*x1 + x0 + 512) >> 10 =>
            // (16*x2 + 4*x2 - 4*x1 - x1 + x0 + 512) >> 10 =>
            // ((((x0 - x1) >> 2) + (x2 - x1)) >> 2) + x2 + 32 >> 6
            __m128i x1 = _mm_add_epi16(_mm_load_si128((__m128i*)(src16 - 1*16)), _mm_load_si128((__m128i*)(src16 + 2*16)));
            __m128i x2 = _mm_add_epi16(_mm_load_si128((__m128i*)(src16 - 0*16)), _mm_load_si128((__m128i*)(src16 + 1*16)));
            _mm_storel_epi64((__m128i*)dst,
                _mm_packus_epi16(
                    _mm_srai_epi16(
                        _mm_add_epi16(
                            _mm_srai_epi16(
                                _mm_sub_epi16(
                                    _mm_srai_epi16(
                                        _mm_sub_epi16(
                                            _mm_add_epi16(_mm_load_si128((__m128i*)(src16 - 2*16)), _mm_load_si128((__m128i*)(src16 + 3*16))),
                                            x1),
                                        2),
                                    _mm_sub_epi16(x1, x2)),
                                2),
                            _mm_add_epi16(x2, c32)),
                        6),
                    zero));
            src16 += 8;
            dst += 8;
        } while(--cloop);
    }
}

static __inline void hpel_lpf_hor_sse(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    __m128i zero = _mm_setzero_si128();
    const __m128i five = _mm_set1_epi16(5);
    if (w == 8)
    {
        do
        {
            __m128i inp = _mm_loadu_si128((__m128i*)(src - 2));
            _mm_storel_epi64((__m128i*)dst, _mm_packus_epi16(
                _mm_srai_epi16(
                    _mm_add_epi16(
                        _mm_add_epi16(
                            _mm_mullo_epi16(
                                _mm_sub_epi16(
                                    _mm_slli_epi16(_mm_add_epi16(MM_LOAD_REG(inp, 2), MM_LOAD_REG(inp, 3)), 2),
                                    _mm_add_epi16(MM_LOAD_REG(inp, 1), MM_LOAD_REG(inp, 4))),
                                 five),
                            _mm_add_epi16(_mm_unpacklo_epi8(inp, zero), MM_LOAD_REG(inp, 5))),
                        _mm_set1_epi16(16)),
                    5),
                zero));
            src += src_stride;
            dst += 16;
        } while (--h);
    } else do
    {
        __m128i inp = _mm_loadu_si128((__m128i*)(src - 2));
        _mm_storel_epi64((__m128i*)dst, _mm_packus_epi16(
            _mm_srai_epi16(
                _mm_add_epi16(
                    _mm_add_epi16(
                        _mm_mullo_epi16(
                            _mm_sub_epi16(
                                _mm_slli_epi16(_mm_add_epi16(MM_LOAD_REG(inp, 2), MM_LOAD_REG(inp, 3)), 2),
                                _mm_add_epi16(MM_LOAD_REG(inp, 1), MM_LOAD_REG(inp, 4))),
                             five),
                        _mm_add_epi16(_mm_unpacklo_epi8(inp, zero), MM_LOAD_REG(inp, 5))),
                    _mm_set1_epi16(16)),
                5),
            zero));
        inp = _mm_loadu_si128((__m128i*)(src + 8 - 2));
        _mm_storel_epi64((__m128i*)(dst + 8), _mm_packus_epi16(
            _mm_srai_epi16(
                _mm_add_epi16(
                    _mm_add_epi16(
                        _mm_mullo_epi16(
                            _mm_sub_epi16(
                                _mm_slli_epi16(_mm_add_epi16(MM_LOAD_REG(inp, 2), MM_LOAD_REG(inp, 3)), 2),
                                _mm_add_epi16(MM_LOAD_REG(inp, 1), MM_LOAD_REG(inp, 4))),
                             five),
                        _mm_add_epi16(_mm_unpacklo_epi8(inp, zero), MM_LOAD_REG(inp, 5))),
                    _mm_set1_epi16(16)),
                5),
            zero));
        src += src_stride;
        dst += 16;
    } while (--h);
}

static __inline void hpel_lpf_ver_sse(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    __m128i zero = _mm_setzero_si128();
    __m128i five = _mm_set1_epi16(5);
    __m128i const16 = _mm_set1_epi16(16);

    do
    {
        int cloop = h;
        do
        {
            _mm_storel_epi64((__m128i*)dst, _mm_packus_epi16(
                _mm_srai_epi16(
                    _mm_add_epi16(
                        _mm_add_epi16(
                            _mm_mullo_epi16(
                                _mm_sub_epi16(
                                     _mm_slli_epi16(_mm_add_epi16(MM_LOAD_8TO16(src - 0*src_stride), MM_LOAD_8TO16(src + 1*src_stride)), 2),
                                    _mm_add_epi16(MM_LOAD_8TO16(src - 1*src_stride), MM_LOAD_8TO16(src + 2*src_stride))),
                                five),
                            _mm_add_epi16(MM_LOAD_8TO16(src - 2*src_stride), MM_LOAD_8TO16(src + 3*src_stride))),
                        const16),
                    5),
                zero));
            src += src_stride;
            dst += 16;
        } while(--cloop);
        src += 8 - src_stride*h;
        dst += 8 - 16*h;
    } while ((w -= 8) > 0);
}

static void average_16x16_unalign_sse(uint8_t *dst, const uint8_t *src, int src_stride)
{
    __m128i *d = (__m128i *)dst;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
    _mm_store_si128(d, _mm_avg_epu8(_mm_load_si128(d), _mm_loadu_si128((__m128i *)src))); src += src_stride; d++;
}

static void h264e_qpel_average_wh_align_sse2(const uint8_t *src0, const uint8_t *src1, uint8_t *h264e_restrict dst, point_t wh)
{
    int w = wh.s.x;
    int h = wh.s.y;
    __m128i *d = (__m128i *)dst;
    const __m128i *s0 = (const __m128i *)src0;
    const __m128i *s1 = (const __m128i *)src1;
    if (w == 16)
    {
        do
        {
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
            _mm_store_si128(d++, _mm_avg_epu8(_mm_load_si128(s0++), _mm_load_si128(s1++)));
        } while((h -= 8) > 0);
    } else
    {
        do
        {
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
            _mm_storel_epi64(d++, _mm_avg_epu8(_mm_loadl_epi64(s0++), _mm_loadl_epi64(s1++)));
        } while((h -= 8) > 0);
    }
}

static void h264e_qpel_interpolate_luma_sse2(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, point_t wh, point_t dxdy)
{
    ALIGN(16) uint8_t scratch[16*16] ALIGN2(16);
//    src += ((dx + 1) >> 2) + ((dy + 1) >> 2)*src_stride;            // dx == 3 ? next row; dy == 3 ? next line
//    dxdy              actions: Horizontal, Vertical, Diagonal, Average
//    0 1 2 3 +1        -   ha    h    ha+
//    1                 va  hva   hda  hv+a
//    2                 v   vda   d    v+da
//    3                 va+ h+va h+da  h+v+a
//    +stride
    int32_t pos = 1 << (dxdy.s.x + 4*dxdy.s.y);
    uint8_t *h264e_restrict dst0 = dst;

    if (pos == 1)
    {
        copy_wh_sse(src, src_stride, dst, wh.s.x, wh.s.y);
        return;
    }
    if (pos & 0xe0ee) // 1110 0000 1110 1110
    {
        hpel_lpf_hor_sse(src + ((dxdy.s.y + 1) >> 2)*src_stride, src_stride, dst, wh.s.x, wh.s.y);
        dst = scratch;
    }
    if (pos & 0xbbb0) // 1011 1011 1011 0000
    {
        hpel_lpf_ver_sse(src + ((dxdy.s.x + 1) >> 2), src_stride, dst, wh.s.x, wh.s.y);
        dst = scratch;
    }
    if (pos & 0x4e40) // 0100 1110 0100 0000
    {
        hpel_lpf_diag_sse(src, src_stride, dst, wh.s.x, wh.s.y);
        dst = scratch;
    }
    if (pos & 0xfafa) // 1111 1010 1111 1010
    {
        assert(wh.s.x == 16 && wh.s.y == 16);
        if (pos & 0xeae0)// 1110 1010 1110 0000
        {
            point_t p;
            p.u32 = 16 + (16 << 16);
            h264e_qpel_average_wh_align_sse2(scratch, dst0, dst0, p);
        } else
        {
            src += ((dxdy.s.x + 1) >> 2) + ((dxdy.s.y + 1) >> 2)*src_stride;
            average_16x16_unalign_sse(dst0, src, src_stride);
        }
    }
}

static void h264e_qpel_interpolate_chroma_sse2(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, point_t wh, point_t dxdy)
{
    __m128i zero = _mm_setzero_si128();
    int w = wh.s.x;
    int h = wh.s.y;
    __m128i a, b, c, d;

//        __m128i a = _mm_set1_epi16((short)((8-dx) * (8-dy)));
//        __m128i b = _mm_set1_epi16((short)(dx * (8-dy)));
//        __m128i c = _mm_set1_epi16((short)((8-dx) * dy));
//        __m128i d = _mm_set1_epi16((short)(dx * dy));
    __m128i c8 = _mm_set1_epi16(8);
    __m128i y,x = _mm_cvtsi32_si128(dxdy.u32);
    x = _mm_unpacklo_epi16(x, x);
    x = _mm_unpacklo_epi32(x, x);
    y = _mm_unpackhi_epi64(x, x);
    x = _mm_unpacklo_epi64(x, x);
    a = _mm_mullo_epi16(_mm_sub_epi16(c8, x), _mm_sub_epi16(c8, y));
    b = _mm_mullo_epi16(x, _mm_sub_epi16(c8, y));
    c = _mm_mullo_epi16(_mm_sub_epi16(c8, x), y);
    d = _mm_mullo_epi16(x, y);

    if (!dxdy.u32)
    {
        // 10%
        if (w == 8) do
        {
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
            _mm_storel_epi64((__m128i *)dst, _mm_loadl_epi64((__m128i *)src)); src += src_stride; dst += 16;
        } while(h -= 4);
        else
        {
            do
            {
                *(int *)dst = *(int_u *)src; src += src_stride; dst += 16;
                *(int *)dst = *(int_u *)src; src += src_stride; dst += 16;
                *(int *)dst = *(int_u *)src; src += src_stride; dst += 16;
                *(int *)dst = *(int_u *)src; src += src_stride; dst += 16;
            } while(h -= 4);
        }
    } else
    if (!dxdy.s.x || !dxdy.s.y)
    {
        // 40%
        int dsrc = dxdy.s.x?1:src_stride;
        c = _mm_or_si128(c,b);

        if (w==8)
        {
            do
            {
                _mm_storel_epi64((__m128i *)dst,
                _mm_packus_epi16(
                    _mm_srai_epi16(
                        _mm_add_epi16(
                            _mm_add_epi16(
                                    _mm_mullo_epi16(a, MM_LOAD_8TO16(src)),
                                    _mm_mullo_epi16(c, MM_LOAD_8TO16(src + dsrc))),
                            _mm_set1_epi16(32)),
                        6),
                    zero)) ;
                dst += 16;
                src += src_stride;
            } while (--h);
        } else
        {
            do
            {
                *(int* )(dst) = _mm_cvtsi128_si32 (
                _mm_packus_epi16(
                    _mm_srai_epi16(
                        _mm_add_epi16(
                            _mm_add_epi16(
                                    _mm_mullo_epi16(a, MM_LOAD_8TO16(src)),
                                    _mm_mullo_epi16(c, MM_LOAD_8TO16(src + dsrc))),
                            _mm_set1_epi16(32)),
                        6),
                    zero));
                dst += 16;
                src += src_stride;
            } while (--h);
        }
    } else
    {
        // 50%
        if (w == 8)
        {
            __m128i x1,x0;
            x0 = _mm_loadl_epi64((__m128i*)(src));
            x1 = _mm_loadl_epi64((__m128i*)(src + 1));
            x0 = _mm_unpacklo_epi8(x0, zero);
            x1 = _mm_unpacklo_epi8(x1, zero);
            do
            {
                __m128i y0, y1;
                src += src_stride;
                y0 = _mm_loadl_epi64((__m128i*)(src));
                y1 = _mm_loadl_epi64((__m128i*)(src + 1));
                y0 = _mm_unpacklo_epi8(y0, zero);
                y1 = _mm_unpacklo_epi8(y1, zero);
                _mm_storel_epi64((__m128i *)dst,
                    _mm_packus_epi16(
                        _mm_srai_epi16(
                            _mm_add_epi16(
                                _mm_add_epi16(
                                    _mm_add_epi16(
                                        _mm_mullo_epi16(x0, a),
                                        _mm_mullo_epi16(x1, b)),
                                    _mm_add_epi16(
                                        _mm_mullo_epi16(y0, c),
                                        _mm_mullo_epi16(y1, d))),
                                _mm_set1_epi16(32)),
                            6),
                        zero));
                x0 = y0;
                x1 = y1;
                dst += 16;
            } while (--h);
        } else
        {
            // TODO: load 32!
            __m128i x1, x0 = MM_LOAD_8TO16(src);
            do
            {
                src += src_stride;
                x1 = MM_LOAD_8TO16(src);
                *(int*)(dst) = _mm_cvtsi128_si32(
                    _mm_packus_epi16(
                        _mm_srai_epi16(
                            _mm_add_epi16(
                                _mm_add_epi16(
                                    _mm_add_epi16(
                                        _mm_mullo_epi16(x0, a),
                                        _mm_mullo_epi16(_mm_srli_si128(x0, 2), b)),
                                    _mm_add_epi16(
                                        _mm_mullo_epi16(x1, c),
                                        _mm_mullo_epi16(_mm_srli_si128(x1, 2), d))),
                                _mm_set1_epi16(32)),
                            6),
                        zero));
                x0 = x1;
                dst += 16;
            } while (--h);
        }
    }
}

static int h264e_sad_mb_unlaign_8x8_sse2(const pix_t *a, int a_stride, const pix_t *b, int sad[4])
{
    __m128i *mb = (__m128i *)b;
    __m128i s01, s23;
    s01 = _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++)); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s01 = _mm_add_epi64(s01, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;

    s23 = _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++)); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s23 = _mm_add_epi64(s23, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;

    sad[0] = _mm_cvtsi128_si32(s01);
    sad[1] = _mm_extract_epi16(s01, 4);
    sad[2] = _mm_cvtsi128_si32(s23);
    sad[3] = _mm_extract_epi16(s23, 4);
    return sad[0] + sad[1] + sad[2] + sad[3];
}


static int h264e_sad_mb_unlaign_wh_sse2(const pix_t *a, int a_stride, const pix_t *b, point_t wh)
{
    __m128i *mb = (__m128i *)b;
    __m128i s;

    assert(wh.s.x == 8 || wh.s.x == 16);
    assert(wh.s.y == 8 || wh.s.y == 16);

    if (wh.s.x == 8)
    {
        s =                  _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++));  a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;

        if (wh.s.y == 16)
        {
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
            s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadl_epi64((__m128i *)a), _mm_loadl_epi64(mb++))); a += a_stride;
        }
        return _mm_extract_epi16 (s, 0);
    }

    s =                  _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++));  a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;

    if (wh.s.y == 16)
    {
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
        s = _mm_add_epi16(s, _mm_sad_epu8(_mm_loadu_si128((__m128i *)a), _mm_loadu_si128(mb++))); a += a_stride;
    }

    s = _mm_adds_epu16(s, _mm_shuffle_epi32(s, 2));
    return _mm_cvtsi128_si32(s);
}

static void h264e_copy_8x8_sse2(pix_t *d, int d_stride, const pix_t *s)
{
    assert(IS_ALIGNED(d, 8));
    assert(IS_ALIGNED(s, 8));
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s))); s += 16; d += d_stride;
    _mm_storel_epi64((__m128i*)(d), _mm_loadl_epi64((__m128i*)(s)));
}

static void h264e_copy_16x16_sse2(pix_t *d, int d_stride, const pix_t *s, int s_stride)
{
    assert(IS_ALIGNED(d, 8));
    assert(IS_ALIGNED(s, 8));
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s))); s += s_stride; d += d_stride;
    _mm_storeu_si128((__m128i*)(d), _mm_loadu_si128((__m128i*)(s)));
}

static void h264e_copy_borders_sse2(unsigned char *pic, int w, int h, int guard)
{
    int rowbytes = w + 2*guard;
    int topbot = 2;
    pix_t *s = pic;
    pix_t *d = pic - guard*rowbytes;
    assert(guard == 8 || guard == 16);
    assert((w % 8) == 0);
    do
    {
        int cloop = w;
        do
        {
            __m128i t = _mm_loadu_si128((__m128i*)(s));
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            if (guard == 16)
            {
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
                _mm_storeu_si128((__m128i*)d, t); d += rowbytes;
            }
            s += 16;
            d += 16 - guard*rowbytes;
        } while((cloop -= 16) > 0);
        s = pic + (h - 1)*rowbytes;
        d = s + rowbytes;
    } while(--topbot);

    {
        pix_t *s0 = pic - guard*rowbytes;
        pix_t *s1 = pic - guard*rowbytes + w - 1;
        int cloop = 2*guard + h;
        if (guard == 8) do
        {
            _mm_storel_epi64((__m128i*)(s0-8), _mm_set1_epi8(*s0));
            _mm_storel_epi64((__m128i*)(s1+1), _mm_set1_epi8(*s1));
            s0 += rowbytes;
            s1 += rowbytes;
        } while(--cloop); else do
        {
            _mm_storeu_si128((__m128i*)(s0-16), _mm_set1_epi8(*s0));
            _mm_storeu_si128((__m128i*)(s1+1), _mm_set1_epi8(*s1));
            s0 += rowbytes;
            s1 += rowbytes;
        } while(--cloop);

    }
}

static void hadamar4_2d_sse(int16_t *x)
{
    __m128i a = _mm_loadl_epi64((__m128i*)x);
    __m128i b = _mm_loadl_epi64((__m128i*)(x + 4));
    __m128i c = _mm_loadl_epi64((__m128i*)(x + 8));
    __m128i d = _mm_loadl_epi64((__m128i*)(x + 12));

    __m128i u0 = _mm_add_epi16(a, c);
    __m128i u1 = _mm_sub_epi16(a, c);
    __m128i u2 = _mm_add_epi16(b, d);
    __m128i u3 = _mm_sub_epi16(b, d);
    __m128i v0 = _mm_add_epi16(u0, u2);
    __m128i v3 = _mm_sub_epi16(u0, u2);
    __m128i v1 = _mm_add_epi16(u1, u3);
    __m128i v2 = _mm_sub_epi16(u1, u3);

    //    v0: a0 a1 a2 a3
    //    v1: b0 ......
    //    v2: c0 ......
    //    v4: d0 d1 .. d3
    //
    __m128i t0 = _mm_unpacklo_epi16(v0, v1);    // a0, b0, a1, b1, a2, b2, a3, b3
    __m128i t2 = _mm_unpacklo_epi16(v2, v3);    // c0, d0, c1, d1, c2, d2, c3, d3
    a = _mm_unpacklo_epi32(t0, t2);    // a0, b0, c0, d0, a1, b1, c1, d1
    c = _mm_unpackhi_epi32(t0, t2);    // a2, b2, c2, d2, a3, b3, c3, d3
    u0 = _mm_add_epi16(a, c); // u0 u2
    u1 = _mm_sub_epi16(a, c); // u1 u3
    v0 = _mm_unpacklo_epi64(u0, u1); // u0 u1
    v1 = _mm_unpackhi_epi64(u0, u1); // u2 u3
    u0 = _mm_add_epi16(v0, v1); // v0 v1
    u1 = _mm_sub_epi16(v0, v1); // v3 v2

    v1 = _mm_shuffle_epi32(u1, 0x4e); // u2 u3      01001110
    _mm_store_si128((__m128i*)x, u0);
    _mm_store_si128((__m128i*)(x + 8), v1);

}

static void dequant_dc_sse(quant_t *q, int16_t *qval, int dequant, int n)
{
    do q++->dq[0] = (int16_t)(*qval++*(int16_t)dequant); while (--n);
}

static void quant_dc_sse(int16_t *qval, int16_t *deq, int16_t quant, int n, int round_q18)
{
    int r_minus = (1 << 18) - round_q18;
    do
    {
        int v = *qval;
        int r = v < 0 ? r_minus : round_q18;
        *deq++ = *qval++ = (v * quant + r) >> 18;
    } while (--n);
}

static void hadamar2_2d_sse(int16_t *x)
{
    int a = x[0];
    int b = x[1];
    int c = x[2];
    int d = x[3];
    x[0] = (int16_t)(a + b + c + d);
    x[1] = (int16_t)(a - b + c - d);
    x[2] = (int16_t)(a + b - c - d);
    x[3] = (int16_t)(a - b - c + d);
}

static void h264e_quant_luma_dc_sse2(quant_t *q, int16_t *deq, const uint16_t *qdat)
{
    int16_t *tmp = ((int16_t*)q) - 16;
    hadamar4_2d_sse(tmp);
    quant_dc_sse(tmp, deq, qdat[0], 16, 0x20000);//0x15555);
    hadamar4_2d_sse(tmp);
    assert(!(qdat[1] & 3));
    // dirty trick here: shift w/o rounding, since it have no effect  for qp >= 10 (or, to be precise, for qp => 9)
    dequant_dc_sse(q, tmp, qdat[1] >> 2, 16);
}

static int h264e_quant_chroma_dc_sse2(quant_t *q, int16_t *deq, const uint16_t *qdat)
{
    int16_t *tmp = ((int16_t*)q) - 16;
    hadamar2_2d_sse(tmp);
    quant_dc_sse(tmp, deq, (int16_t)(qdat[0] << 1), 4, 0xAAAA);
    hadamar2_2d_sse(tmp);
    assert(!(qdat[1] & 1));
    dequant_dc_sse(q, tmp, qdat[1] >> 1, 4);
    return !!(tmp[0] | tmp[1] | tmp[2] | tmp[3]);
}

static int is_zero_sse(const int16_t *dat, int i0, const uint16_t *thr)
{
    __m128i t = _mm_loadu_si128((__m128i*)(thr));
    __m128i d = _mm_load_si128((__m128i*)(dat));
    __m128i z = _mm_setzero_si128();
    __m128i m, sign;
    if (i0) d = _mm_insert_epi16 (d, 0, 0);

    sign = _mm_cmpgt_epi16(z, d);
    d = _mm_sub_epi16(_mm_xor_si128(d, sign), sign);

    m = _mm_cmpgt_epi16(d, t);
    d = _mm_loadu_si128((__m128i*)(dat + 8));
    sign = _mm_cmpgt_epi16(z, d);
    d = _mm_sub_epi16(_mm_xor_si128(d, sign), sign);
    m = _mm_or_si128(m, _mm_cmpgt_epi16(d, t));
    return !_mm_movemask_epi8(m);
}

static int is_zero4_sse(const quant_t *q, int i0, const uint16_t *thr)
{
    return is_zero_sse(q[0].dq, i0, thr) &&
           is_zero_sse(q[1].dq, i0, thr) &&
           is_zero_sse(q[4].dq, i0, thr) &&
           is_zero_sse(q[5].dq, i0, thr);
}

static int h264e_transform_sub_quant_dequant_sse2(const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat)
{
    int crow = mode >> 1;
    int ccol = crow;
    int i, i0 = mode & 1;
    int nz_block_mask = 0;
    int zmask = 0;
    quant_t *q_0 = q;

    int y, x;
    for (y = 0; y < crow; y++)
    {
        for (x = 0; x < ccol; x += 2)
        {
            const pix_t *pinp  = inp  + inp_stride*4*y + 4*x;
            const pix_t *ppred = pred +         16*4*y + 4*x;

            __m128i d0, d1, d2, d3;
            __m128i t0, t1, t2, t3;
            __m128i q0, q1, q2, q3;
            __m128i zero = _mm_setzero_si128();
            __m128i inp8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)pinp),  zero);
            __m128i pred8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)ppred), zero);

            d0 =_mm_sub_epi16(inp8, pred8);
            pinp += inp_stride;
            inp8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)pinp),  zero);
            pred8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(ppred + 16)), zero);
            d1 =_mm_sub_epi16(inp8, pred8);
            pinp += inp_stride;
            inp8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)pinp),  zero);
            pred8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(ppred + 32)), zero);
            d2 =_mm_sub_epi16(inp8, pred8);
            pinp += inp_stride;
            inp8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)pinp),  zero);
            pred8 = _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i*)(ppred + 48)), zero);
            d3 =_mm_sub_epi16(inp8, pred8);
            t0 = _mm_add_epi16(d0, d3);
            t1 = _mm_sub_epi16(d0, d3);
            t2 = _mm_add_epi16(d1, d2);
            t3 = _mm_sub_epi16(d1, d2);
            q0 = _mm_add_epi16(t0, t2);
            q1 = _mm_add_epi16(_mm_add_epi16(t1, t1), t3);
            q2 = _mm_sub_epi16(t0, t2);
            q3 = _mm_sub_epi16(t1, _mm_add_epi16(t3, t3));

            //    q0: a0 a1 ....... a7
            //    q1: b0 .............
            //    q2: c0 .............
            //    q3: d0 d1 ....... d7
            //
            t0 = _mm_unpacklo_epi16(q0, q1);    // a0, b0, a1, b1, a2, b2, a3, b3
            t1 = _mm_unpackhi_epi16(q0, q1);    // a4, b4, a5, b5, a6, b6, a7, b7
            t2 = _mm_unpacklo_epi16(q2, q3);    // c0, d0
            t3 = _mm_unpackhi_epi16(q2, q3);    // c4, d4

            q0 = _mm_unpacklo_epi32(t0, t2);    // a0, b0, c0, d0, a1, b1, c1, d1
            q1 = _mm_unpackhi_epi32(t0, t2);    // a2, b2,
            q2 = _mm_unpacklo_epi32(t1, t3);    // a4, b4
            q3 = _mm_unpackhi_epi32(t1, t3);    // a6, b6

            d0 = _mm_unpacklo_epi64(q0, q2);    // a0, b0, c0, d0, a4, b4, c4, d4
            d1 = _mm_unpackhi_epi64(q0, q2);    // a1, b1, c1, d1
            d2 = _mm_unpacklo_epi64(q1, q3);    // a2, b2,
            d3 = _mm_unpackhi_epi64(q1, q3);    // a3, b3,

            t0 = _mm_add_epi16(d0, d3);
            t1 = _mm_sub_epi16(d0, d3);
            t2 = _mm_add_epi16(d1, d2);
            t3 = _mm_sub_epi16(d1, d2);
            q0 = _mm_add_epi16(t0, t2);
            q1 = _mm_add_epi16(_mm_add_epi16(t1, t1), t3);
            q2 = _mm_sub_epi16(t0, t2);
            q3 = _mm_sub_epi16(t1, _mm_add_epi16(t3, t3));

            _mm_storel_epi64((__m128i*)(q[0].dq    ), q0);
            _mm_storel_epi64((__m128i*)(q[0].dq + 4), q1);
            _mm_storel_epi64((__m128i*)(q[0].dq + 8), q2);
            _mm_storel_epi64((__m128i*)(q[0].dq + 12), q3);
            if (ccol > 1)
            {
                q0 = _mm_unpackhi_epi64(q0, q0); _mm_storel_epi64((__m128i*)(q[1].dq    ), q0);
                q1 = _mm_unpackhi_epi64(q1, q1); _mm_storel_epi64((__m128i*)(q[1].dq + 4), q1);
                q2 = _mm_unpackhi_epi64(q2, q2); _mm_storel_epi64((__m128i*)(q[1].dq + 8), q2);
                q3 = _mm_unpackhi_epi64(q3, q3); _mm_storel_epi64((__m128i*)(q[1].dq + 12), q3);
            }
            q += 2;
        }
    }
    q = q_0;
    crow = mode >> 1;
    ccol = crow;

    if (mode & 1) // QDQ_MODE_INTRA_16 || QDQ_MODE_CHROMA
    {
        int cloop = (mode >> 1)*(mode >> 1);
        short *dc = ((short *)q) - 16;
        quant_t *pq = q;
        do
        {
            *dc++ = pq->dq[0];
            pq++;
        } while (--cloop);
    }

    if (mode == QDQ_MODE_INTER || mode == QDQ_MODE_CHROMA)
    {
        for (i = 0; i < crow*ccol; i++)
        {
            if (is_zero_sse(q[i].dq, i0, qdat + OFFS_THR_1_OFF))
            {
                zmask |= (1 << i);
            }
        }

        if (mode == QDQ_MODE_INTER)
        {
            if ((~zmask & 0x0033) && is_zero4_sse(q +  0, i0, qdat + OFFS_THR_2_OFF)) zmask |= 0x33;
            if ((~zmask & 0x00CC) && is_zero4_sse(q +  2, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 2);
            if ((~zmask & 0x3300) && is_zero4_sse(q +  8, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 8);
            if ((~zmask & 0xCC00) && is_zero4_sse(q + 10, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 10);
        }
    }

    do
    {
        do
        {
            int nz_mask = 0;
            if (zmask & 1)
            {
                _mm_store_si128((__m128i*)(q->qv),     _mm_setzero_si128());
                _mm_store_si128((__m128i*)(q->qv) + 1, _mm_setzero_si128());
            } else
            {
                int16_t *qv_tmp = q->qv;//[16];
                __m128i t;
                const __m128i const_q  = _mm_loadu_si128((__m128i*)(qdat + OFFS_QUANT_VECT));
                const __m128i const_dq = _mm_loadu_si128((__m128i*)(qdat + OFFS_DEQUANT_VECT));

                __m128i src = _mm_load_si128((__m128i*)(q[0].dq));
                __m128i r = _mm_xor_si128(_mm_set1_epi16(qdat[OFFS_RND_INTER]), _mm_cmpgt_epi16(_mm_setzero_si128(), src));
                __m128i lo = _mm_mullo_epi16(src, const_q);
                __m128i hi = _mm_mulhi_epi16(src, const_q);
                __m128i dst0 = _mm_unpacklo_epi16(lo, hi);
                __m128i dst1 = _mm_unpackhi_epi16(lo, hi);
                dst0 = _mm_srai_epi32(_mm_add_epi32(dst0, _mm_unpacklo_epi16(r, _mm_setzero_si128())), 16);
                dst1 = _mm_srai_epi32(_mm_add_epi32(dst1, _mm_unpackhi_epi16(r, _mm_setzero_si128())), 16);
                dst0 = _mm_packs_epi32(dst0, dst1);
                _mm_store_si128((__m128i*)(qv_tmp), dst0);

                t = _mm_cmpeq_epi16(_mm_setzero_si128(), dst0);
                nz_mask = _mm_movemask_epi8( _mm_packs_epi16(t, t)) & 0xff;
                dst0 = _mm_mullo_epi16(dst0, const_dq);
                _mm_store_si128((__m128i*)(q[0].dq), dst0);


                src = _mm_load_si128((__m128i*)(q[0].dq + 8));
                r = _mm_xor_si128(_mm_set1_epi16(qdat[OFFS_RND_INTER]), _mm_cmpgt_epi16(_mm_setzero_si128(), src));
                lo = _mm_mullo_epi16(src, const_q);
                hi = _mm_mulhi_epi16(src, const_q);
                dst0 = _mm_unpacklo_epi16(lo, hi);
                dst1 = _mm_unpackhi_epi16(lo, hi);

                dst0 = _mm_srai_epi32(_mm_add_epi32(dst0, _mm_unpacklo_epi16(r, _mm_setzero_si128())), 16);
                dst1 = _mm_srai_epi32(_mm_add_epi32(dst1, _mm_unpackhi_epi16(r, _mm_setzero_si128())), 16);
                dst0 = _mm_packs_epi32(dst0, dst1);
                _mm_store_si128((__m128i*)(qv_tmp + 8), dst0);

                t = _mm_cmpeq_epi16(_mm_setzero_si128(), dst0);
                nz_mask |= _mm_movemask_epi8( _mm_packs_epi16(t, t)) << 8;
                dst0 = _mm_mullo_epi16(dst0, const_dq);
                _mm_store_si128((__m128i*)(q[0].dq + 8), dst0);
                nz_mask = ~nz_mask & 0xffff;
                if (i0)
                {
                    nz_mask &= ~1;
                }
            }

            zmask >>= 1;
            nz_block_mask <<= 1;
            if (nz_mask)
                nz_block_mask |= 1;
            q++;
        } while (--ccol);
        ccol = mode >> 1;
    } while (--crow);
    return nz_block_mask;
}

static void h264e_transform_add_sse2(pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask)
{
    int crow = side;
    int ccol = crow;

    assert(IS_ALIGNED(out, 4));
    assert(IS_ALIGNED(pred, 4));
    assert(!(out_stride % 4));

    do
    {
        do
        {
            if (mask >= 0)
            {
                // copy 4x4
                pix_t *dst = out;
                *(uint32_t*)dst = *(uint32_t*)(pred + 0 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 1 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 2 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 3 * 16);
            }
            else
            {
                __m128i zero = _mm_setzero_si128();
                __m128i c32 = _mm_set1_epi16(32);
                __m128i d0, d1, d2, d3;
                __m128i e0, e1, e2, e3;
                d0 = _mm_load_si128((__m128i*)(q->dq + 0));
                d2 = _mm_load_si128((__m128i*)(q->dq + 8));
                d1 = _mm_unpackhi_epi64(d0, d2);
                d3 = _mm_unpackhi_epi64(d2, d0);

                e0 = _mm_add_epi16(d0, d2);
                e1 = _mm_sub_epi16(d0, d2);

                e2 = _mm_srai_epi16(d1, 1);
                e2 = _mm_sub_epi16(e2, d3);
                e3 = _mm_srai_epi16(d3, 1);
                e3 = _mm_add_epi16(e3, d1);

                d0 = _mm_add_epi16(e0, e3);
                d1 = _mm_add_epi16(e1, e2);
                d2 = _mm_sub_epi16(e1, e2);
                d3 = _mm_sub_epi16(e0, e3);

                e1 = _mm_unpacklo_epi16(d0, d1);    // a0, b0, a1, b1, a2, b2, a3, b3
                e3 = _mm_unpacklo_epi16(d2, d3);    // c0, d0

                e0 = _mm_unpacklo_epi32(e1, e3);    // a0, b0, c0, d0, a1, b1, c1, d1
                e2 = _mm_unpackhi_epi32(e1, e3);    // a2, b2,

                e1 = _mm_unpackhi_epi64(e0, e2);
                e3 = _mm_unpackhi_epi64(e2, e0);

                d0 = _mm_add_epi16(e0, e2);
                d1 = _mm_sub_epi16(e0, e2);
                d2 = _mm_srai_epi16(e1, 1);
                d2 = _mm_sub_epi16(d2, e3);
                d3 = _mm_srai_epi16(e3, 1);
                d3 = _mm_add_epi16(d3, e1);

                // Pack 4x64 to 2x128
                e0 = _mm_unpacklo_epi64(d0, d1);
                e1 = _mm_unpacklo_epi64(d3, d2);

                e0 = _mm_add_epi16(e0, c32);
                d0 = _mm_srai_epi16(_mm_add_epi16(e0, e1), 6);
                d3 = _mm_srai_epi16(_mm_sub_epi16(e0, e1), 6);
                // Unpack back to 4x64
                d1 = _mm_unpackhi_epi64(d0, zero);
                d2 = _mm_unpackhi_epi64(d3, zero);

                *(int* )(out)                = _mm_cvtsi128_si32(_mm_packus_epi16(_mm_add_epi16(_mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int*)(pred +  0)), zero), d0), zero));
                *(int* )(out + 1*out_stride) = _mm_cvtsi128_si32(_mm_packus_epi16(_mm_add_epi16(_mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int*)(pred + 16)), zero), d1), zero));
                *(int* )(out + 2*out_stride) = _mm_cvtsi128_si32(_mm_packus_epi16(_mm_add_epi16(_mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int*)(pred + 32)), zero), d2), zero));
                *(int* )(out + 3*out_stride) = _mm_cvtsi128_si32(_mm_packus_epi16(_mm_add_epi16(_mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int*)(pred + 48)), zero), d3), zero));

            }
            mask = (uint32_t)mask << 1;
            q++;
            out += 4;
            pred += 4;
        } while (--ccol);
        ccol = side;
        out += 4*(out_stride - ccol);
        pred += 4*(16 - ccol);
    } while (--crow);
}
#endif

#if H264E_ENABLE_NEON && !defined(MINIH264_ASM)
#define TR32(x, y) tr0 = vtrnq_u32(vreinterpretq_u32_u8(x), vreinterpretq_u32_u8(y)); x = vreinterpretq_u8_u32(tr0.val[0]); y = vreinterpretq_u8_u32(tr0.val[1]);
#define TR16(x, y) tr1 = vtrnq_u16(vreinterpretq_u16_u8(x), vreinterpretq_u16_u8(y)); x = vreinterpretq_u8_u16(tr1.val[0]); y = vreinterpretq_u8_u16(tr1.val[1]);
#define TR8(x, y)  tr2 = vtrnq_u8((x), (y)); x = (tr2.val[0]); y = (tr2.val[1]);

static void deblock_luma_v_neon(uint8_t *pix, int stride, int alpha, int beta, const uint8_t *pthr, const uint8_t *pstr)
{
    uint8x16_t q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, q15;
    uint8x16_t tmp;
    uint32x4x2_t tr0;
    uint16x8x2_t tr1;
    uint8x16x2_t tr2;
    q8 = vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q9 = vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q10= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q11= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q12= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q13= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q14= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q15= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;

    TR32(q8,  q12);
    TR32(q9,  q13);
    TR32(q10, q14);
    TR32(q11, q15);
    TR16(q8,  q10);
    TR16(q9,  q11);
    TR16(q12, q14);
    TR16(q13, q15);
    TR8(q8,   q9 );
    TR8(q10,  q11);
    TR8(q12,  q13);
    TR8(q14,  q15);

    q1  = vabdq_u8(q11, q12);
    q2  = vcltq_u8(q1, vdupq_n_u8(alpha));
    q1  = vcltq_u8(vmaxq_u8(vabdq_u8(q11, q10), vabdq_u8(q12, q13)), vdupq_n_u8(beta));
    q2  = vandq_u8(q2, q1);

    tmp = vreinterpretq_u8_u32(vdupq_n_u32(*(uint32_t*)pstr));
    tmp = vzipq_u8(tmp, tmp).val[0];
    tmp = vzipq_u8(tmp, tmp).val[0];
    q1  = tmp;

    q1  = vcgtq_s8(vreinterpretq_s8_u8(q1), vdupq_n_s8(0));
    q2  = vandq_u8(q2, q1);
    q7 = vhsubq_u8(q10, q13);
    q7 = vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_u8(q7), 1));
    q0 = veorq_u8(q12, q11);
    q6 = vandq_u8(vdupq_n_u8(1), q0);

    q0 = vhsubq_u8(q12, q11);// ;(q0-p0))>>1

    q7 = vreinterpretq_u8_s8(vrhaddq_s8(vreinterpretq_s8_u8(q7), vreinterpretq_s8_u8(q6))); //((p1-q1)>>2 + carry + 1) >> 1
    q7 = vreinterpretq_u8_s8(vqaddq_s8(vreinterpretq_s8_u8(q0),  vreinterpretq_s8_u8(q7))); //=delta = (((q0-p0)<<2) + (p1-q1) + 4) >> 3;
    q7 = vandq_u8(q7, q2);

    tmp = vreinterpretq_u8_u32(vdupq_n_u32(*(uint32_t*)pthr));
    tmp = vzipq_u8(tmp, tmp).val[0];
    tmp = vzipq_u8(tmp, tmp).val[0];
    q1  = tmp;

    q1  = vandq_u8(q2, q1);

    q0 = vabdq_u8(q9,  q11); // ap = ABS(p2 - p0);
    q0 = vcltq_u8(q0,  vdupq_n_u8(beta)); //sp = (ap - beta) >> 31;
    q4 = vandq_u8(q0,  q2);  // & sp
    q0 = vabdq_u8(q14, q12); //aq = ABS(q2 - q0);
    q0 = vcltq_u8(q0,  vdupq_n_u8(beta))  ;//sq = (aq - beta) >> 31;
    q3 = vandq_u8(q0,  q2);  //  & sq

    q0  = vrhaddq_u8(q11, q12);//((p0+q0+1)>>1)
    q0  = vhaddq_u8 (q0,  q9 );//((p2 + ((p0+q0+1)>>1))>>1)
    q5  = vandq_u8  (q1,  q4 );
    q6  = vqaddq_u8 (q10, q5 );//{p1+thr}
    q0  = vminq_u8  (q0,  q6 );
    q6  = vqsubq_u8 (q10, q5 );//{p1-thr}
    q10 = vmaxq_u8  (q0,  q6 );

    q0  = vrhaddq_u8(q11, q12);// ;((p0+q0+1)>>1)
    q0  = vhaddq_u8 (q0,  q14);// ;((q2 + ((p0+q0+1)>>1))>>1)
    q5  = vandq_u8  (q1,  q3 );
    q6  = vqaddq_u8 (q13, q5 );// ;{q1+thr}
    q0  = vminq_u8  (q0,  q6 );
    q6  = vqsubq_u8 (q13, q5 );// ;{q1-thr}
    q13 = vmaxq_u8  (q0,  q6 );

    q1  = vreinterpretq_u8_s8(vsubq_s8(vreinterpretq_s8_u8(q1), vreinterpretq_s8_u8(q3)));
    q1  = vreinterpretq_u8_s8(vsubq_s8(vreinterpretq_s8_u8(q1), vreinterpretq_s8_u8(q4))); //tC = thr - sp - sq;
    q1  = vandq_u8(q1, q2);// ; set thr = 0 if str==0

    q6  = veorq_u8(q6, q6);
    q5  = vreinterpretq_u8_s8(vmaxq_s8(vreinterpretq_s8_u8(q6), vreinterpretq_s8_u8(q7))); //delta > 0
    q7  = vreinterpretq_u8_s8(vsubq_s8(vreinterpretq_s8_u8(q6), vreinterpretq_s8_u8(q7)));
    q6  = vreinterpretq_u8_s8(vmaxq_s8(vreinterpretq_s8_u8(q6), vreinterpretq_s8_u8(q7))); //-(delta < 0)
    q5  =  vminq_u8(q1, q5);
    q6  =  vminq_u8(q1, q6);

    q11 = vqaddq_u8(q11, q5);
    q11 = vqsubq_u8(q11, q6);
    q12 = vqsubq_u8(q12, q5);
    q12 = vqaddq_u8(q12, q6);

    TR8(q8,   q9 );
    TR8(q10,  q11);
    TR8(q12,  q13);
    TR8(q14,  q15);
    TR16(q8,  q10);
    TR16(q9,  q11);
    TR16(q12, q14);
    TR16(q13, q15);
    TR32(q8,  q12);
    TR32(q9,  q13);
    TR32(q10, q14);
    TR32(q11, q15);

    pix -= 8*stride + 4;
    vst1_u8(pix, vget_low_u8(q8));  pix += stride;
    vst1_u8(pix, vget_low_u8(q9));  pix += stride;
    vst1_u8(pix, vget_low_u8(q10)); pix += stride;
    vst1_u8(pix, vget_low_u8(q11)); pix += stride;
    vst1_u8(pix, vget_low_u8(q12)); pix += stride;
    vst1_u8(pix, vget_low_u8(q13)); pix += stride;
    vst1_u8(pix, vget_low_u8(q14)); pix += stride;
    vst1_u8(pix, vget_low_u8(q15)); pix += stride;

    vst1_u8(pix, vget_high_u8(q8)); pix += stride;
    vst1_u8(pix, vget_high_u8(q9)); pix += stride;
    vst1_u8(pix, vget_high_u8(q10)); pix += stride;
    vst1_u8(pix, vget_high_u8(q11)); pix += stride;
    vst1_u8(pix, vget_high_u8(q12)); pix += stride;
    vst1_u8(pix, vget_high_u8(q13)); pix += stride;
    vst1_u8(pix, vget_high_u8(q14)); pix += stride;
    vst1_u8(pix, vget_high_u8(q15)); pix += stride;
}

static void deblock_luma_h_s4_neon(uint8_t *pix, int stride, int alpha, int beta)
{
    uint8x16_t q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, q15, vspill0, vspill1;
    q8  = vld1q_u8(pix - 4*stride);
    q9  = vld1q_u8(pix - 3*stride);
    q10 = vld1q_u8(pix - 2*stride);
    q11 = vld1q_u8(pix - 1*stride);
    q12 = vld1q_u8(pix);
    q13 = vld1q_u8(pix + 1*stride);
    q14 = vld1q_u8(pix + 2*stride);
    q15 = vld1q_u8(pix + 3*stride);
    q0  = vabdq_u8(q11, q12);
    q2  = vcltq_u8(q0, vdupq_n_u8(alpha));
    q2  = vandq_u8(q2, vcltq_u8(vabdq_u8(q11, q10), vdupq_n_u8(beta)));
    q2  = vandq_u8(q2, vcltq_u8(vabdq_u8(q12, q13), vdupq_n_u8(beta)));
    q1  = vandq_u8(q2, vcltq_u8(q0, vdupq_n_u8(((alpha >> 2) + 2))));
    q0  = vandq_u8(q1, vcltq_u8(vabdq_u8(q9,  q11), vdupq_n_u8(beta)));
    q3  = vandq_u8(q1, vcltq_u8(vabdq_u8(q14, q12), vdupq_n_u8(beta)));
    q4 = vhaddq_u8(q9,  q10);
    q5 = vhaddq_u8(q11, q12);
    q6 = vsubq_u8(vrhaddq_u8(q9,  q10), q4);
    q7 = vsubq_u8(vrhaddq_u8(q11, q12), q5);
    q6 = vhaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q4, q8);
    q4 = vhaddq_u8(q4, q8);
    q7 = vsubq_u8(q7, q4);
    q6 = vaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q5, q9);
    q5 = vhaddq_u8(q5, q9);
    q7 = vsubq_u8(q7, q5);
    q6 = vhaddq_u8(q6, q7);

    q7 = vrhaddq_u8(q4, q5);
    q4 = vhaddq_u8(q4, q5);
    q7 = vsubq_u8(q7, q4);
    q6 = vrhaddq_u8(q6, q7);
    q4 = vaddq_u8(q4, q6);
    vspill0 =  vbslq_u8(q0, q4, q9);   // VMOV        q6,     q9   VBIT        q6,     q4,     q0

    q4 = vhaddq_u8(q14, q13);
    q5 = vhaddq_u8(q12, q11);
    q6 = vsubq_u8(vrhaddq_u8(q14, q13), q4);
    q7 = vsubq_u8(vrhaddq_u8(q12, q11), q5);
    q6 = vhaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q4, q15);
    q4 = vhaddq_u8(q4, q15);
    q7 = vsubq_u8(q7, q4);
    q6 = vaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q5, q14);
    q5 = vhaddq_u8(q5, q14);
    q7 = vsubq_u8(q7, q5);
    q6 = vhaddq_u8(q6, q7);

    q7 = vrhaddq_u8(q4, q5);
    q4 = vhaddq_u8(q4, q5);
    q7 = vsubq_u8(q7, q4);
    q6 = vrhaddq_u8(q6, q7);
    q4 = vaddq_u8(q4, q6);
    vspill1 =  vbslq_u8(q3, q4, q14);   //     VMOV        q6,     q14    VBIT        q6,     q4,     q3

    q1 = vhaddq_u8 (q9,  q13);
    q4 = vrhaddq_u8(q1,  q10);
    q5 = vrhaddq_u8(q11, q12);
    q6 = vhaddq_u8 (q1,  q10);
    q7 = vhaddq_u8 (q11, q12);
    q4 = vhaddq_u8 (q4,  q5);
    q6 = vrhaddq_u8(q6,  q7);
    q1 = vrhaddq_u8(q4,  q6);
    q4 = vrhaddq_u8(q9,  q10);
    q5 = vrhaddq_u8(q11, q12);
    q6 = vhaddq_u8 (q9,  q10);
    q7 = vhaddq_u8 (q11, q12);
    q4 = vhaddq_u8 (q4,  q5);
    q6 = vrhaddq_u8(q6,  q7);
    q4 = vrhaddq_u8(q4,  q6);
    q5 = vhaddq_u8 (q11, q13);
    q5 = vrhaddq_u8(q5,  q10);

    q1 = vbslq_u8(q0, q1, q5); //VBIF        q1,     q5,     q0
    q0 = vbslq_u8(q0, q4, q10);//VBSL        q0,     q4,     q10

    q7 = vhaddq_u8 (q14, q10);
    q4 = vrhaddq_u8(q7,  q13);
    q5 = vrhaddq_u8(q11, q12);
    q6 = vhaddq_u8 (q7,  q13);
    q7 = vhaddq_u8 (q11, q12);
    q4 = vhaddq_u8 (q4,  q5 );
    q6 = vrhaddq_u8(q6,  q7 );
    q4 = vrhaddq_u8(q4,  q6 );
    q6 = vrhaddq_u8(q14, q13);
    q5 = vrhaddq_u8(q11, q12);
    q5 = vhaddq_u8 (q6,  q5 );
    q6 = vhaddq_u8 (q14, q13);
    q7 = vhaddq_u8 (q11, q12);
    q6 = vrhaddq_u8(q6,  q7 );
    q5 = vrhaddq_u8(q5,  q6 );
    q6 = vhaddq_u8 (q12, q10);
    q6 = vrhaddq_u8(q6,  q13);

    q4 = vbslq_u8(q3, q4, q6); //    VBIF        q4,     q6,     q3    ;q0
    q3 = vbslq_u8(q3, q5, q13);//    VBSL        q3,     q5,     q13   ;q1

    q10 = vbslq_u8(q2, q0, q10);
    q11 = vbslq_u8(q2, q1, q11);
    q12 = vbslq_u8(q2, q4, q12);
    q13 = vbslq_u8(q2, q3, q13);

    vst1q_u8(pix - 3*stride, vspill0);
    vst1q_u8(pix - 2*stride, q10);
    vst1q_u8(pix - 1*stride, q11);
    vst1q_u8(pix           , q12);
    vst1q_u8(pix + 1*stride, q13);
    vst1q_u8(pix + 2*stride, vspill1);

}

static void deblock_luma_v_s4_neon(uint8_t *pix, int stride, int alpha, int beta)
{
    uint32x4x2_t tr0;
    uint16x8x2_t tr1;
    uint8x16x2_t tr2;
    uint8x16_t q0, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, q15, vspill0, vspill1;
    q8 = vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q9 = vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q10= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q11= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q12= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q13= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q14= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;
    q15= vcombine_u8(vld1_u8(pix - 4), vld1_u8(pix - 4 + 8*stride)); pix += stride;

    TR32(q8,  q12);
    TR32(q9,  q13);
    TR32(q10, q14);
    TR32(q11, q15);
    TR16(q8,  q10);
    TR16(q9,  q11);
    TR16(q12, q14);
    TR16(q13, q15);
    TR8(q8,   q9 );
    TR8(q10,  q11);
    TR8(q12,  q13);
    TR8(q14,  q15);

    q0 = vabdq_u8(q11, q12);
    q2 = vcltq_u8(q0, vdupq_n_u8(alpha));
    q2 = vandq_u8(q2, vcltq_u8(vabdq_u8(q11,    q10), vdupq_n_u8(beta)));
    q2 = vandq_u8(q2, vcltq_u8(vabdq_u8(q12,    q13), vdupq_n_u8(beta)));
    q1 = vandq_u8(q2, vcltq_u8(q0, vdupq_n_u8(((alpha >> 2) + 2))));
    q0 = vandq_u8(q1, vcltq_u8(vabdq_u8(q9,     q11), vdupq_n_u8(beta)));
    q3 = vandq_u8(q1, vcltq_u8(vabdq_u8(q14,    q12), vdupq_n_u8(beta)));
    q4 = vhaddq_u8(q9,  q10);
    q5 = vhaddq_u8(q11, q12);
    q6 = vsubq_u8(vrhaddq_u8(q9,  q10), q4);
    q7 = vsubq_u8(vrhaddq_u8(q11, q12), q5);
    q6 = vhaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q4, q8);
    q4 = vhaddq_u8(q4, q8);
    q7 = vsubq_u8(q7, q4);
    q6 = vaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q5, q9);
    q5 = vhaddq_u8(q5, q9);
    q7 = vsubq_u8(q7, q5);
    q6 = vhaddq_u8(q6, q7);

    q7 = vrhaddq_u8(q4, q5);
    q4 = vhaddq_u8(q4, q5);
    q7 = vsubq_u8(q7, q4);
    q6 = vrhaddq_u8(q6, q7);
    q4 = vaddq_u8(q4, q6);
    vspill0 =  vbslq_u8(q0, q4, q9);   // VMOV        q6,     q9   VBIT        q6,     q4,     q0

    q4 = vhaddq_u8(q14, q13);
    q5 = vhaddq_u8(q12, q11);
    q6 = vsubq_u8(vrhaddq_u8(q14, q13), q4);
    q7 = vsubq_u8(vrhaddq_u8(q12, q11), q5);
    q6 = vhaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q4, q15);
    q4 = vhaddq_u8(q4, q15);
    q7 = vsubq_u8(q7, q4);
    q6 = vaddq_u8(q6, q7);
    q7 = vrhaddq_u8(q5, q14);
    q5 = vhaddq_u8(q5, q14);
    q7 = vsubq_u8(q7, q5);
    q6 = vhaddq_u8(q6, q7);

    q7 = vrhaddq_u8(q4, q5);
    q4 = vhaddq_u8(q4, q5);
    q7 = vsubq_u8(q7, q4);
    q6 = vrhaddq_u8(q6, q7);
    q4 = vaddq_u8(q4, q6);
    vspill1 =  vbslq_u8(q3, q4, q14);   //     VMOV        q6,     q14    VBIT        q6,     q4,     q3

    q1 = vhaddq_u8 (q9,  q13);
    q4 = vrhaddq_u8(q1,  q10);
    q5 = vrhaddq_u8(q11, q12);
    q6 = vhaddq_u8 (q1,  q10);
    q7 = vhaddq_u8 (q11, q12);
    q4 = vhaddq_u8 (q4,  q5);
    q6 = vrhaddq_u8(q6,  q7);
    q1 = vrhaddq_u8(q4,  q6);
    q4 = vrhaddq_u8(q9,  q10);
    q5 = vrhaddq_u8(q11, q12);
    q6 = vhaddq_u8 (q9,  q10);
    q7 = vhaddq_u8 (q11, q12);
    q4 = vhaddq_u8 (q4,  q5);
    q6 = vrhaddq_u8(q6,  q7);
    q4 = vrhaddq_u8(q4,  q6);
    q5 = vhaddq_u8 (q11, q13);
    q5 = vrhaddq_u8(q5,  q10);

    q1 = vbslq_u8(q0, q1, q5); //VBIF        q1,     q5,     q0
    q0 = vbslq_u8(q0, q4, q10);//VBSL        q0,     q4,     q10

    q7 = vhaddq_u8 (q14, q10);
    q4 = vrhaddq_u8(q7,  q13);
    q5 = vrhaddq_u8(q11, q12);
    q6 = vhaddq_u8 (q7,  q13);
    q7 = vhaddq_u8 (q11, q12);
    q4 = vhaddq_u8 (q4,  q5 );
    q6 = vrhaddq_u8(q6,  q7 );
    q4 = vrhaddq_u8(q4,  q6 );
    q6 = vrhaddq_u8(q14, q13);
    q5 = vrhaddq_u8(q11, q12);
    q5 = vhaddq_u8 (q6,  q5 );
    q6 = vhaddq_u8 (q14, q13);
    q7 = vhaddq_u8 (q11, q12);
    q6 = vrhaddq_u8(q6,  q7 );
    q5 = vrhaddq_u8(q5,  q6 );
    q6 = vhaddq_u8 (q12, q10);
    q6 = vrhaddq_u8(q6,  q13);

    q4 = vbslq_u8(q3, q4, q6); //    VBIF        q4,     q6,     q3    ;q0
    q3 = vbslq_u8(q3, q5, q13);//    VBSL        q3,     q5,     q13   ;q1

    q10 = vbslq_u8(q2,q0, q10);
    q11 = vbslq_u8(q2,q1, q11);
    q12 = vbslq_u8(q2,q4, q12);
    q13 = vbslq_u8(q2,q3, q13);

    q9 = vspill0;
    q14 = vspill1;

    TR8(q8,   q9 );
    TR8(q10,  q11);
    TR8(q12,  q13);
    TR8(q14,  q15);
    TR16(q8,  q10);
    TR16(q9,  q11);
    TR16(q12, q14);
    TR16(q13, q15);
    TR32(q8,  q12);
    TR32(q9,  q13);
    TR32(q10, q14);
    TR32(q11, q15);

    pix -= 8*stride + 4;
    vst1_u8(pix, vget_low_u8(q8)); pix += stride;
    vst1_u8(pix, vget_low_u8(q9)); pix += stride;
    vst1_u8(pix, vget_low_u8(q10)); pix += stride;
    vst1_u8(pix, vget_low_u8(q11)); pix += stride;
    vst1_u8(pix, vget_low_u8(q12)); pix += stride;
    vst1_u8(pix, vget_low_u8(q13)); pix += stride;
    vst1_u8(pix, vget_low_u8(q14)); pix += stride;
    vst1_u8(pix, vget_low_u8(q15)); pix += stride;

    vst1_u8(pix, vget_high_u8(q8)); pix += stride;
    vst1_u8(pix, vget_high_u8(q9)); pix += stride;
    vst1_u8(pix, vget_high_u8(q10)); pix += stride;
    vst1_u8(pix, vget_high_u8(q11)); pix += stride;
    vst1_u8(pix, vget_high_u8(q12)); pix += stride;
    vst1_u8(pix, vget_high_u8(q13)); pix += stride;
    vst1_u8(pix, vget_high_u8(q14)); pix += stride;
    vst1_u8(pix, vget_high_u8(q15)); pix += stride;
}

static void deblock_luma_h_neon(uint8_t *pix, int stride, int alpha, int beta, const uint8_t *pthr, const uint8_t *pstr)
{
    uint8x16_t q0, q1, q2, q3, q4, q5, q6, q7, q9, q10, q11, q12, q13, q14;
    uint8x16_t tmp;

    q9  = vld1q_u8(pix - 3*stride);
    q10 = vld1q_u8(pix - 2*stride);
    q11 = vld1q_u8(pix - 1*stride);
    q12 = vld1q_u8(pix);
    q13 = vld1q_u8(pix + 1*stride);
    q14 = vld1q_u8(pix + 2*stride);

    q1  = vabdq_u8(q11, q12);
    q2  = vcltq_u8(q1, vdupq_n_u8(alpha));
    q1  = vcltq_u8(vmaxq_u8(vabdq_u8(q11, q10), vabdq_u8(q12, q13)), vdupq_n_u8(beta));
    q2  = vandq_u8(q2, q1);

    tmp = vreinterpretq_u8_u32(vdupq_n_u32(*(uint32_t*)pstr));
    tmp = vzipq_u8(tmp, tmp).val[0];
    tmp = vzipq_u8(tmp, tmp).val[0];
    q1  = tmp;

    q1  = vcgtq_s8(vreinterpretq_s8_u8(q1), vdupq_n_s8(0));
    q2  = vandq_u8(q2, q1);
    q7 = vhsubq_u8(q10, q13);
    q7 = vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_u8(q7), 1));
    q0 = veorq_u8(q12, q11);
    q6 = vandq_u8(vdupq_n_u8(1), q0);

    q0 = vhsubq_u8(q12, q11);// ;(q0-p0))>>1

    q7 = vreinterpretq_u8_s8(vrhaddq_s8(vreinterpretq_s8_u8(q7), vreinterpretq_s8_u8(q6))); //((p1-q1)>>2 + carry + 1) >> 1
    q7 = vreinterpretq_u8_s8(vqaddq_s8(vreinterpretq_s8_u8(q0),  vreinterpretq_s8_u8(q7))); //=delta = (((q0-p0)<<2) + (p1-q1) + 4) >> 3;
    q7 = vandq_u8(q7, q2);

    tmp = vreinterpretq_u8_u32(vdupq_n_u32(*(uint32_t*)pthr));
    tmp = vzipq_u8(tmp, tmp).val[0];
    tmp = vzipq_u8(tmp, tmp).val[0];
    q1  = tmp;

    q1  = vandq_u8(q2, q1);

    q0 = vabdq_u8(q9,  q11); // ap = ABS(p2 - p0);
    q0 = vcltq_u8(q0,  vdupq_n_u8(beta)); //sp = (ap - beta) >> 31;
    q4 = vandq_u8(q0,  q2); // & sp
    q0 = vabdq_u8(q14, q12);//aq = ABS(q2 - q0);
    q0 = vcltq_u8(q0,  vdupq_n_u8(beta));//sq = (aq - beta) >> 31;
    q3 = vandq_u8(q0,  q2); // & sq

    q0  = vrhaddq_u8(q11, q12);//((p0+q0+1)>>1)
    q0  = vhaddq_u8 (q0,  q9 );//((p2 + ((p0+q0+1)>>1))>>1)
    q5  = vandq_u8  (q1,  q4 );
    q6  = vqaddq_u8 (q10, q5 );//{p1+thr}
    q0  = vminq_u8  (q0,  q6 );
    q6  = vqsubq_u8 (q10, q5 );//{p1-thr}
    q10 = vmaxq_u8  (q0,  q6 );

    q0   = vrhaddq_u8(q11, q12);// ;((p0+q0+1)>>1)
    q0   = vhaddq_u8 (q0,  q14);// ;((q2 + ((p0+q0+1)>>1))>>1)
    q5   = vandq_u8  (q1,  q3 );
    q6   = vqaddq_u8 (q13, q5 );// ;{q1+thr}
    q0   = vminq_u8  (q0,  q6 );
    q6   = vqsubq_u8 (q13, q5 );// ;{q1-thr}
    q13  = vmaxq_u8  (q0,  q6 );

    q1  = vreinterpretq_u8_s8(vsubq_s8(vreinterpretq_s8_u8(q1), vreinterpretq_s8_u8(q3)));
    q1  = vreinterpretq_u8_s8(vsubq_s8(vreinterpretq_s8_u8(q1), vreinterpretq_s8_u8(q4))); //tC = thr - sp - sq;
    q1  = vandq_u8(q1, q2);// ; set thr = 0 if str==0

    q6  = veorq_u8(q6, q6);
    q5  = vreinterpretq_u8_s8(vmaxq_s8(vreinterpretq_s8_u8(q6), vreinterpretq_s8_u8(q7))); //delta > 0
    q7  = vreinterpretq_u8_s8(vsubq_s8(vreinterpretq_s8_u8(q6), vreinterpretq_s8_u8(q7)));
    q6  = vreinterpretq_u8_s8(vmaxq_s8(vreinterpretq_s8_u8(q6), vreinterpretq_s8_u8(q7))); //-(delta < 0)
    q5  =  vminq_u8(q1, q5);
    q6  =  vminq_u8(q1, q6);

    q11 = vqaddq_u8(q11, q5);
    q11 = vqsubq_u8(q11, q6);
    q12 = vqsubq_u8(q12, q5);
    q12 = vqaddq_u8(q12, q6);

    vst1q_u8(pix - 2*stride, q10);
    vst1q_u8(pix - 1*stride, q11);
    vst1q_u8(pix           , q12);
    vst1q_u8(pix + 1*stride, q13);
}

static void deblock_chroma_v_neon(uint8_t *pix, int32_t stride, int a, int b, const uint8_t *thr, const uint8_t *str)
{
    int32x2_t d16 = vld1_s32((int32_t*)(pix - 2 + 0*stride));
    int32x2_t d18 = vld1_s32((int32_t*)(pix - 2 + 1*stride));
    int32x2_t d20 = vld1_s32((int32_t*)(pix - 2 + 2*stride));
    int32x2_t d22 = vld1_s32((int32_t*)(pix - 2 + 3*stride));
    int32x2_t d17 = vld1_s32((int32_t*)(pix - 2 + 4*stride));
    int32x2_t d19 = vld1_s32((int32_t*)(pix - 2 + 5*stride));
    int32x2_t d21 = vld1_s32((int32_t*)(pix - 2 + 6*stride));
    int32x2_t d23 = vld1_s32((int32_t*)(pix - 2 + 7*stride));
    int32x2x2_t tr0 = vtrn_s32(d16, d17);
    int32x2x2_t tr1 = vtrn_s32(d18, d19);
    int32x2x2_t tr2 = vtrn_s32(d20, d21);
    int32x2x2_t tr3 = vtrn_s32(d22, d23);
    int16x8x2_t tr4 = vtrnq_s16(vreinterpretq_s16_s32(vcombine_s32(tr0.val[0], tr0.val[1])), vreinterpretq_s16_s32(vcombine_s32(tr2.val[0], tr2.val[1])));
    int16x8x2_t tr5 = vtrnq_s16(vreinterpretq_s16_s32(vcombine_s32(tr1.val[0], tr1.val[1])), vreinterpretq_s16_s32(vcombine_s32(tr3.val[0], tr3.val[1])));
    uint8x16x2_t tr6 = vtrnq_u8(vreinterpretq_u8_s16(tr4.val[0]), vreinterpretq_u8_s16(tr5.val[0]));
    uint8x16x2_t tr7 = vtrnq_u8(vreinterpretq_u8_s16(tr4.val[1]), vreinterpretq_u8_s16(tr5.val[1]));

{
    uint8x16_t q8  = tr6.val[0];
    uint8x16_t q9  = tr6.val[1];
    uint8x16_t q10 = tr7.val[0];
    uint8x16_t q11 = tr7.val[1];

    uint8x16_t q1  = vabdq_u8(q9, q10);
    uint8x16_t q2  = vcltq_u8(q1, vdupq_n_u8(a));
    uint8x16_t q4  = vmaxq_u8(vabdq_u8(q10, q11), vabdq_u8(q8, q9));
    uint8x16_t q0;
    uint8x16_t q3;
    uint8x16_t q6;
     int8x16_t q4s;
     int8x16_t q7;
    uint8x16_t q7u;
    uint8x16_t q5;
    uint8x16_t vstr = vld1q_u8(str);
    uint8x16_t vthr = vld1q_u8(thr);

    q4 = vcltq_u8(q4, vdupq_n_u8(b));
    q2 = vandq_u8(q2, q4);
    q1 = vzipq_u8(vstr, vstr).val[0];
    q3 = vcgtq_s8(vreinterpretq_s8_u8(q1), vdupq_n_s8(0));
    q1 = vshrq_n_u8(q1, 2);
    q1 = vcgtq_s8(vreinterpretq_s8_u8(q1), vdupq_n_s8(0));
    q2 = vandq_u8(q2, q3);

    q0 = vzipq_u8(vthr, vthr).val[0];
    q0 = vaddq_u8(q0, vdupq_n_u8(1));
    q0 = vandq_u8(q0, q2);

    q7 = vshrq_n_s8(vreinterpretq_s8_u8(vhsubq_u8(q8, q11)), 1);
    q6 = vandq_u8(vdupq_n_u8(1), veorq_u8(q10, q9));
    q4 = vhsubq_u8(q10, q9);
    q7 = vrhaddq_s8(q7, vreinterpretq_s8_u8(q6));
    q7 = vqaddq_s8(vreinterpretq_s8_u8(q4), q7);

    q4s = vdupq_n_s8(0);
    q5 = vreinterpretq_u8_s8(vmaxq_s8(q4s,               q7));
    q4 = vreinterpretq_u8_s8(vmaxq_s8(q4s, vsubq_s8(q4s, q7)));
    q5 = vminq_u8(q0, q5);
    q4 = vminq_u8(q0, q4);

    q0 = vqaddq_u8(q9,  q5);
    q0 = vqsubq_u8(q0,  q4);
    q3 = vqsubq_u8(q10, q5);
    q3 = vqaddq_u8(q3,  q4);

    q6  = vrhaddq_u8(vhaddq_u8(q9, q11), q8);
    q7u = vrhaddq_u8(vhaddq_u8(q8, q10), q11);

    q0 = vbslq_u8(q1,  q6, q0 );
    q3 = vbslq_u8(q1, q7u, q3 );
    q9 = vbslq_u8(q2,  q0, q9 );
    q10= vbslq_u8(q2,  q3, q10);

    tr6 = vtrnq_u8(q8,  q9);
    tr7 = vtrnq_u8(q10, q11);

    tr4 = vtrnq_s16(vreinterpretq_s16_u8(tr6.val[0]), vreinterpretq_s16_u8(tr7.val[0]));
    tr5 = vtrnq_s16(vreinterpretq_s16_u8(tr6.val[1]), vreinterpretq_s16_u8(tr7.val[1]));

    tr0 = vtrn_s32(vget_low_s32(vreinterpretq_s32_s16(tr4.val[0])), vget_high_s32(vreinterpretq_s32_s16(tr4.val[0])));
    tr1 = vtrn_s32(vget_low_s32(vreinterpretq_s32_s16(tr5.val[0])), vget_high_s32(vreinterpretq_s32_s16(tr5.val[0])));
    tr2 = vtrn_s32(vget_low_s32(vreinterpretq_s32_s16(tr4.val[1])), vget_high_s32(vreinterpretq_s32_s16(tr4.val[1])));
    tr3 = vtrn_s32(vget_low_s32(vreinterpretq_s32_s16(tr5.val[1])), vget_high_s32(vreinterpretq_s32_s16(tr5.val[1])));

#if 0
    // unaligned store fools Android NDK 15 optimizer
    *(int32_t*)(uint8_t*)(pix - 2 + 0*stride) = vget_lane_s32(tr0.val[0], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 1*stride) = vget_lane_s32(tr1.val[0], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 2*stride) = vget_lane_s32(tr2.val[0], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 3*stride) = vget_lane_s32(tr3.val[0], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 4*stride) = vget_lane_s32(tr0.val[1], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 5*stride) = vget_lane_s32(tr1.val[1], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 6*stride) = vget_lane_s32(tr2.val[1], 0);
    *(int32_t*)(uint8_t*)(pix - 2 + 7*stride) = vget_lane_s32(tr3.val[1], 0);
#else
    vst1_lane_s16((int16_t*)(pix - 2 + 0*stride),     vreinterpret_s16_s32(tr0.val[0]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 0*stride) + 1, vreinterpret_s16_s32(tr0.val[0]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 1*stride),     vreinterpret_s16_s32(tr1.val[0]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 1*stride) + 1, vreinterpret_s16_s32(tr1.val[0]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 2*stride),     vreinterpret_s16_s32(tr2.val[0]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 2*stride) + 1, vreinterpret_s16_s32(tr2.val[0]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 3*stride),     vreinterpret_s16_s32(tr3.val[0]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 3*stride) + 1, vreinterpret_s16_s32(tr3.val[0]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 4*stride),     vreinterpret_s16_s32(tr0.val[1]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 4*stride) + 1, vreinterpret_s16_s32(tr0.val[1]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 5*stride),     vreinterpret_s16_s32(tr1.val[1]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 5*stride) + 1, vreinterpret_s16_s32(tr1.val[1]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 6*stride),     vreinterpret_s16_s32(tr2.val[1]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 6*stride) + 1, vreinterpret_s16_s32(tr2.val[1]), 1);
    vst1_lane_s16((int16_t*)(pix - 2 + 7*stride),     vreinterpret_s16_s32(tr3.val[1]), 0);
    vst1_lane_s16((int16_t*)(pix - 2 + 7*stride) + 1, vreinterpret_s16_s32(tr3.val[1]), 1);
#endif
}
}

static void deblock_chroma_h_neon(uint8_t *pix, int32_t stride, int a, int b, const uint8_t *thr, const uint8_t *str)
{
    uint8x16_t q0;
    uint8x16_t q8  = vld1q_u8(pix - 2*stride);
    uint8x16_t q9  = vld1q_u8(pix - 1*stride);
    uint8x16_t q10 = vld1q_u8(pix);
    uint8x16_t q11 = vld1q_u8(pix + stride);
    uint8x16_t q1  = vabdq_u8(q9, q10);
    uint8x16_t q2  = vcltq_u8(q1, vdupq_n_u8(a));
    uint8x16_t q4  = vmaxq_u8(vabdq_u8(q10, q11), vabdq_u8(q8, q9));
    uint8x16_t q3;
    uint8x16_t q6;
     int8x16_t q4s;
     int8x16_t q7;
    uint8x16_t q7u;
    uint8x16_t q5;
    uint8x16_t vstr = vld1q_u8(str);
    uint8x16_t vthr = vld1q_u8(thr);

    q4 = vcltq_u8(q4, vdupq_n_u8(b));
    q2 = vandq_u8(q2, q4);
    q1 = vzipq_u8(vstr, vstr).val[0];
    q3 = vcgtq_s8(vreinterpretq_s8_u8(q1), vdupq_n_s8(0));
    q1 = vshrq_n_u8(q1, 2);
    q1 = vcgtq_s8(vreinterpretq_s8_u8(q1), vdupq_n_s8(0));
    q2 = vandq_u8(q2, q3);

    q0 = vzipq_u8(vthr, vthr).val[0];
    q0 = vaddq_u8(q0, vdupq_n_u8(1));
    q0 = vandq_u8(q0, q2);

    q7 = vshrq_n_s8(vreinterpretq_s8_u8(vhsubq_u8(q8, q11)), 1);
    q6 = vandq_u8(vdupq_n_u8(1), veorq_u8(q10, q9));
    q4 = vhsubq_u8(q10, q9);
    q7 = vrhaddq_s8(q7, vreinterpretq_s8_u8(q6));
    q7 = vqaddq_s8(vreinterpretq_s8_u8(q4), q7);

    q4s = vdupq_n_s8(0);
    q5 = vreinterpretq_u8_s8(vmaxq_s8(q4s,               q7));
    q4 = vreinterpretq_u8_s8(vmaxq_s8(q4s, vsubq_s8(q4s, q7)));
    q5 = vminq_u8(q0, q5);
    q4 = vminq_u8(q0, q4);

    q0 = vqaddq_u8(q9,  q5);
    q0 = vqsubq_u8(q0,  q4);
    q3 = vqsubq_u8(q10, q5);
    q3 = vqaddq_u8(q3,  q4);

    q6  = vrhaddq_u8(vhaddq_u8(q9, q11), q8);
    q7u = vrhaddq_u8(vhaddq_u8(q8, q10), q11);

    q0 = vbslq_u8(q1,  q6, q0 );
    q3 = vbslq_u8(q1, q7u, q3 );
    q9 = vbslq_u8(q2,  q0, q9 );
    q10= vbslq_u8(q2,  q3, q10);

    vst1_u8(pix - stride, vget_low_u8(q9));
    vst1_u8(pix,          vget_low_u8(q10));
}

static void h264e_deblock_chroma_neon(uint8_t *pix, int32_t stride, const deblock_params_t *par)
{
    const uint8_t *alpha = par->alpha;
    const uint8_t *beta = par->beta;
    const uint8_t *thr = par->tc0;
    const uint8_t *strength = (uint8_t *)par->strength32;
    int a, b, x, y;
    a = alpha[0];
    b = beta[0];
    for (x = 0; x < 16; x += 8)
    {
        uint32_t str = *(uint32_t*)&strength[x];
        if (str && a)
        {
            deblock_chroma_v_neon(pix + (x >> 1), stride, a, b, thr + x, strength + x);
        }
        a = alpha[1];
        b = beta[1];
    }
    thr += 16;
    strength += 16;
    a = alpha[2];
    b = beta[2];
    for (y = 0; y < 16; y += 8)
    {
        uint32_t str = *(uint32_t*)&strength[y];
        if (str && a)
        {
            deblock_chroma_h_neon(pix, stride, a, b, thr + y, strength + y);
        }
        pix += 4*stride;
        a = alpha[3];
        b = beta[3];
    }
}

static void h264e_deblock_luma_neon(uint8_t *pix, int32_t stride, const deblock_params_t *par)
{
    const uint8_t *alpha = par->alpha;
    const uint8_t *beta = par->beta;
    const uint8_t *thr = par->tc0;
    const uint8_t *strength = (uint8_t *)par->strength32;
    int a = alpha[0];
    int b = beta[0];
    int x, y;
    for (x = 0; x < 16; x += 4)
    {
        uint32_t str = *(uint32_t*)&strength[x];
        if ((uint8_t)str == 4)
        {
            deblock_luma_v_s4_neon(pix + x, stride, a, b);
        } else if (str && a)
        {
            deblock_luma_v_neon(pix + x, stride, a, b, thr + x, strength + x);
        }
        a = alpha[1];
        b = beta[1];
    }
    a = alpha[2];
    b = beta[2];
    thr += 16;
    strength += 16;
    for (y = 0; y < 16; y += 4)
    {
        uint32_t str = *(uint32_t*)&strength[y];
        if ((uint8_t)str == 4)
        {
            deblock_luma_h_s4_neon(pix, stride, a, b);
        } else if (str && a)
        {
            deblock_luma_h_neon(pix, stride, a, b, thr + y, strength + y);
        }
        a = alpha[3];
        b = beta[3];
        pix += 4*stride;
    }
}

static void h264e_denoise_run_neon(unsigned char *frm, unsigned char *frmprev, int w, int h_arg, int stride_frm, int stride_frmprev)
{
    int cloop, h = h_arg;
    if (w <= 2 || h <= 2)
    {
        return;
    }
    w -= 2;
    h -= 2;

    do
    {
        unsigned char *pf = frm += stride_frm;
        unsigned char *pp = frmprev += stride_frmprev;
        cloop = w;
        pp[-stride_frmprev] = *pf++;
        pp++;

        for (;cloop >= 8; cloop -= 8, pf += 8, pp += 8)
        {
            uint16x8_t vp0w;
            uint32x4_t vpr0;
            uint32x4_t vpr1;
            uint16x8_t vf0w;
            int16x8_t vcls, vt, vcl, vgn, vgd;
            uint16x8_t vg;
            uint8x8_t vf0 = vld1_u8(pf);
            uint8x8_t vft = vld1_u8(pf - stride_frm);
            uint8x8_t vfb = vld1_u8(pf + stride_frm);
            uint8x8_t vfl = vld1_u8(pf - 1);
            uint8x8_t vfr = vld1_u8(pf + 1);
            uint8x8_t vp0 = vld1_u8(pp);
            uint8x8_t vpt = vld1_u8(pp - stride_frmprev);
            uint8x8_t vpb = vld1_u8(pp + stride_frmprev);
            uint8x8_t vpl = vld1_u8(pp - 1);
            uint8x8_t vpr = vld1_u8(pp + 1);
            uint16x8_t vd  = vabdl_u8(vf0, vp0);
            uint16x8_t vfs = vaddw_u8(vaddw_u8(vaddl_u8(vft, vfb), vfl), vfr);
            uint16x8_t vps = vaddw_u8(vaddw_u8(vaddl_u8(vpt, vpb), vpl), vpr);
            uint16x8_t vneighbourhood = vshrq_n_u16(vabdq_u16(vfs, vps), 2);

            vt = vaddq_s16(vreinterpretq_s16_u16(vd), vdupq_n_s16(1));

            vt = vqshlq_n_s16(vt, 7);
            vcls = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcls);
            vt = vqdmulhq_s16(vt,vt);                             // 1

            vcl = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);
            vt = vqdmulhq_s16(vt,vt);                             // 2
            vcl = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);
            vt = vqdmulhq_s16(vt,vt);                             // 3
            vcl = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);
            vt = vqdmulhq_s16(vt,vt);                             // 4
            vcl = vclsq_s16(vt);
            // vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);

            vgd = vsubq_s16(vdupq_n_s16(127), vcls);

            // same as above
            vt = vaddq_s16(vreinterpretq_s16_u16(vneighbourhood), vdupq_n_s16(1));

            vt = vqshlq_n_s16(vt, 7);
            vcls = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcls);
            vt = vqdmulhq_s16(vt,vt);                             // 1
            vcl = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);
            vt = vqdmulhq_s16(vt,vt);                             // 2
            vcl = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);
            vt = vqdmulhq_s16(vt,vt);                             // 3
            vcl = vclsq_s16(vt);
            vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);
            vt = vqdmulhq_s16(vt,vt);                             // 4
            vcl = vclsq_s16(vt);
            // vt = vshlq_s16(vt, vcl);
            vcls = vaddq_s16(vaddq_s16(vcls, vcls), vcl);

            vgn = vsubq_s16(vdupq_n_s16(127), vcls);

            vgn = vreinterpretq_s16_u16(vshrq_n_u16(vqshlq_n_u16(vreinterpretq_u16_s16(vgn), 10), 8));            // <<=2, saturated

            vgd = vsubq_s16(vdupq_n_s16(255), vgd);
            vgn = vsubq_s16(vdupq_n_s16(255), vgn);

            //vst1_u8(pp - stride_frmprev, vreinterpret_u8_s8(vmovn_s16(vgn)));
            //vst1_u8(pp - stride_frmprev, vreinterpret_u8_s8(vmovn_s16(vreinterpretq_s16_u16(vneighbourhood))));
            //vst1_u8(pp - stride_frmprev, vp0);

            vg  = vmulq_u16(vreinterpretq_u16_s16(vgn), vreinterpretq_u16_s16(vgd));

            vp0w = vmovl_u8(vp0);
            vpr0 = vmull_u16(vget_low_u16(vp0w), vget_low_u16(vg));
            vpr1 = vmull_u16(vget_high_u16(vp0w), vget_high_u16(vg));
            vg = vreinterpretq_u16_s16(vsubq_s16(vreinterpretq_s16_u8(vdupq_n_u8(255)), vreinterpretq_s16_u16(vg)));

            vf0w = vmovl_u8(vf0);
            vpr0 = vmlal_u16(vpr0, vget_low_u16(vf0w), vget_low_u16(vg));
            vpr1 = vmlal_u16(vpr1, vget_high_u16(vf0w), vget_high_u16(vg));

            vst1_u8(pp - stride_frmprev, vmovn_u16(vcombine_u16(vrshrn_n_u32(vpr0, 16), vrshrn_n_u32(vpr1, 16))));
        }

        while (cloop--)
        {
            int d, neighbourhood;
            unsigned g, gd, gn, out_val;
            d = pf[0] - pp[0];
            neighbourhood  = pf[-1] - pp[-1];
            neighbourhood += pf[+1] - pp[+1];
            neighbourhood += pf[-stride_frm] - pp[-stride_frmprev];
            neighbourhood += pf[+stride_frm] - pp[+stride_frmprev];

            if (d < 0)
            {
                d = -d;
            }
            if (neighbourhood < 0)
            {
                neighbourhood = -neighbourhood;
            }
            neighbourhood >>= 2;

            gd = g_diff_to_gainQ8[d];
            gn = g_diff_to_gainQ8[neighbourhood];

            gn <<= 2;
            if (gn > 255)
            {
                gn = 255;
            }

            gn = 255 - gn;
            gd = 255 - gd;
            g = gn*gd;  // Q8*Q8 = Q16;

            //out_val = ((pp[0]*g ) >> 16) + (((0xffff-g)*pf[0] ) >> 16);
            //out_val = ((pp[0]*g + (1<<15)) >> 16) + (((0xffff-g)*pf[0]  + (1<<15)) >> 16);
            out_val = (pp[0]*g + (0xffff - g)*pf[0]  + (1 << 15)) >> 16;

            assert(out_val <= 255);

            pp[-stride_frmprev] = (unsigned char)out_val;
            //pp[-stride_frmprev] = gn;
            //pp[-stride_frmprev] = neighbourhood;
            //pp[-stride_frmprev] = pp[0];

            pf++, pp++;
        }

        pp[-stride_frmprev] = *pf++;
    } while(--h);

    memcpy(frmprev + stride_frmprev, frm + stride_frm, w + 2);
    h = h_arg - 2;
    do
    {
        memcpy(frmprev, frmprev - stride_frmprev, w + 2);
        frmprev -= stride_frmprev;
    } while(--h);
    memcpy(frmprev, frm - stride_frm*(h_arg - 2), w + 2);
}

#undef IS_NULL
#define IS_NULL(p) ((p) < (pix_t *)32)

static uint32_t intra_predict_dc4_neon(const pix_t *left, const pix_t *top)
{
    unsigned dc = 0, side = 4, round = 0;
    uint32x2_t s = vdup_n_u32(0);

    if (!IS_NULL(left))
    {
        s = vpaddl_u16(vpaddl_u8(vld1_u8(left)));
        round += side >> 1;
    }
    if (!IS_NULL(top))
    {
        s = vadd_u32(s, vpaddl_u16(vpaddl_u8(vld1_u8(top))));
        round += side >> 1;
    }
    dc = vget_lane_u32(s, 0);

    dc += round;
    if (round == side) dc >>= 1;
    dc >>= 2;
    if (!round) dc = 128;
    return dc * 0x01010101;
}

static uint8x16_t intra_predict_dc16_neon(const pix_t *left, const pix_t *top)
{
    unsigned dc = 0, side = 16, round = 0;

    if (!IS_NULL(left))
    {
        uint8x16_t v = vld1q_u8(left);
        uint64x2_t s = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(v)));
        uint64x1_t q = vadd_u64(vget_high_u64(s), vget_low_u64(s));
        dc += vget_lane_u32(vreinterpret_u32_u64(q), 0);
        round += side >> 1;
    }
    if (!IS_NULL(top))
    {
        uint8x16_t v = vld1q_u8(top);
        uint64x2_t s = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(v)));
        uint64x1_t q = vadd_u64(vget_high_u64(s), vget_low_u64(s));
        dc += vget_lane_u32(vreinterpret_u32_u64(q), 0);
        round += side >> 1;
    }
    dc += round;
    if (round == side) dc >>= 1;
    dc >>= 4;
    if (!round) dc = 128;
    return vdupq_n_u8(dc);
}

/*
 * Note: To make the code more readable we refer to the neighboring pixels
 * in variables named as below:
 *
 *    UL U0 U1 U2 U3 U4 U5 U6 U7
 *    L0 xx xx xx xx
 *    L1 xx xx xx xx
 *    L2 xx xx xx xx
 *    L3 xx xx xx xx
 */
#define UL edge[-1]
#define U0 edge[0]
#define T1 edge[1]
#define U2 edge[2]
#define U3 edge[3]
#define U4 edge[4]
#define U5 edge[5]
#define U6 edge[6]
#define U7 edge[7]
#define L0 edge[-2]
#define L1 edge[-3]
#define L2 edge[-4]
#define L3 edge[-5]

static void h264e_intra_predict_16x16_neon(pix_t *predict, const pix_t *left, const pix_t *top, int mode)
{
    int cloop = 4;
    uint32_t *d = (uint32_t*)predict;
    uint32x4_t v;
    assert(IS_ALIGNED(predict, 4));
    assert(IS_ALIGNED(top, 4));
    if (mode != 1)
    {
        if (mode < 1)
        {
            v = vld1q_u32((uint32_t*)top);
        } else //(mode == 2)
        {
            v = vreinterpretq_u32_u8(intra_predict_dc16_neon(left, top));
        }
        do
        {
            vst1q_u32(d, v); d += 4;
            vst1q_u32(d, v); d += 4;
            vst1q_u32(d, v); d += 4;
            vst1q_u32(d, v); d += 4;
        } while (--cloop);
    } else //if (mode == 1)
    {
        do
        {
            vst1q_u8((uint8_t*)d, vdupq_n_u8(*left++)); d += 4;
            vst1q_u8((uint8_t*)d, vdupq_n_u8(*left++)); d += 4;
            vst1q_u8((uint8_t*)d, vdupq_n_u8(*left++)); d += 4;
            vst1q_u8((uint8_t*)d, vdupq_n_u8(*left++)); d += 4;
        } while (--cloop);
    }
}

static void h264e_intra_predict_chroma_neon(pix_t *predict, const pix_t *left, const pix_t *top, int mode)
{
    int cloop = 8;
    uint32_t *d = (uint32_t*)predict;
    uint32x4_t v;
    assert(IS_ALIGNED(predict, 4));
    assert(IS_ALIGNED(top, 4));
    if (mode < 1)
    {
        v = vld1q_u32((uint32_t*)top);
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
        vst1q_u32(d, v); d += 4;
    } else if (mode == 1)
    {
        do
        {
            v = vreinterpretq_u32_u8(vcombine_u8(vdup_n_u8(left[0]), vdup_n_u8(left[8])));
            vst1q_u32(d, v); d += 4;
            left++;
        } while(--cloop);
    } else //if (mode == 2)
    {
        int ccloop = 2;
        cloop = 2;
        do
        {
            d[0] = d[1] = d[16] = intra_predict_dc4_neon(left, top);
            d[17] = intra_predict_dc4_neon(left + 4, top + 4);
            if (!IS_NULL(top))
            {
                d[1] = intra_predict_dc4_neon(NULL, top + 4);
            }
            if (!IS_NULL(left))
            {
                d[16] = intra_predict_dc4_neon(NULL, left + 4);
            }
            d += 2;
            left += 8;
            top += 8;
        } while(--cloop);

        do
        {
            v = vld1q_u32(d - 4);
            vst1q_u32(d, v); d += 4;
            vst1q_u32(d, v); d += 4;
            vst1q_u32(d, v); d += 4;
            d += 4;
        } while(--ccloop);
    }
}

static __inline int vsad_neon(uint8x16_t a, uint8x16_t b)
{
    uint64x2_t s = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vabdq_u8(a, b))));
    uint64x1_t q = vadd_u64(vget_high_u64(s), vget_low_u64(s));
    return vget_lane_u32(vreinterpret_u32_u64(q), 0);
}

static int h264e_intra_choose_4x4_neon(const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty)
{
    int sad, best_sad, best_m = 2;

    uint32_t r0, r1, r2, r3;
    uint8x16_t vx, vt, vr, vpred, v1, v2, v8, v9, q1, q2, q10, q11, q12;
    uint8x8_t d2, d3;

    r0 = ((uint32_t *)blockin)[ 0];
    r1 = ((uint32_t *)blockin)[ 4];
    r2 = ((uint32_t *)blockin)[ 8];
    r3 = ((uint32_t *)blockin)[12];
    vr = vcombine_u8(vcreate_u8(((uint64_t)r1 << 32) | r0), vcreate_u8(((uint64_t)r3 << 32) | r2));

#define VTEST(mode) sad = vsad_neon(vr,vx);    \
            if (mode != mpred) sad += penalty; \
            if (sad < best_sad)                \
            {                                  \
                vpred = vx;                    \
                best_sad = sad;                \
                best_m = mode;                 \
            }

    // DC
    vx = vdupq_n_u8(intra_predict_dc4_neon((avail & AVAIL_L) ? &L3 : 0, (avail & AVAIL_T) ? &U0 : 0));

    best_sad = vsad_neon(vx, vr);
    if (2 != mpred)
    {
        best_sad += penalty;
    }
    vpred = vx;

    vt = vld1q_u8(&L3);
    vt = vreinterpretq_u8_u32(vsetq_lane_u32(U7*0x01010101, vreinterpretq_u32_u8(vt), 3));
    if (avail & AVAIL_T)
    {
        uint32x2_t t2;
        if (!(avail & AVAIL_TR))
        {
            vt = vcombine_u8(vget_low_u8(vt), vdup_n_u8(U3));
        }

        vx =  vreinterpretq_u8_u32(vdupq_n_u32(*(uint32_t*)&U0));
        VTEST(0);

        vx = vt;
        vx = vrhaddq_u8(vhaddq_u8(vextq_u8(vx, vx, 5), vextq_u8(vx, vx, 7)), vextq_u8(vx, vx, 6));

        v1 = vextq_u8(vx, vx, 1);
        d2 = vext_u8(vget_low_u8(vx), vget_low_u8(vx), 2);
        d3 = vext_u8(vget_low_u8(vx), vget_low_u8(vx), 3);
        vx = vreinterpretq_u8_u32(vcombine_u32(
            t2 = vzip_u32(vreinterpret_u32_u8(vget_low_u8(vx)), vreinterpret_u32_u8(vget_low_u8(v1))).val[0],
            vzip_u32(vreinterpret_u32_u8(d2), vreinterpret_u32_u8(d3)).val[0]));
        VTEST(3);

        vx = vt;
        vx = vrhaddq_u8(vextq_u8(vt, vt, 5), vextq_u8(vt, vt, 6));
        vx = vreinterpretq_u8_u32(vzipq_u32(vreinterpretq_u32_u8(vx), vreinterpretq_u32_u8(vextq_u8(vx, vx, 1))).val[0]);
        vx = vreinterpretq_u8_u32(vzipq_u32(vreinterpretq_u32_u8(vx),
        vreinterpretq_u32_u8(vcombine_u8(vreinterpret_u8_u32(t2), vget_high_u8(vextq_u8(vt, vt, 7))))).val[0]);

        VTEST(7);
    }

    if (avail & AVAIL_L)
    {
        vx = vrev32q_u8(vt);
        vx = vzipq_u8(vx, vx).val[0];
        vx = vzipq_u8(vx, vx).val[0];
        VTEST(1);

        v2 = vrev32q_u8(vt);
        v8 = vrev32q_u8(vt);
        vx = vrev32q_u8(vt);
        v8 = vzipq_u8(vx, vx).val[0];
        {
            int tmp = vgetq_lane_u16(vreinterpretq_u16_u8(v8), 3);
            v2 = vreinterpretq_u8_u16(vsetq_lane_u16(tmp, vreinterpretq_u16_u8(v2), 2));
            v8 = vreinterpretq_u8_u16(vsetq_lane_u16(tmp, vreinterpretq_u16_u8(v8), 4));
            v9 = vextq_u8(v2, v2, 14);
            v9 = vzipq_u8(v9, vhaddq_u8(v9, v2)).val[0];
            v9 = vrhaddq_u8(v9, vextq_u8(v8, v8, 14));
            tmp |= tmp << 16;
            vx = vreinterpretq_u8_u32(vzipq_u32(vreinterpretq_u32_u8(vextq_u8(v9, v9, 4)),
                                                vreinterpretq_u32_u8(vextq_u8(v9, v9, 6))).val[0]);
            vx = vreinterpretq_u8_u32(vsetq_lane_u32(tmp, vreinterpretq_u32_u8(vx), 3));
        }
        VTEST(8);
    }

    if ((avail & (AVAIL_T | AVAIL_L | AVAIL_TL)) == (AVAIL_T | AVAIL_L | AVAIL_TL))
    {
        uint32x2x2_t pair;
        uint8x8_t d4, d6;
        int lr;
        q11 = q2 = vrhaddq_u8(vhaddq_u8(vt, vextq_u8(vt, vt, 2)), q10 = vextq_u8(vt, vt, 1));
        d4 = vget_low_u8(q2);
        d6 = vreinterpret_u8_u32(vzip_u32(vreinterpret_u32_u8(vext_u8(d4, d4, 3)), vreinterpret_u32_u8(vext_u8(d4, d4, 1))).val[0]);
        d4 = vreinterpret_u8_u32(vzip_u32(vreinterpret_u32_u8(vext_u8(d4, d4, 2)), vreinterpret_u32_u8(d4)).val[0]);
        pair = vzip_u32(vreinterpret_u32_u8(d6), vreinterpret_u32_u8(d4));
        vx = vcombine_u8(vreinterpret_u8_u32(pair.val[0]), vreinterpret_u8_u32(pair.val[1]));
        VTEST(4);

        vx  = q12 = vrhaddq_u8(vt, q10);
        q1  = vzipq_u8(vx, q11).val[0];
        q1  = vreinterpretq_u8_u32(vzipq_u32(vreinterpretq_u32_u8(q1), vreinterpretq_u32_u8(vextq_u8(q1, q1, 2))).val[0]);
        q1  = vreinterpretq_u8_u32(vrev64q_u32(vreinterpretq_u32_u8(q1)));
        vx  = vcombine_u8(vget_high_u8(q1), vget_low_u8(q1));
        vx = vreinterpretq_u8_u16(
            vsetq_lane_u16(vgetq_lane_u16(vreinterpretq_u16_u8(q11), 2), vreinterpretq_u16_u8(vx), 1));
        VTEST(6);

        q11 = vextq_u8(q11, q11, 1);
        q1  = vextq_u8(q12, q12, 4);
        q2  = vextq_u8(q11, q11, 2);
        q1  = vreinterpretq_u8_u32(vzipq_u32(vreinterpretq_u32_u8(q1), vreinterpretq_u32_u8(q2)).val[0]);
        q12 = vreinterpretq_u8_u16(vsetq_lane_u16(lr = vgetq_lane_u16(vreinterpretq_u16_u8(q11), 0), vreinterpretq_u16_u8(q12), 1));
        q11 = vreinterpretq_u8_u16(vsetq_lane_u16((lr << 8) & 0xffff, vreinterpretq_u16_u8(q11), 0));
        vx = vcombine_u8(vget_low_u8(q1), vreinterpret_u8_u32(vzip_u32(
            vreinterpret_u32_u8(vext_u8(vget_low_u8(q12), vget_low_u8(q12), 3)),
            vreinterpret_u32_u8(vext_u8(vget_low_u8(q11), vget_low_u8(q11), 1))
            ).val[0]));
        VTEST(5);
    }

    vst1q_lane_u32(((uint32_t *)blockpred) + 0, vreinterpretq_u32_u8(vpred ), 0);
    vst1q_lane_u32(((uint32_t *)blockpred) + 4, vreinterpretq_u32_u8(vpred ), 1);
    vst1q_lane_u32(((uint32_t *)blockpred) + 8, vreinterpretq_u32_u8(vpred ), 2);
    vst1q_lane_u32(((uint32_t *)blockpred) +12, vreinterpretq_u32_u8(vpred ), 3);
    return best_m + (best_sad << 4); // pack result
}

static void copy_wh_neon(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    if (w == 4)
    {
        do
        {
            *(int32_t*)dst = *(int32_t*)src; dst += 16; src += src_stride;
            *(int32_t*)dst = *(int32_t*)src; dst += 16; src += src_stride;
            *(int32_t*)dst = *(int32_t*)src; dst += 16; src += src_stride;
            *(int32_t*)dst = *(int32_t*)src; dst += 16; src += src_stride;
        } while (h -= 4);
    } else if (w == 8)
    {
        do
        {
            vst1_u8(dst, vld1_u8(src)); dst += 16; src += src_stride;
            vst1_u8(dst, vld1_u8(src)); dst += 16; src += src_stride;
            vst1_u8(dst, vld1_u8(src)); dst += 16; src += src_stride;
            vst1_u8(dst, vld1_u8(src)); dst += 16; src += src_stride;
        } while (h -= 4);
    } else
    {
        do
        {
            uint8x16_t v0, v1, v2, v3;
            v0 = vld1q_u8(src); src += src_stride;
            v1 = vld1q_u8(src); src += src_stride;
            v2 = vld1q_u8(src); src += src_stride;
            v3 = vld1q_u8(src); src += src_stride;

            vst1q_u8(dst, v0); dst += 16;
            vst1q_u8(dst, v1); dst += 16;
            vst1q_u8(dst, v2); dst += 16;
            vst1q_u8(dst, v3); dst += 16;
        } while (h -= 4);
    }
}

static void hpel_lpf_hor_neon(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    uint8x8_t c5 = vdup_n_u8(5);
    uint8x8_t c20 = vshl_n_u8(c5, 2);
    if (w == 16)
    {
        do
        {
            uint8x16_t s0 = vld1q_u8(src - 2);
            uint8x16_t s1 = vld1q_u8(src - 2 + 16);
            uint8x16_t v0 = s0;
            uint8x16_t v1 = vextq_u8(s0, s1, 1);
            uint8x16_t v2 = vextq_u8(s0, s1, 2);
            uint8x16_t v3 = vextq_u8(s0, s1, 3);
            uint8x16_t v4 = vextq_u8(s0, s1, 4);
            uint8x16_t v5 = vextq_u8(s0, s1, 5);

            uint16x8_t q, s = vaddl_u8(vget_low_u8(v0), vget_low_u8(v5));
            s = vmlsl_u8(s, vget_low_u8(v1), c5);
            s = vmlsl_u8(s, vget_low_u8(v4), c5);
            s = vmlal_u8(s, vget_low_u8(v2), c20);
            s = vmlal_u8(s, vget_low_u8(v3), c20);

            q = vaddl_u8(vget_high_u8(v0), vget_high_u8(v5));
            q = vmlsl_u8(q, vget_high_u8(v1), c5);
            q = vmlsl_u8(q, vget_high_u8(v4), c5);
            q = vmlal_u8(q, vget_high_u8(v2), c20);
            q = vmlal_u8(q, vget_high_u8(v3), c20);

            vst1q_u8(dst, vcombine_u8(
                vqrshrun_n_s16(vreinterpretq_s16_u16(s), 5),
                vqrshrun_n_s16(vreinterpretq_s16_u16(q), 5)));

            dst += 16;
            src += src_stride;
        } while (--h);
    } else
    {
        do
        {
            uint8x16_t line = vld1q_u8(src - 2);
            uint8x8_t s0 = vget_low_u8(line);
            uint8x8_t s1 = vget_high_u8(line);
            uint8x8_t v0 = s0;
            uint8x8_t v1 = vext_u8(s0, s1, 1);
            uint8x8_t v2 = vext_u8(s0, s1, 2);
            uint8x8_t v3 = vext_u8(s0, s1, 3);
            uint8x8_t v4 = vext_u8(s0, s1, 4);
            uint8x8_t v5 = vext_u8(s0, s1, 5);

            uint16x8_t s = vaddl_u8(v0, v5);
            s = vmlsl_u8(s, v1, c5);
            s = vmlsl_u8(s, v4, c5);
            s = vmlal_u8(s, v2, c20);
            s = vmlal_u8(s, v3, c20);

            vst1_u8(dst, vqrshrun_n_s16(vreinterpretq_s16_u16(s), 5));

            dst += 16;
            src += src_stride;
        } while (--h);
    }
}

static void hpel_lpf_hor16_neon(const uint8_t *src, int src_stride, int16_t *h264e_restrict dst, int w, int h)
{
    uint8x8_t c5 = vdup_n_u8(5);
    uint8x8_t c20 = vshl_n_u8(c5, 2);
    if (w == 16)
    {
        do
        {
            uint8x16_t s0 = vld1q_u8(src - 2);
            uint8x16_t s1 = vld1q_u8(src - 2 + 16);
            uint8x16_t v0 = s0;
            uint8x16_t v1 = vextq_u8(s0, s1, 1);
            uint8x16_t v2 = vextq_u8(s0, s1, 2);
            uint8x16_t v3 = vextq_u8(s0, s1, 3);
            uint8x16_t v4 = vextq_u8(s0, s1, 4);
            uint8x16_t v5 = vextq_u8(s0, s1, 5);

            uint16x8_t q, s = vaddl_u8(vget_low_u8(v0), vget_low_u8(v5));
            s = vmlsl_u8(s, vget_low_u8(v1), c5);
            s = vmlsl_u8(s, vget_low_u8(v4), c5);
            s = vmlal_u8(s, vget_low_u8(v2), c20);
            s = vmlal_u8(s, vget_low_u8(v3), c20);

            q = vaddl_u8(vget_high_u8(v0), vget_high_u8(v5));
            q = vmlsl_u8(q, vget_high_u8(v1), c5);
            q = vmlsl_u8(q, vget_high_u8(v4), c5);
            q = vmlal_u8(q, vget_high_u8(v2), c20);
            q = vmlal_u8(q, vget_high_u8(v3), c20);

            vst1q_s16(dst, vreinterpretq_s16_u16(s));
            vst1q_s16(dst + 8, vreinterpretq_s16_u16(q));

            dst += 16;
            src += src_stride;
        } while (--h);
    } else
    {
        do
        {
            uint8x16_t line = vld1q_u8(src - 2);
            uint8x8_t s0 = vget_low_u8(line);
            uint8x8_t s1 = vget_high_u8(line);
            uint8x8_t v0 = s0;
            uint8x8_t v1 = vext_u8(s0, s1,  1);
            uint8x8_t v2 = vext_u8(s0, s1, 2);
            uint8x8_t v3 = vext_u8(s0, s1, 3);
            uint8x8_t v4 = vext_u8(s0, s1, 4);
            uint8x8_t v5 = vext_u8(s0, s1, 5);

            uint16x8_t s = vaddl_u8(v0, v5);
            s = vmlsl_u8(s, v1, c5);
            s = vmlsl_u8(s, v4, c5);
            s = vmlal_u8(s, v2, c20);
            s = vmlal_u8(s, v3, c20);

            vst1q_s16(dst, vreinterpretq_s16_u16(s));

            dst += 16;
            src += src_stride;
        } while (--h);
    }
}

static void hpel_lpf_ver_neon(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    uint8x8_t c5 = vdup_n_u8(5);
    uint8x8_t c20 = vshl_n_u8(c5, 2);

    if (w == 16)
    {
        uint8x16_t v0 = vld1q_u8(src - 2*src_stride);
        uint8x16_t v1 = vld1q_u8(src - 1*src_stride);
        uint8x16_t v2 = vld1q_u8(src);
        uint8x16_t v3 = vld1q_u8(src + 1*src_stride);
        uint8x16_t v4 = vld1q_u8(src + 2*src_stride);
        do
        {
            uint8x16_t v5 = vld1q_u8(src + 3*src_stride);
            uint16x8_t q, s = vaddl_u8(vget_low_u8(v0), vget_low_u8(v5));
            s = vmlsl_u8(s, vget_low_u8(v1), c5);
            s = vmlsl_u8(s, vget_low_u8(v4), c5);
            s = vmlal_u8(s, vget_low_u8(v2), c20);
            s = vmlal_u8(s, vget_low_u8(v3), c20);

            q = vaddl_u8(vget_high_u8(v0), vget_high_u8(v5));
            q = vmlsl_u8(q, vget_high_u8(v1), c5);
            q = vmlsl_u8(q, vget_high_u8(v4), c5);
            q = vmlal_u8(q, vget_high_u8(v2), c20);
            q = vmlal_u8(q, vget_high_u8(v3), c20);

            vst1q_u8(dst, vcombine_u8(
                vqrshrun_n_s16(vreinterpretq_s16_u16(s), 5),
                vqrshrun_n_s16(vreinterpretq_s16_u16(q), 5)));
            dst += 16;
            src += src_stride;
            v0 = v1;
            v1 = v2;
            v2 = v3;
            v3 = v4;
            v4 = v5;
        } while (--h);
    } else
    {
        uint8x8_t v0 = vld1_u8(src - 2*src_stride);
        uint8x8_t v1 = vld1_u8(src - 1*src_stride);
        uint8x8_t v2 = vld1_u8(src);
        uint8x8_t v3 = vld1_u8(src + 1*src_stride);
        uint8x8_t v4 = vld1_u8(src + 2*src_stride);
        do
        {
            uint8x8_t v5 = vld1_u8(src + 3*src_stride);
            uint16x8_t s = vaddl_u8(v0, v5);
            s = vmlsl_u8(s, v1, c5);
            s = vmlsl_u8(s, v4, c5);
            s = vmlal_u8(s, v2, c20);
            s = vmlal_u8(s, v3, c20);

            vst1_u8(dst, vqrshrun_n_s16(vreinterpretq_s16_u16(s), 5));
            dst += 16;
            src += src_stride;
            v0 = v1;
            v1 = v2;
            v2 = v3;
            v3 = v4;
            v4 = v5;
        } while (--h);
    }
}

static void hpel_lpf_ver16_neon(const int16_t *src, uint8_t *h264e_restrict dst, int w, int h)
{
    do
    {
        int cloop = h;
        int16x8_t v0 = vld1q_s16(src);
        int16x8_t v1 = vld1q_s16(src + 16);
        int16x8_t v2 = vld1q_s16(src + 16*2);
        int16x8_t v3 = vld1q_s16(src + 16*3);
        int16x8_t v4 = vld1q_s16(src + 16*4);
        do
        {
            int16x8_t v5 = vld1q_s16(src+16*5);

            int16x8_t s0 = vaddq_s16(v0, v5);
            int16x8_t s1 = vaddq_s16(v1, v4);
            int16x8_t s2 = vaddq_s16(v2, v3);

            int16x8_t vs = vshrq_n_s16(vsubq_s16(s0, s1), 2);
            int16x8_t vq = vsubq_s16(s2, s1);
            s0 = vshrq_n_s16(vaddq_s16(vq, vs), 2);
            s0 = vaddq_s16(s0, s2);

            vst1_u8(dst, vqrshrun_n_s16(s0, 6));

            dst += 16;
            src += 16;
            v0 = v1;
            v1 = v2;
            v2 = v3;
            v3 = v4;
            v4 = v5;
        } while (--cloop);

        src -= 16*h - 8;
        dst -= 16*h - 8;
    } while (w -= 8);
}

static void hpel_lpf_diag_neon(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    ALIGN(16) int16_t scratch[21 * 16] ALIGN2(16);  /* 21 rows by 16 pixels per row */

    /*
     * Intermediate values will be 1/2 pel at Horizontal direction
     * Starting at (0.5, -2) at top extending to (0.5, height + 3) at bottom
     * scratch contains a 2D array of size (w)X(h + 5)
     */
    hpel_lpf_hor16_neon(src - 2*src_stride, src_stride, scratch, w, h + 5);
    hpel_lpf_ver16_neon(scratch, dst, w, h);
}

static void average_16x16_unalign_neon(uint8_t *dst, const uint8_t *src, int src_stride)
{
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
    vst1q_u8(dst, vrhaddq_u8(vld1q_u8(dst), vld1q_u8(src)));  src += src_stride; dst += 16;
}

static void h264e_qpel_average_wh_align_neon(const uint8_t *src0, const uint8_t *src1, uint8_t *dst, point_t wh)
{
    int w = wh.s.x;
    int h = wh.s.y;
    int cloop = h;
    if (w == 8)
    {
        do
        {
            vst1_u8(dst, vrhadd_u8(vld1_u8(src0), vld1_u8(src1)));
            dst += 16;
            src0 += 16;
            src1 += 16;
        } while (--cloop);
    } else
    {
        do
        {
            vst1q_u8(dst, vrhaddq_u8(vld1q_u8(src0), vld1q_u8(src1)));
            dst += 16;
            src0 += 16;
            src1 += 16;
        } while (--cloop);
    }
}

static void h264e_qpel_interpolate_luma_neon(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, point_t wh, point_t dxdy)
{
//    src += ((dx + 1) >> 2) + ((dy + 1) >> 2)*src_stride;            // dx == 3 ? next row; dy == 3 ? next line
//    dxdy              actions: Horizontal, Vertical, Diagonal, Average
//    0 1 2 3 +1        -   ha    h    ha+
//    1                 va  hva   hda  hv+a
//    2                 v   vda   d    v+da
//    3                 va+ h+va h+da  h+v+a
//    +stride
    int32_t pos = 1 << (dxdy.s.x + 4*dxdy.s.y);

    if (pos == 1)
    {
        copy_wh_neon(src, src_stride, dst, wh.s.x, wh.s.y);
    } else
    {
        ALIGN(16) uint8_t scratch[16*16] ALIGN2(16);
        int dstused = 0;
        if (pos & 0xe0ee)// 1110 0000 1110 1110
        {
            hpel_lpf_hor_neon(src + ((pos & 0xe000) ? src_stride : 0), src_stride, dst, wh.s.x, wh.s.y);
            dstused++;
        }
        if (pos & 0xbbb0)// 1011 1011 1011 0000
        {
            hpel_lpf_ver_neon(src + ((pos & 0x8880) ? 1 : 0), src_stride, dstused ? scratch : dst, wh.s.x, wh.s.y);
            dstused++;
        }
        if (pos & 0x4e40)// 0100 1110 0100 0000
        {
            hpel_lpf_diag_neon(src, src_stride, dstused ? scratch : dst, wh.s.x, wh.s.y);
            dstused++;
        }
        if (pos & 0xfafa)// 1111 1010 1111 1010
        {
            assert(wh.s.x == 16 && wh.s.y == 16);
            if (dstused == 2)
            {
                point_t p;

                src = scratch;
                src_stride = 16;
                p.u32 = 16 + (16 << 16);

                h264e_qpel_average_wh_align_neon(src, dst, dst, p);
            } else
            {
                src += ((dxdy.s.x + 1) >> 2) + ((dxdy.s.y + 1) >> 2)*src_stride;
                average_16x16_unalign_neon(dst, src, src_stride);
            }
        }
    }
}

static void h264e_qpel_interpolate_chroma_neon(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, point_t wh, point_t dxdy)
{
    /* if fractionl mv is not (0, 0) */
    if (dxdy.u32)
    {
        uint8x8_t v8 = vdup_n_u8(8);
        uint8x8_t vx = vdup_n_u8(dxdy.s.x);
        uint8x8_t vy = vdup_n_u8(dxdy.s.y);
        uint8x8_t v8x = vsub_u8(v8, vx);
        uint8x8_t v8y = vsub_u8(v8, vy);
        uint8x8_t va = vmul_u8(v8x, v8y);
        uint8x8_t vb = vmul_u8(vx, v8y);
        uint8x8_t vc = vmul_u8(v8x, vy);
        uint8x8_t vd = vmul_u8(vx, vy);
        int h = wh.s.y;
        if (wh.s.x == 8)
        {
            uint8x16_t vt0 = vld1q_u8(src);
            uint8x16_t vt1 = vextq_u8(vt0, vt0, 1);
            src += src_stride;
            do
            {
                uint8x16_t vb0 = vld1q_u8(src);
                uint8x16_t vb1 = vextq_u8(vb0, vb0, 1);
                uint16x8_t vs = vmull_u8(vget_low_u8(vt0), va);
                vs = vmlal_u8(vs, vget_low_u8(vt1), vb);
                vs = vmlal_u8(vs, vget_low_u8(vb0), vc);
                vs = vmlal_u8(vs, vget_low_u8(vb1), vd);
                vst1_u8(dst, vqrshrun_n_s16(vreinterpretq_s16_u16(vs), 6));
                vt0 = vb0;
                vt1 = vb1;
                dst += 16;
                src += src_stride;
             } while(--h);
         } else
         {
            uint8x8_t vt0 = vld1_u8(src);
            uint8x8_t vt1 = vext_u8(vt0, vt0, 1);
            src += src_stride;
            do
            {
                uint8x8_t vb0 = vld1_u8(src);
                uint8x8_t vb1 = vext_u8(vb0, vb0, 1);
                uint16x8_t vs = vmull_u8(vt0, va);
                vs = vmlal_u8(vs, vt1, vb);
                vs = vmlal_u8(vs, vb0, vc);
                vs = vmlal_u8(vs, vb1, vd);
                *(int32_t*)dst = vget_lane_s32(vreinterpret_s32_u8(vqrshrun_n_s16(vreinterpretq_s16_u16(vs), 6)), 0);
                vt0 = vb0;
                vt1 = vb1;
                dst += 16;
                src += src_stride;
             } while(--h);
         }
    } else
    {
        copy_wh_neon(src, src_stride, dst, wh.s.x, wh.s.y);
    }
}

static int h264e_sad_mb_unlaign_8x8_neon(const pix_t *a, int a_stride, const pix_t *b, int _sad[4])
{
    uint16x8_t s0, s1;
    uint8x16_t va, vb;
    int cloop = 2, sum = 0;
    do
    {
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabdl_u8(    vget_low_u8(va), vget_low_u8(vb));   s1 = vabdl_u8(    vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
        s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
        {
        uint32x4_t v0 = vpaddlq_u16(s0);
        uint64x2_t v1 = vpaddlq_u32(v0);
        sum += _sad[0] = (int)(vgetq_lane_u64(v1, 0)+vgetq_lane_u64(v1, 1));
        v0 = vpaddlq_u16(s1);
        v1 = vpaddlq_u32(v0);
        sum += _sad[1] = (int)(vgetq_lane_u64(v1, 0)+vgetq_lane_u64(v1, 1));
        _sad += 2;
        }
    } while(--cloop);
    return sum;
}

static int h264e_sad_mb_unlaign_wh_neon(const pix_t *a, int a_stride, const pix_t *b, point_t wh)
{
    uint16x8_t s0, s1;
    uint8x16_t va, vb;
    int cloop = wh.s.y/8, sum = 0;
    if (wh.s.x == 16)
    {
        do
        {
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabdl_u8(    vget_low_u8(va), vget_low_u8(vb));   s1 = vabdl_u8(    vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));   s1 = vabal_u8(s1, vget_high_u8(va), vget_high_u8(vb));

            uint32x4_t v0 = vpaddlq_u16(s0);
            uint64x2_t v1 = vpaddlq_u32(v0);
            sum += vgetq_lane_u64(v1, 0) + vgetq_lane_u64(v1, 1);

            v0 = vpaddlq_u16(s1);
            v1 = vpaddlq_u32(v0);
            sum += vgetq_lane_u64(v1, 0) + vgetq_lane_u64(v1, 1);
        } while(--cloop);
    } else
    {
        do
        {
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabdl_u8(    vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));
            va = vld1q_u8(a), vb = vld1q_u8(b);  a += a_stride, b += 16;
            s0 = vabal_u8(s0, vget_low_u8(va), vget_low_u8(vb));

            uint32x4_t v0 = vpaddlq_u16(s0);
            uint64x2_t v1 = vpaddlq_u32(v0);
            sum += vgetq_lane_u64(v1, 0) + vgetq_lane_u64(v1, 1);
        } while(--cloop);
    }
    return sum;
}

static void h264e_copy_8x8_neon(pix_t *d, int d_stride, const pix_t *s)
{
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;

    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
    vst1_u8(d, vld1_u8(s)); s += 16;  d += d_stride;
}


static void h264e_copy_16x16_neon(pix_t *d, int d_stride, const pix_t *s, int s_stride)
{
    assert(!((unsigned)d & 7));
    assert(!((unsigned)s & 7));
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;

    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;

    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;

    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
    vst1q_u8(d, vld1q_u8(s)); s += s_stride; d += d_stride;
}

// Keep intermediate data in transposed format.
// Save transpose for vectorized implementation
// TODO: TRANSPOSE_BLOCK==0 broken
#define TRANSPOSE_BLOCK     0
#define UNZIGSAG_IN_QUANT   0

#define SUM_DIF(a, b) { int t = a + b; b = a - b; a = t; }

static void hadamar4_2d_neon(int16_t *x)
{
    int16x8_t q0 = vld1q_s16(x);
    int16x8_t q1 = vld1q_s16(x + 8);
    int16x8_t s = vaddq_s16(q0, q1);
    int16x8_t d = vsubq_s16(q0, q1);
    int16x8_t q2 = vcombine_s16(vget_low_s16(s), vget_low_s16(d));
    int16x8_t q3 = vcombine_s16(vget_high_s16(s), vget_high_s16(d));
    q0 = vaddq_s16(q2, q3);
    d  = vsubq_s16(q2, q3);
    q1 = vcombine_s16(vget_high_s16(d), vget_low_s16(d));
{
    int16x4x2_t t0 = vtrn_s16(vget_low_s16(q0), vget_high_s16(q0));
    int16x4x2_t t1 = vtrn_s16(vget_low_s16(q1), vget_high_s16(q1));
    int32x4x2_t tq = vtrnq_s32(vreinterpretq_s32_s16(vcombine_s16(t0.val[0], t0.val[1])), vreinterpretq_s32_s16(vcombine_s16(t1.val[0], t1.val[1])));

    q0 = vcombine_s16(vget_low_s16(vreinterpretq_s16_s32(tq.val[0])), vget_high_s16(vreinterpretq_s16_s32(tq.val[0])));
    q1 = vcombine_s16(vget_low_s16(vreinterpretq_s16_s32(tq.val[1])), vget_high_s16(vreinterpretq_s16_s32(tq.val[1])));

    s = vaddq_s16(q0, q1);
    d = vsubq_s16(q0, q1);
    q2 = vcombine_s16(vget_low_s16(s), vget_low_s16(d));
    q3 = vcombine_s16(vget_high_s16(s), vget_high_s16(d));
    q0 = vaddq_s16(q2, q3);
    d = vsubq_s16(q2, q3);
    q1 = vcombine_s16(vget_high_s16(d), vget_low_s16(d));
    vst1q_s16(x, q0);
    vst1q_s16(x + 8, q1);
}
}

static void dequant_dc_neon(quant_t *q, int16_t *qval, int dequant, int n)
{
    do q++->dq[0] = (int16_t)(*qval++*(int16_t)dequant); while (--n);
}

static void quant_dc_neon(int16_t *qval, int16_t *deq, int16_t quant, int n, int round_q18)
{
#if 1
    int r_minus =  (1 << 18) - round_q18;
    static const uint8_t iscan16[16] = {0, 2, 3, 9, 1, 4, 8, 10, 5, 7, 11, 14, 6, 12, 13, 15};
    static const uint8_t iscan4[4] = {0, 1, 2, 3};
    const uint8_t *scan = n == 4 ? iscan4 : iscan16;
    do
    {
        int v = *qval;
        int r = v < 0 ? r_minus : round_q18;
        deq[*scan++] = *qval++ = (v * quant + r) >> 18;
    } while (--n);
#else
    int r_minus =  (1 << 18) - round_q18;
    do
    {
        int v = *qval;
        int r = v < 0 ? r_minus : round_q18;
        *deq++ = *qval++ = (v * quant + r) >> 18;
    } while (--n);
#endif
}

static void hadamar2_2d_neon(int16_t *x)
{
    int a = x[0];
    int b = x[1];
    int c = x[2];
    int d = x[3];
    x[0] = (int16_t)(a + b + c + d);
    x[1] = (int16_t)(a - b + c - d);
    x[2] = (int16_t)(a + b - c - d);
    x[3] = (int16_t)(a - b - c + d);
}

static void h264e_quant_luma_dc_neon(quant_t *q, int16_t *deq, const uint16_t *qdat)
{
    int16_t *tmp = ((int16_t*)q) - 16;
    hadamar4_2d_neon(tmp);
    quant_dc_neon(tmp, deq, qdat[0], 16, 0x20000);//0x15555);
    hadamar4_2d_neon(tmp);
    assert(!(qdat[1] & 3));
    // dirty trick here: shift w/o rounding, since it have no effect  for qp >= 10 (or, to be precise, for qp => 9)
    dequant_dc_neon(q, tmp, qdat[1] >> 2, 16);
}

static int h264e_quant_chroma_dc_neon(quant_t *q, int16_t *deq, const uint16_t *qdat)
{
    int16_t *tmp = ((int16_t*)q) - 16;
    hadamar2_2d_neon(tmp);
    quant_dc_neon(tmp, deq, (int16_t)(qdat[0] << 1), 4, 0xAAAA);
    hadamar2_2d_neon(tmp);
    assert(!(qdat[1] & 1));
    dequant_dc_neon(q, tmp, qdat[1] >> 1, 4);
    return !!(tmp[0] | tmp[1] | tmp[2] | tmp[3]);
}

#define TRANSFORM(x0, x1, x2, x3, p, s) { \
    int t0 = x0 + x3;                     \
    int t1 = x0 - x3;                     \
    int t2 = x1 + x2;                     \
    int t3 = x1 - x2;                     \
    (p)[  0] = (int16_t)(t0 + t2);        \
    (p)[  s] = (int16_t)(t1*2 + t3);      \
    (p)[2*s] = (int16_t)(t0 - t2);        \
    (p)[3*s] = (int16_t)(t1 - t3*2);      \
}

static void FwdTransformResidual4x42_neon(const uint8_t *inp, const uint8_t *pred, uint32_t inp_stride, int16_t *out)
{
#if TRANSPOSE_BLOCK
    int i;
    int16_t tmp[16];
    // Transform columns
    for (i = 0; i < 4; i++, pred++, inp++)
    {
        int f0 = inp[0] - pred[0];
        int f1 = inp[1*inp_stride] - pred[1*16];
        int f2 = inp[2*inp_stride] - pred[2*16];
        int f3 = inp[3*inp_stride] - pred[3*16];
        TRANSFORM(f0, f1, f2, f3, tmp + i*4, 1);
    }
    // Transform rows
    for (i = 0; i < 4; i++)
    {
        int d0 = tmp[i + 0];
        int d1 = tmp[i + 4];
        int d2 = tmp[i + 8];
        int d3 = tmp[i + 12];
        TRANSFORM(d0, d1, d2, d3, out + i, 4);
    }
#else
    /* Transform rows */
    uint8x8_t inp0  = vreinterpret_u8_s32(vtrn_s32(vreinterpret_s32_u8(vld1_u8(inp)),  vreinterpret_s32_u8(vld1_u8(inp + inp_stride))).val[0]);
    uint8x8_t inp1  = vreinterpret_u8_s32(vtrn_s32(vreinterpret_s32_u8(vld1_u8(inp + 2*inp_stride)), vreinterpret_s32_u8(vld1_u8(inp + 3*inp_stride))).val[0]);
    uint8x8_t pred0 = vreinterpret_u8_s32(vtrn_s32(vreinterpret_s32_u8(vld1_u8(pred)),  vreinterpret_s32_u8(vld1_u8(pred + 16))).val[0]);
    uint8x8_t pred1 = vreinterpret_u8_s32(vtrn_s32(vreinterpret_s32_u8(vld1_u8(pred + 2*16)), vreinterpret_s32_u8(vld1_u8(pred + 3*16))).val[0]);
    int16x8_t q0 = vreinterpretq_s16_u16(vsubl_u8(inp0, pred0));
    int16x8_t q1 = vreinterpretq_s16_u16(vsubl_u8(inp1, pred1));

    int16x4x2_t  t0 = vtrn_s16(vget_low_s16(q0), vget_high_s16(q0));
    int16x4x2_t  t1 = vtrn_s16(vget_low_s16(q1), vget_high_s16(q1));
    int32x4x2_t  tq = vtrnq_s32(vreinterpretq_s32_s16(vcombine_s16(t0.val[0], t0.val[1])), vreinterpretq_s32_s16(vcombine_s16(t1.val[0], t1.val[1])));

    int16x4_t d4 = vadd_s16(vget_low_s16(vreinterpretq_s16_s32(tq.val[0])), vget_high_s16(vreinterpretq_s16_s32(tq.val[1])));
    int16x4_t d5 = vsub_s16(vget_low_s16(vreinterpretq_s16_s32(tq.val[0])), vget_high_s16(vreinterpretq_s16_s32(tq.val[1])));
    int16x4_t d6 = vadd_s16(vget_high_s16(vreinterpretq_s16_s32(tq.val[0])), vget_low_s16(vreinterpretq_s16_s32(tq.val[1])));
    int16x4_t d7 = vsub_s16(vget_high_s16(vreinterpretq_s16_s32(tq.val[0])), vget_low_s16(vreinterpretq_s16_s32(tq.val[1])));
    int16x8_t q2 = vcombine_s16(d4, d5);
    int16x8_t q3 = vcombine_s16(d6, d7);
    q0 = vaddq_s16(q2, q3);
    q0 = vcombine_s16(vget_low_s16(q0), vadd_s16(vget_high_s16(q0), d5));
    q1 = vsubq_s16(q2, q3);
    q1 = vcombine_s16(vget_low_s16(q1), vsub_s16(vget_high_s16(q1), d7));

    t0 = vtrn_s16(vget_low_s16(q0), vget_high_s16(q0));
    t1 = vtrn_s16(vget_low_s16(q1), vget_high_s16(q1));
    tq = vtrnq_s32(vreinterpretq_s32_s16(vcombine_s16(t0.val[0], t0.val[1])), vreinterpretq_s32_s16(vcombine_s16(t1.val[0], t1.val[1])));

    d4 = vadd_s16(vget_low_s16(vreinterpretq_s16_s32(tq.val[0])), vget_high_s16(vreinterpretq_s16_s32(tq.val[1])));
    d5 = vsub_s16(vget_low_s16(vreinterpretq_s16_s32(tq.val[0])), vget_high_s16(vreinterpretq_s16_s32(tq.val[1])));
    d6 = vadd_s16(vget_high_s16(vreinterpretq_s16_s32(tq.val[0])), vget_low_s16(vreinterpretq_s16_s32(tq.val[1])));
    d7 = vsub_s16(vget_high_s16(vreinterpretq_s16_s32(tq.val[0])), vget_low_s16(vreinterpretq_s16_s32(tq.val[1])));
    q2 = vcombine_s16(d4, d5);
    q3 = vcombine_s16(d6, d7);
    q0 = vaddq_s16(q2, q3);
    q0 = vcombine_s16(vget_low_s16(q0), vadd_s16(vget_high_s16(q0), d5));
    q1 = vsubq_s16(q2, q3);
    q1 = vcombine_s16(vget_low_s16(q1), vsub_s16(vget_high_s16(q1), d7));

    vst1q_s16(out, q0);
    vst1q_s16(out + 8, q1);
#endif
}

static void TransformResidual4x4_neon(const int16_t *pSrc, const pix_t *pred, pix_t *out, int out_stride)
{
    int16x4_t e0, e1, e2, e3;
    int16x4_t f0, f1, f2, f3;
    int16x4_t g0, g1, g2, g3;
    int16x4_t h0, h1, h2, h3;
    int16x4_t d0 = vld1_s16(pSrc);
    int16x4_t d1 = vld1_s16(pSrc + 4);
    int16x4_t d2 = vld1_s16(pSrc + 8);
    int16x4_t d3 = vld1_s16(pSrc + 12);
    int16x4x2_t dd0 = vtrn_s16(d0, d1);
    int16x4x2_t dd1 = vtrn_s16(d2, d3);
    int32x4x2_t d = vtrnq_s32(vreinterpretq_s32_s16(vcombine_s16(dd0.val[0], dd0.val[1])), vreinterpretq_s32_s16(vcombine_s16(dd1.val[0], dd1.val[1])));
    d0 = vreinterpret_s16_s32(vget_low_s32(d.val[0]));
    d1 = vreinterpret_s16_s32(vget_high_s32(d.val[0]));
    d2 = vreinterpret_s16_s32(vget_low_s32(d.val[1]));
    d3 = vreinterpret_s16_s32(vget_high_s32(d.val[1]));

    e0 = vadd_s16(d0, d2);
    e1 = vsub_s16(d0, d2);
    e2 = vsub_s16(vshr_n_s16(d1, 1), d3);
    e3 = vadd_s16(d1, vshr_n_s16(d3, 1));
    f0 = vadd_s16(e0, e3);
    f1 = vadd_s16(e1, e2);
    f2 = vsub_s16(e1, e2);
    f3 = vsub_s16(e0, e3);

    dd0 = vtrn_s16(f0, f1);
    dd1 = vtrn_s16(f2, f3);
    d = vtrnq_s32(vreinterpretq_s32_s16(vcombine_s16(dd0.val[0], dd0.val[1])), vreinterpretq_s32_s16(vcombine_s16(dd1.val[0], dd1.val[1])));
    f0 = vreinterpret_s16_s32(vget_low_s32(d.val[0]));
    f1 = vreinterpret_s16_s32(vget_high_s32(d.val[0]));
    f2 = vreinterpret_s16_s32(vget_low_s32(d.val[1]));
    f3 = vreinterpret_s16_s32(vget_high_s32(d.val[1]));

    g0 = vadd_s16(f0, f2);
    g1 = vsub_s16(f0, f2);
    g2 = vsub_s16(vshr_n_s16(f1, 1), f3);
    g3 = vadd_s16(f1, vshr_n_s16(f3, 1));
    h0 = vadd_s16(g0, g3);
    h1 = vadd_s16(g1, g2);
    h2 = vsub_s16(g1, g2);
    h3 = vsub_s16(g0, g3);

    {
        uint8x8_t inp0 = vreinterpret_u8_s32(vtrn_s32(vreinterpret_s32_u8(vld1_u8(pred)),  vreinterpret_s32_u8(vld1_u8(pred + 16))).val[0]);
        uint8x8_t inp1 = vreinterpret_u8_s32(vtrn_s32(vreinterpret_s32_u8(vld1_u8(pred + 2*16)), vreinterpret_s32_u8(vld1_u8(pred + 3*16))).val[0]);
        int16x8_t a0 = vaddq_s16(vcombine_s16(h0, h1), vreinterpretq_s16_u16(vshll_n_u8(inp0, 6)));
        int16x8_t a1 = vaddq_s16(vcombine_s16(h2, h3), vreinterpretq_s16_u16(vshll_n_u8(inp1, 6)));
        uint8x8_t r0 = vqrshrun_n_s16(a0, 6);
        uint8x8_t r1 = vqrshrun_n_s16(a1, 6);
        *(uint32_t*)(&out[0*out_stride]) = vget_lane_u32(vreinterpret_u32_u8(r0), 0);
        *(uint32_t*)(&out[1*out_stride]) = vget_lane_u32(vreinterpret_u32_u8(r0), 1);
        *(uint32_t*)(&out[2*out_stride]) = vget_lane_u32(vreinterpret_u32_u8(r1), 0);
        *(uint32_t*)(&out[3*out_stride]) = vget_lane_u32(vreinterpret_u32_u8(r1), 1);
    }
}

static int is_zero_neon(const int16_t *dat, int i0, const uint16_t *thr)
{
    static const uint16x8_t g_ign_first = { 0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff };
    int16x8_t v0 = vabsq_s16(*(int16x8_t *)dat);
    int16x8_t v1 = vabsq_s16(*(int16x8_t *)(dat + 8));
    int16x8_t t = *(int16x8_t *)thr;
    uint16x8_t m0 = vcgtq_s16(v0, t);
    uint16x8_t m1 = vcgtq_s16(v1, t);
    if (i0)
        m0 = vandq_u16(m0, g_ign_first);
    m0 = vorrq_u16(m0, m1);
    uint16x4_t m4 = vorr_u16(vget_low_u16(m0), vget_high_u16(m0));
    return !(vget_lane_u32(vreinterpret_u32_u16(m4), 0) | vget_lane_u32(vreinterpret_u32_u16(m4), 1));
}

static int is_zero4_neon(const quant_t *q, int i0, const uint16_t *thr)
{
    return is_zero_neon(q[0].dq, i0, thr) &&
           is_zero_neon(q[1].dq, i0, thr) &&
           is_zero_neon(q[4].dq, i0, thr) &&
           is_zero_neon(q[5].dq, i0, thr);
}

static int zero_smallq_neon(quant_t *q, int mode, const uint16_t *qdat)
{
    int zmask = 0;
    int i, i0 = mode & 1, n = mode >> 1;
    if (mode == QDQ_MODE_INTER || mode == QDQ_MODE_CHROMA)
    {
        for (i = 0; i < n*n; i++)
        {
            if (is_zero_neon(q[i].dq, i0, qdat + OFFS_THR_1_OFF))
            {
                zmask |= (1 << i); //9.19
            }
        }
        if (mode == QDQ_MODE_INTER)   //8.27
        {
            if ((~zmask & 0x0033) && is_zero4_neon(q +  0, i0, qdat + OFFS_THR_2_OFF)) zmask |= 0x33;
            if ((~zmask & 0x00CC) && is_zero4_neon(q +  2, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 2);
            if ((~zmask & 0x3300) && is_zero4_neon(q +  8, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 8);
            if ((~zmask & 0xCC00) && is_zero4_neon(q + 10, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 10);
        }
    }
    return zmask;
}

static int quantize_neon(quant_t *q, int mode, const uint16_t *qdat, int zmask)
{
#if UNZIGSAG_IN_QUANT
#if TRANSPOSE_BLOCK
    //         ; Zig-zag scan      Transposed zig-zag
    //         ;    0 1 5 6        0 2 3 9
    //         ;    2 4 7 C        1 4 8 A
    //         ;    3 8 B D        5 7 B E
    //         ;    9 A E F        6 C D F
    static const unsigned char iscan16[16] = {0, 2, 3, 9, 1, 4, 8, 10, 5, 7, 11, 14, 6, 12, 13, 15};
#else
    static const unsigned char iscan16[16] = {0, 1, 5, 6, 2, 4, 7, 12, 3, 8, 11, 13, 9, 10, 14, 15};
#endif
#endif
    int ccol, crow, nz_block_mask = 0;
    ccol = mode >> 1;
    crow = ccol;
    do
    {
        do
        {
            int nz_mask = 0;

            if (zmask & 1)
            {
                int32_t *p = (int32_t *)q->qv;
                *p++ = 0; *p++ = 0; *p++ = 0; *p++ = 0;
                *p++ = 0; *p++ = 0; *p++ = 0; *p++ = 0;
            } else
            {
                static const uint8_t iscan16_neon [] = {
                    0x00,0x01,0x02,0x03,0x08,0x09,0x10,0x11,
                    0x0A,0x0B,0x04,0x05,0x06,0x07,0x0C,0x0D,
                    0x12,0x13,0x18,0x19,0x1A,0x1B,0x14,0x15,
                    0x0E,0x0F,0x16,0x17,0x1C,0x1D,0x1E,0x1F};
                static const uint16_t imask16_neon [] = {
                    0x0001,0x0002,0x0004,0x0008,
                    0x0010,0x0020,0x0040,0x0080,
                    0x0100,0x0200,0x0400,0x0800,
                    0x1000,0x2000,0x4000,0x8000};
                short save = 0;
                uint8x16_t q8,q9;
                int16x8_t q0 = vld1q_s16(q->dq);
                int16x8_t q1 = vld1q_s16(q->dq + 8);
                uint16x8_t r =  vdupq_n_u16(qdat[OFFS_RND_INTER]);
                uint16x8_t r0 = veorq_u16(r, vcltq_s16(q0, vdupq_n_s16(0)));
                uint16x8_t r1 = veorq_u16(r, vcltq_s16(q1, vdupq_n_s16(0)));
                int16x4_t d4, d5, d6, d7;
                int16x4_t d22, d23, d24, d25;
                int16x4_t d26, d27, d28, d29;

                d4 = d6 = vdup_n_s16(qdat[2]);
                d5 = d7 = vdup_n_s16(qdat[3]);
                d4 = vset_lane_s16(qdat[0], d4, 0);
                d4 = vset_lane_s16(qdat[0], d4, 2);
                d5 = vset_lane_s16(qdat[1], d5, 0);
                d5 = vset_lane_s16(qdat[1], d5, 2);
                d6 = vset_lane_s16(qdat[4], d6, 1);
                d6 = vset_lane_s16(qdat[4], d6, 3);
                d7 = vset_lane_s16(qdat[5], d7, 1);
                d7 = vset_lane_s16(qdat[5], d7, 3);

                d22 = vqshrn_n_s32(vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vmull_s16(vget_low_s16(q0), d4)), vget_low_u16(r0))), 16);
                d26 = vmul_s16(d22, d5);
                d23 = vqshrn_n_s32(vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vmull_s16(vget_high_s16(q0), d6)), vget_high_u16(r0))), 16);
                d27 = vmul_s16(d23, d7);
                d24 = vqshrn_n_s32(vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vmull_s16(vget_low_s16(q1), d4)), vget_low_u16(r1))), 16);
                d28 = vmul_s16(d24, d5);
                d25 = vqshrn_n_s32(vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vmull_s16(vget_high_s16(q1), d6)), vget_high_u16(r1))), 16);
                d29 = vmul_s16(d25, d7);
                if (mode & 1)
                {
                    save = q->dq[0];
                }
                vst1q_s16(q->dq,     vcombine_s16(d26, d27));
                vst1q_s16(q->dq + 8, vcombine_s16(d28, d29));
                if (mode & 1)
                {
                    q->dq[0] = save;
                }

                if (mode & 1)
                {
                    save = q->qv[0];
                }
                q8 = vld1q_u8(iscan16_neon);
                q9 = vld1q_u8(iscan16_neon + 16);

                {
// vtbl4_u8 is marked unavailable for iOS arm64, use wider versions there.
#if defined(__APPLE__) && defined(__aarch64__) &&  defined(__apple_build_version__)
                uint8x16x2_t vlut;
                vlut.val[0] = vreinterpretq_u8_s16(vcombine_s16(d22, d23));
                vlut.val[1] = vreinterpretq_u8_s16(vcombine_s16(d24, d25));
                vst1_s16(q->qv + 0, d4 = vreinterpret_s16_u8(vtbl2q_u8(vlut, vget_low_u8(q8))));
                vst1_s16(q->qv + 4, d5 = vreinterpret_s16_u8(vtbl2q_u8(vlut, vget_high_u8(q8))));
                vst1_s16(q->qv + 8, d6 = vreinterpret_s16_u8(vtbl2q_u8(vlut, vget_low_u8(q9))));
                vst1_s16(q->qv +12, d7 = vreinterpret_s16_u8(vtbl2q_u8(vlut, vget_high_u8(q9))));
#else
                uint8x8x4_t vlut;
                vlut.val[0] = vreinterpret_u8_s16(d22);
                vlut.val[1] = vreinterpret_u8_s16(d23);
                vlut.val[2] = vreinterpret_u8_s16(d24);
                vlut.val[3] = vreinterpret_u8_s16(d25);
                vst1_s16(q->qv + 0, d4 = vreinterpret_s16_u8(vtbl4_u8(vlut, vget_low_u8(q8))));
                vst1_s16(q->qv + 4, d5 = vreinterpret_s16_u8(vtbl4_u8(vlut, vget_high_u8(q8))));
                vst1_s16(q->qv + 8, d6 = vreinterpret_s16_u8(vtbl4_u8(vlut, vget_low_u8(q9))));
                vst1_s16(q->qv +12, d7 = vreinterpret_s16_u8(vtbl4_u8(vlut, vget_high_u8(q9))));
#endif
                }
                {
                    uint16x8_t bm0 = vld1q_u16(imask16_neon);
                    uint16x8_t bm1 = vld1q_u16(imask16_neon + 8);
                    uint16x4_t m;
                    bm0 = vandq_u16(bm0, vceqq_s16(vcombine_s16(d4, d5), vdupq_n_s16(0)));
                    bm1 = vandq_u16(bm1, vceqq_s16(vcombine_s16(d6, d7), vdupq_n_s16(0)));
                    bm0 = vorrq_u16(bm0, bm1);
                    m = vorr_u16(vget_low_u16(bm0), vget_high_u16(bm0));
                    m = vpadd_u16(m, m);
                    m = vpadd_u16(m, m);
                    nz_mask = vget_lane_u16(vmvn_u16(m), 0);
                }

                if (mode & 1)
                {
                    q->qv[0] = save;
                    nz_mask &= ~1;
                }
            }

            zmask >>= 1;
            nz_block_mask <<= 1;
            if (nz_mask)
                nz_block_mask |= 1;
            q++;
        } while (--ccol);
        ccol = mode >> 1;
    } while (--crow);
    return nz_block_mask;
}

static void transform_neon(const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q)
{
    int crow = mode >> 1;
    int ccol = crow;

    do
    {
        do
        {
            FwdTransformResidual4x42_neon(inp, pred, inp_stride, q->dq);
            q++;
            inp += 4;
            pred += 4;
        } while (--ccol);
        ccol = mode >> 1;
        inp += 4*(inp_stride - ccol);
        pred += 4*(16 - ccol);
    } while (--crow);
}

static int h264e_transform_sub_quant_dequant_neon(const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat)
{
    int zmask;
    transform_neon(inp, pred, inp_stride, mode, q);
    if (mode & 1) // QDQ_MODE_INTRA_16 || QDQ_MODE_CHROMA
    {
        int cloop = (mode >> 1)*(mode >> 1);
        short *dc = ((short *)q) - 16;
        quant_t *pq = q;
        do
        {
            *dc++ = pq->dq[0];
            pq++;
        } while (--cloop);
    }
    zmask = zero_smallq_neon(q, mode, qdat);
    return quantize_neon(q, mode, qdat, zmask);
}

static void h264e_transform_add_neon(pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask)
{
    int crow = side;
    int ccol = crow;

    assert(!((unsigned)out % 4));
    assert(!((unsigned)pred % 4));
    assert(!(out_stride % 4));
    do
    {
        do
        {
            if (mask >= 0)
            {
                // copy 4x4
                pix_t *dst = out;
                *(uint32_t*)dst = *(uint32_t*)(pred + 0 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 1 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 2 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 3 * 16);
            } else
            {
                TransformResidual4x4_neon(q->dq, pred, out, out_stride);
            }
            mask <<= 1;
            q++;
            out += 4;
            pred += 4;
        } while (--ccol);
        ccol = side;
        out += 4*(out_stride - ccol);
        pred += 4*(16 - ccol);
    } while (--crow);
}
#endif

#if H264E_ENABLE_PLAIN_C

static uint8_t byteclip_deblock(int x)
{
    if (x > 255)
    {
        return 255;
    }
    if (x < 0)
    {
        return 0;
    }
    return (uint8_t)x;
}

static int clip_range(int range, int src)
{
    if (src > range)
    {
        src = range;
    }
    if (src < -range)
    {
        src = -range;
    }
    return src;
}

static void deblock_chroma(uint8_t *pix, int stride, int alpha, int beta, int thr, int strength)
{
    int p1, p0, q0, q1;
    int delta;

    if (strength == 0)
    {
        return;
    }

    p1 = pix[-2*stride];
    p0 = pix[-1*stride];
    q0 = pix[ 0*stride];
    q1 = pix[ 1*stride];

    if (ABS(p0 - q0) >= alpha || ABS(p1 - p0) >= beta || ABS(q1 - q0) >= beta)
    {
        return;
    }

    if (strength < 4)
    {
        int tC = thr + 1;
        delta = (((q0 - p0)*4) + (p1 - q1) + 4) >> 3;
        delta = clip_range(tC, delta);
        pix[-1*stride] = byteclip_deblock(p0 + delta);
        pix[ 0*stride] = byteclip_deblock(q0 - delta);
    } else
    {
        pix[-1*stride] = (pix_t)((2*p1 + p0 + q1 + 2) >> 2);
        pix[ 0*stride] = (pix_t)((2*q1 + q0 + p1 + 2) >> 2);
    }
}

static void deblock_luma_v(uint8_t *pix, int stride, int alpha, int beta, const uint8_t *pthr, const uint8_t *pstr)
{
    int p2, p1, p0, q0, q1, q2, thr;
    int ap, aq, delta, cloop, i;
    for (i = 0; i < 4; i++)
    {
        cloop = 4;
        if (pstr[i])
        {
            thr = pthr[i];
            do
            {
                p1 = pix[-2];
                p0 = pix[-1];
                q0 = pix[ 0];
                q1 = pix[ 1];

                //if (ABS(p0 - q0) < alpha && ABS(p1 - p0) < beta && ABS(q1 - q0) < beta)
                if (((ABS(p0 - q0) - alpha) & (ABS(p1 - p0) - beta) & (ABS(q1 - q0) - beta)) < 0)
                {
                    int tC = thr;
                    // avoid conditons
                    int sp, sq, d2;
                    p2 = pix[-3];
                    q2 = pix[ 2];
                    ap = ABS(p2 - p0);
                    aq = ABS(q2 - q0);
                    delta = (((q0 - p0)*4) + (p1 - q1) + 4) >> 3;

                    sp = (ap - beta) >> 31;
                    sq = (aq - beta) >> 31;
                    d2 = (((p2 + ((p0 + q0 + 1) >> 1)) >> 1) - p1) & sp;
                    d2 = clip_range(thr, d2);
                    pix[-2] = (pix_t)(p1 + d2);
                    d2 = (((q2 + ((p0 + q0 + 1) >> 1)) >> 1) - q1) & sq;
                    d2 = clip_range(thr, d2);
                    pix[ 1] = (pix_t)(q1 + d2);
                    tC = thr - sp - sq;
                    delta = clip_range(tC, delta);
                    pix[-1] = byteclip_deblock(p0 + delta);
                    pix[ 0] = byteclip_deblock(q0 - delta);
                }
                pix += stride;
            } while (--cloop);
        } else
        {
                pix += 4*stride;
        }
    }
}

static void deblock_luma_h_s4(uint8_t *pix, int stride, int alpha, int beta)
{
    int p3, p2, p1, p0, q0, q1, q2, q3;
    int ap, aq, cloop = 16;
    do
    {
        int abs_p0_q0, abs_p1_p0, abs_q1_q0;
        p1 = pix[-2*stride];
        p0 = pix[-1*stride];
        q0 = pix[ 0*stride];
        q1 = pix[ 1*stride];
        abs_p0_q0 = ABS(p0 - q0);
        abs_p1_p0 = ABS(p1 - p0);
        abs_q1_q0 = ABS(q1 - q0);
        if (abs_p0_q0 < alpha && abs_p1_p0 < beta && abs_q1_q0 < beta)
        {
            int short_p = (2*p1 + p0 + q1 + 2);
            int short_q = (2*q1 + q0 + p1 + 2);

            if (abs_p0_q0 < ((alpha>>2)+2))
            {
                p2 = pix[-3*stride];
                q2 = pix[ 2*stride];
                ap = ABS(p2 - p0);
                aq = ABS(q2 - q0);
                if (ap < beta)
                {
                    int t = p2 + p1 + p0 + q0 + 2;
                    p3 = pix[-4*stride];
                    short_p += t - p1 + q0; //(p2 + 2*p1 + 2*p0 + 2*q0 + q1 + 4) >> 3);
                    short_p >>= 1;
                    pix[-2*stride] = (pix_t)(t >> 2);
                    pix[-3*stride] = (pix_t)((2*p3 + 2*p2 + t + 2) >> 3); //(2*p3 + 3*p2 + p1 + p0 + q0 + 4) >> 3);
                }
                if (aq < beta)
                {
                    int t = q2 + q1 + p0 + q0 + 2;
                    q3 = pix[ 3*stride];
                    short_q += (t - q1 + p0);//(q2 + 2*q1 + 2*q0 + 2*p0 + p1 + 4)>>3);
                    short_q >>= 1;
                    pix[ 1*stride] = (pix_t)(t >> 2);
                    pix[ 2*stride] = (pix_t)((2*q3 + 2*q2 + t + 2) >> 3); //((2*q3 + 3*q2 + q1 + q0 + p0 + 4) >> 3);
                }
            }
            pix[-1*stride] = (pix_t)(short_p >> 2);
            pix[ 0*stride] = (pix_t)(short_q >> 2);
        }
        pix += 1;
    } while (--cloop);
}

static void deblock_luma_v_s4(uint8_t *pix, int stride, int alpha, int beta)
{
    int p3, p2, p1, p0, q0, q1, q2, q3;
    int ap, aq, cloop = 16;
    do
    {
        p2 = pix[-3];
        p1 = pix[-2];
        p0 = pix[-1];
        q0 = pix[ 0];
        q1 = pix[ 1];
        q2 = pix[ 2];
        if (ABS(p0 - q0) < alpha && ABS(p1 - p0) < beta && ABS(q1 - q0) < beta)
        {
            ap = ABS(p2 - p0);
            aq = ABS(q2 - q0);

            if (ap < beta && ABS(p0 - q0) < ((alpha >> 2) + 2))
            {
                p3 = pix[-4];
                pix[-1] = (pix_t)((p2 + 2*p1 + 2*p0 + 2*q0 + q1 + 4) >> 3);
                pix[-2] = (pix_t)((p2 + p1 + p0 + q0 + 2) >> 2);
                pix[-3] = (pix_t)((2*p3 + 3*p2 + p1 + p0 + q0 + 4) >> 3);
            } else
            {
                pix[-1] = (pix_t)((2*p1 + p0 + q1 + 2) >> 2);
            }

            if (aq < beta && ABS(p0 - q0) < ((alpha >> 2) + 2))
            {
                q3 = pix[ 3];
                pix[ 0] = (pix_t)((q2 + 2*q1 + 2*q0 + 2*p0 + p1 + 4) >> 3);
                pix[ 1] = (pix_t)((q2 + q1 + p0 + q0 + 2) >> 2);
                pix[ 2] = (pix_t)((2*q3 + 3*q2 + q1 + q0 + p0 + 4) >> 3);
            } else
            {
                pix[ 0] = (pix_t)((2*q1 + q0 + p1 + 2) >> 2);
            }
        }
        pix += stride;
    } while (--cloop);
}

static void deblock_luma_h(uint8_t *pix, int stride, int alpha, int beta, const uint8_t *pthr, const uint8_t *pstr)
{
    int p2, p1, p0, q0, q1, q2;
    int ap, aq, delta, i;
    for (i = 0; i < 4; i++)
    {
        if (pstr[i])
        {
            int cloop = 4;
            int thr = pthr[i];
            do
            {
                p1 = pix[-2*stride];
                p0 = pix[-1*stride];
                q0 = pix[ 0*stride];
                q1 = pix[ 1*stride];

                //if (ABS(p0-q0) < alpha && ABS(p1-p0) < beta && ABS(q1-q0) < beta)
                if (((ABS(p0-q0) - alpha) & (ABS(p1-p0) - beta) & (ABS(q1-q0) - beta)) < 0)
                {
                    int tC = thr;
                    int sp, sq, d2;
                    p2 = pix[-3*stride];
                    q2 = pix[ 2*stride];
                    ap = ABS(p2 - p0);
                    aq = ABS(q2 - q0);
                    delta = (((q0 - p0)*4) + (p1 - q1) + 4) >> 3;

                    sp = (ap - beta) >> 31;
                    d2 = (((p2 + ((p0 + q0 + 1) >> 1)) >> 1) - p1) & sp;
                    d2 = clip_range(thr, d2);
                    pix[-2*stride] = (pix_t)(p1 + d2);

                    sq = (aq - beta) >> 31;
                    d2 = (((q2 + ((p0 + q0 + 1) >> 1)) >> 1) - q1) & sq;
                    d2 = clip_range(thr, d2);
                    pix[ 1*stride] = (pix_t)(q1 + d2);

                    tC = thr - sp - sq;
                    delta = clip_range(tC, delta);

                    pix[-1*stride] = byteclip_deblock(p0 + delta);
                    pix[ 0*stride] = byteclip_deblock(q0 - delta);
                }
                pix += 1;
            } while (--cloop);
        } else
        {
            pix += 4;
        }
    }
}

static void deblock_chroma_v(uint8_t *pix, int32_t stride, int a, int b, const uint8_t *thr, const uint8_t *str)
{
    int i;
    for (i = 0; i < 8; i++)
    {
        deblock_chroma(pix, 1, a, b, thr[i >> 1], str[i >> 1]);
        pix += stride;
    }
}

static void deblock_chroma_h(uint8_t *pix, int32_t stride, int a, int b, const uint8_t *thr, const uint8_t *str)
{
    int i;
    for (i = 0; i < 8; i++)
    {
        deblock_chroma(pix, stride, a, b, thr[i >> 1], str[i >> 1]);
        pix += 1;
    }
}

static void h264e_deblock_chroma(uint8_t *pix, int32_t stride, const deblock_params_t *par)
{
    const uint8_t *alpha = par->alpha;
    const uint8_t *beta  = par->beta;
    const uint8_t *thr   = par->tc0;
    const uint8_t *strength = (uint8_t *)par->strength32;
    int a,b,x,y;
    a = alpha[0];
    b = beta[0];
    for (x = 0; x < 16; x += 8)
    {
        uint32_t str = *(uint32_t*)&strength[x];
        if (str && a)
        {
            deblock_chroma_v(pix + (x >> 1), stride, a, b, thr + x, strength + x);
        }
        a = alpha[1];
        b = beta[1];
    }
    thr += 16;
    strength += 16;
    a = alpha[2];
    b = beta[2];
    for (y = 0; y < 16; y += 8)
    {
        uint32_t str = *(uint32_t*)&strength[y];
        if (str && a)
        {
            deblock_chroma_h(pix, stride, a, b, thr + y, strength + y);
        }
        pix += 4*stride;
        a = alpha[3];
        b = beta[3];
    }
}

static void h264e_deblock_luma(uint8_t *pix, int32_t stride, const deblock_params_t *par)
{
    const uint8_t *alpha = par->alpha;
    const uint8_t *beta  = par->beta;
    const uint8_t *thr   = par->tc0;
    const uint8_t *strength = (uint8_t *)par->strength32;
    int a = alpha[0];
    int b = beta[0];
    int x, y;
    for (x = 0; x < 16; x += 4)
    {
        uint32_t str = *(uint32_t*)&strength[x];
        if ((uint8_t)str == 4)
        {
            deblock_luma_v_s4(pix + x, stride, a, b);
        } else if (str && a)
        {
            deblock_luma_v(pix + x, stride, a, b, thr + x, strength + x);
        }
        a = alpha[1];
        b = beta[1];
    }
    a = alpha[2];
    b = beta[2];
    thr += 16;
    strength += 16;
    for (y = 0; y < 16; y += 4)
    {
        uint32_t str = *(uint32_t*)&strength[y];
        if ((uint8_t)str == 4)
        {
            deblock_luma_h_s4(pix, stride, a, b);
        } else if (str && a)
        {
            deblock_luma_h(pix, stride, a, b, thr + y, strength + y);
        }
        a = alpha[3];
        b = beta[3];
        pix += 4*stride;
    }
}

static void h264e_denoise_run(unsigned char *frm, unsigned char *frmprev, int w, int h_arg, int stride_frm, int stride_frmprev)
{
    int cloop, h = h_arg;
    if (w <= 2 || h <= 2)
    {
        return;
    }
    w -= 2;
    h -= 2;

    do
    {
        unsigned char *pf = frm += stride_frm;
        unsigned char *pp = frmprev += stride_frmprev;
        cloop = w;
        pp[-stride_frmprev] = *pf++;
        pp++;
        do
        {
            int d, neighbourhood;
            unsigned g, gd, gn, out_val;
            d = pf[0] - pp[0];
            neighbourhood  = pf[-1]      - pp[-1];
            neighbourhood += pf[+1]      - pp[+1];
            neighbourhood += pf[-stride_frm] - pp[-stride_frmprev];
            neighbourhood += pf[+stride_frm] - pp[+stride_frmprev];

            if (d < 0)
            {
                d = -d;
            }
            if (neighbourhood < 0)
            {
                neighbourhood = -neighbourhood;
            }
            neighbourhood >>= 2;

            gd = g_diff_to_gainQ8[d];
            gn = g_diff_to_gainQ8[neighbourhood];

            gn <<= 2;
            if (gn > 255)
            {
                gn = 255;
            }

            gn = 255 - gn;
            gd = 255 - gd;
            g = gn*gd;  // Q8*Q8 = Q16;

            //out_val = ((pp[0]*g ) >> 16) + (((0xffff-g)*pf[0] ) >> 16);
            //out_val = ((pp[0]*g + (1<<15)) >> 16) + (((0xffff-g)*pf[0]  + (1<<15)) >> 16);
            out_val = (pp[0]*g + (0xffff - g)*pf[0]  + (1 << 15)) >> 16;

            assert(out_val <= 255);

            pp[-stride_frmprev] = (unsigned char)out_val;
            //pp[-stride_frmprev] = gd;//(unsigned char)((neighbourhood+1)>255?255:(neighbourhood+1));

            pf++, pp++;
        } while (--cloop);

        pp[-stride_frmprev] = *pf++;
    } while(--h);

    memcpy(frmprev + stride_frmprev, frm + stride_frm, w + 2);
    h = h_arg - 2;
    do
    {
        memcpy(frmprev, frmprev - stride_frmprev, w + 2);
        frmprev -= stride_frmprev;
    } while(--h);
    memcpy(frmprev, frm - stride_frm*(h_arg - 2), w + 2);
}

#undef IS_NULL
#define IS_NULL(p) ((p) < (pix_t *)32)

static uint32_t intra_predict_dc(const pix_t *left, const pix_t *top, int log_side)
{
    unsigned dc = 0, side = 1u << log_side, round = 0;
    do
    {
        if (!IS_NULL(left))
        {
            int cloop = side;
            round += side >> 1;
            do
            {
                dc += *left++;
                dc += *left++;
                dc += *left++;
                dc += *left++;
            } while(cloop -= 4);
        }
        left = top;
        top = NULL;
    } while (left);
    dc += round;
    if (round == side)
        dc >>= 1;
    dc >>= log_side;
    if (!round) dc = 128;
    return dc * 0x01010101;
}

/*
 * Note: To make the code more readable we refer to the neighboring pixels
 * in variables named as below:
 *
 *    UL U0 U1 U2 U3 U4 U5 U6 U7
 *    L0 xx xx xx xx
 *    L1 xx xx xx xx
 *    L2 xx xx xx xx
 *    L3 xx xx xx xx
 */
#define UL edge[-1]
#define U0 edge[0]
#define T1 edge[1]
#define U2 edge[2]
#define U3 edge[3]
#define U4 edge[4]
#define U5 edge[5]
#define U6 edge[6]
#define U7 edge[7]
#define L0 edge[-2]
#define L1 edge[-3]
#define L2 edge[-4]
#define L3 edge[-5]

static void h264e_intra_predict_16x16(pix_t *predict,  const pix_t *left, const pix_t *top, int mode)
{
    int cloop = 16;
    uint32_t *d = (uint32_t*)predict;
    assert(IS_ALIGNED(predict, 4));
    assert(IS_ALIGNED(top, 4));
    if (mode != 1)
    {
        uint32_t t0, t1, t2, t3;
        if (mode < 1)
        {
            t0 = ((uint32_t*)top)[0];
            t1 = ((uint32_t*)top)[1];
            t2 = ((uint32_t*)top)[2];
            t3 = ((uint32_t*)top)[3];
        } else //(mode == 2)
        {
            // 直流系数是top和left的平均值（只有一个数据）
            t0 = t1 = t2 = t3 = intra_predict_dc(left, top, 4);
        }
        do
        {
            *d++ = t0;
            *d++ = t1;
            *d++ = t2;
            *d++ = t3;
        } while (--cloop);
    } else //if (mode == 1)
    {
        do
        {
            uint32_t val = *left++ * 0x01010101u;
            *d++ = val;
            *d++ = val;
            *d++ = val;
            *d++ = val;
        } while (--cloop);
    }
}

static void h264e_intra_predict_chroma(pix_t *predict, const pix_t *left, const pix_t *top, int mode)
{
    int cloop = 8;
    uint32_t *d = (uint32_t*)predict;
    assert(IS_ALIGNED(predict, 4));
    assert(IS_ALIGNED(top, 4));
    if (mode < 1)
    {
        uint32_t t0, t1, t2, t3;
        t0 = ((uint32_t*)top)[0];
        t1 = ((uint32_t*)top)[1];
        t2 = ((uint32_t*)top)[2];
        t3 = ((uint32_t*)top)[3];
        do
        {
            *d++ = t0;
            *d++ = t1;
            *d++ = t2;
            *d++ = t3;
        } while (--cloop);
    } else if (mode == 1)
    {
        do
        {
            uint32_t u = left[0] * 0x01010101u;
            uint32_t v = left[8] * 0x01010101u;
            d[0] = u;
            d[1] = u;
            d[2] = v;
            d[3] = v;
            d += 4;
            left++;
        } while(--cloop);
    } else //if (mode == 2)
    {
        int ccloop = 2;
        cloop = 2;
        do
        {
            d[0] = d[1] = d[16] = intra_predict_dc(left, top, 2);
            d[17] = intra_predict_dc(left + 4, top + 4, 2);
            if (!IS_NULL(top))
            {
                d[1] = intra_predict_dc(NULL, top + 4, 2);
            }
            if (!IS_NULL(left))
            {
                d[16] = intra_predict_dc(NULL, left + 4, 2);
            }
            d += 2;
            left += 8;
            top += 8;
        } while(--cloop);

        do
        {
            cloop = 12;
            do
            {
                *d = d[-4];
                d++;
            } while(--cloop);
            d += 4;
        } while(--ccloop);
    }
}

static int pix_sad_4(uint32_t r0, uint32_t r1, uint32_t r2, uint32_t r3,
                     uint32_t x0, uint32_t x1, uint32_t x2, uint32_t x3)
{
#if defined(__arm__)
    int sad = __usad8(r0, x0);
    sad = __usada8(r1, x1, sad);
    sad = __usada8(r2, x2, sad);
    sad = __usada8(r3, x3, sad);
    return sad;
#else
    int c, sad = 0;
    for (c = 0; c < 4; c++)
    {
        int d = (r0 & 0xff) - (x0 & 0xff); r0 >>= 8; x0 >>= 8;
        sad += ABS(d);
    }
    for (c = 0; c < 4; c++)
    {
        int d = (r1 & 0xff) - (x1 & 0xff); r1 >>= 8; x1 >>= 8;
        sad += ABS(d);
    }
    for (c = 0; c < 4; c++)
    {
        int d = (r2 & 0xff) - (x2 & 0xff); r2 >>= 8; x2 >>= 8;
        sad += ABS(d);
    }
    for (c = 0; c < 4; c++)
    {
        int d = (r3 & 0xff) - (x3 & 0xff); r3 >>= 8; x3 >>= 8;
        sad += ABS(d);
    }
    return sad;
#endif
}

static int h264e_intra_choose_4x4(const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty)
{
    int sad, best_sad, best_m = 2;

    uint32_t r0, r1, r2, r3;
    uint32_t x0, x1, x2, x3, x;

    r0 = ((uint32_t *)blockin)[ 0];
    r1 = ((uint32_t *)blockin)[ 4];
    r2 = ((uint32_t *)blockin)[ 8];
    r3 = ((uint32_t *)blockin)[12];
#undef TEST
#define TEST(mode) sad = pix_sad_4(r0, r1, r2, r3, x0, x1, x2, x3); \
        if (mode != mpred) sad += penalty;    \
        if (sad < best_sad)                   \
        {                                     \
            ((uint32_t *)blockpred)[ 0] = x0; \
            ((uint32_t *)blockpred)[ 4] = x1; \
            ((uint32_t *)blockpred)[ 8] = x2; \
            ((uint32_t *)blockpred)[12] = x3; \
            best_sad = sad;                   \
            best_m = mode;                    \
        }

    // DC
    x0 = x1 = x2 = x3 = intra_predict_dc((avail & AVAIL_L) ? &L3 : 0, (avail & AVAIL_T) ? &U0 : 0, 2);
    best_sad = pix_sad_4(r0, r1, r2, r3, x0, x1, x2, x3);
    if (2 != mpred)
    {
        best_sad += penalty;
    }
    ((uint32_t *)blockpred)[ 0] = x0;
    ((uint32_t *)blockpred)[ 4] = x1;
    ((uint32_t *)blockpred)[ 8] = x2;
    ((uint32_t *)blockpred)[12] = x3;


    if (avail & AVAIL_T)
    {
        uint32_t save = *(uint32_t*)&U4;
        if (!(avail & AVAIL_TR))
        {
            *(uint32_t*)&U4 = U3*0x01010101u;
        }

        x0 = x1 = x2 = x3 = *(uint32_t*)&U0;
        TEST(0)

        x  = ((U6 + 3u*U7      + 2u) >> 2) << 24;
        x |= ((U5 + 2u*U6 + U7 + 2u) >> 2) << 16;
        x |= ((U4 + 2u*U5 + U6 + 2u) >> 2) << 8;
        x |= ((U3 + 2u*U4 + U5 + 2u) >> 2);

        x3 = x;
        x = (x << 8) | ((U2 + 2u*U3 + U4 + 2u) >> 2);
        x2 = x;
        x = (x << 8) | ((T1 + 2u*U2 + U3 + 2u) >> 2);
        x1 = x;
        x = (x << 8) | ((U0 + 2u*T1 + U2 + 2u) >> 2);
        x0 = x;
        TEST(3)

        x3 = x1;
        x1 = x0;

        x  = ((U4 + U5 + 1u) >> 1) << 24;
        x |= ((U3 + U4 + 1u) >> 1) << 16;
        x |= ((U2 + U3 + 1u) >> 1) << 8;
        x |= ((T1 + U2 + 1u) >> 1);
        x2 = x;
        x = (x << 8) | ((U0 + T1 + 1) >> 1);
        x0 = x;
        TEST(7)

        *(uint32_t*)&U4 = save;
    }

    if (avail & AVAIL_L)
    {
        x0 = 0x01010101u * L0;
        x1 = 0x01010101u * L1;
        x2 = 0x01010101u * L2;
        x3 = 0x01010101u * L3;
        TEST(1)

        x = x3;
        x <<= 16;
        x |= ((L2 + 3u*L3 + 2u) >> 2) << 8;
        x |= ((L2 + L3 + 1u) >> 1);
        x2 = x;
        x <<= 16;
        x |= ((L1 + 2u*L2 + L3 + 2u) >> 2) << 8;
        x |= ((L1 + L2 + 1u) >> 1);
        x1 = x;
        x <<= 16;
        x |= ((L0 + 2u*L1 + L2 + 2u) >> 2) << 8;
        x |= ((L0 + L1 + 1u) >> 1);
        x0 = x;
        TEST(8)
    }

    if ((avail & (AVAIL_T | AVAIL_L | AVAIL_TL)) == (AVAIL_T | AVAIL_L | AVAIL_TL))
    {
        uint32_t line0, line3;
        x  = ((U3 + 2u*U2 + T1 + 2u) >> 2) << 24;
        x |= ((U2 + 2u*T1 + U0 + 2u) >> 2) << 16;
        x |= ((T1 + 2u*U0 + UL + 2u) >> 2) << 8;
        x |= ((U0 + 2u*UL + L0 + 2u) >> 2);
        line0 = x;
        x0 = x;
        x = (x << 8) | ((UL + 2u*L0 + L1 + 2u) >> 2);
        x1 = x;
        x = (x << 8) | ((L0 + 2u*L1 + L2 + 2u) >> 2);
        x2 = x;
        x = (x << 8) | ((L1 + 2u*L2 + L3 + 2u) >> 2);
        x3 = x;
        line3 = x;
        TEST(4)

        x = x0 << 8;
        x |= ((UL + L0 + 1u) >> 1);
        x0 = x;
        x <<= 8;
        x |= (line3 >> 16) & 0xff;
        x <<= 8;
        x |= ((L0 + L1 + 1u) >> 1);
        x1 = x;
        x <<= 8;
        x |= (line3 >> 8) & 0xff;
        x <<= 8;
        x |= ((L1 + L2 + 1u) >> 1);
        x2 = x;
        x <<= 8;
        x |= line3 & 0xff;
        x <<= 8;
        x |= ((L2 + L3 + 1u) >> 1);
        x3 = x;
        TEST(6)

        x1 = line0;
        x3 = (x1 << 8) | ((line3 >> 8) & 0xFF);

        x  = ((U2 + U3 + 1u) >> 1) << 24;
        x |= ((T1 + U2 + 1u) >> 1) << 16;
        x |= ((U0 + T1 + 1u) >> 1) << 8;
        x |= ((UL + U0 + 1u) >> 1);
        x0 = x;
        x = (x << 8) | ((line3 >> 16) & 0xFF);
        x2 = x;
        TEST(5)
    }
    return best_m + (best_sad << 4);
}

static uint8_t byteclip(int x)
{
    if (x > 255) x = 255;
    if (x < 0) x = 0;
    return (uint8_t)x;
}

static int hpel_lpf(const uint8_t *p, int s)
{
    return p[0] - 5*p[s] + 20*p[2*s] + 20*p[3*s] - 5*p[4*s] + p[5*s];
}

static void copy_wh(const uint8_t *src, int src_stride, uint8_t *dst, int w, int h)
{
    int x, y;
    for (y = 0; y < h; y++)
    {
        for (x = 0; x < w; x++)
        {
            dst [x] = src [x];
        }
        dst += 16;
        src += src_stride;
    }
}

static void hpel_lpf_diag(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    ALIGN(16) int16_t scratch[21 * 16] ALIGN2(16);  /* 21 rows by 16 pixels per row */

    /*
     * Intermediate values will be 1/2 pel at Horizontal direction
     * Starting at (0.5, -2) at top extending to (0.5, height + 3) at bottom
     * scratch contains a 2D array of size (w)X(h + 5)
     */
    int y, x;
    for (y = 0; y < h + 5; y++)
    {
        for (x = 0; x < w; x++)
        {
            scratch[y * w + x] = (int16_t)hpel_lpf(src + (y - 2) * src_stride + (x - 2), 1);
        }
    }

    /* Vertical interpolate */
    for (y = 0; y < h; y++)
    {
        for (x = 0; x < w; x++)
        {
            int pos = y * w + x;
            int HalfCoeff =
                scratch [pos] -
                5 * scratch [pos + 1 * w] +
                20 * scratch [pos + 2 * w] +
                20 * scratch [pos + 3 * w] -
                5 * scratch [pos + 4 * w] +
                scratch [pos + 5 * w];

            HalfCoeff = byteclip((HalfCoeff + 512) >> 10);

            dst [y * 16 + x] = (uint8_t)HalfCoeff;
        }
    }
}

static void hpel_lpf_hor(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    int x, y;
    for (y = 0; y < h; y++)
    {
        for (x = 0; x < w; x++)
        {
            dst [y * 16 + x] = byteclip((hpel_lpf(src + y * src_stride + (x - 2), 1) + 16) >> 5);
        }
    }
}

static void hpel_lpf_ver(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, int w, int h)
{
    int y, x;
    for (y = 0; y < h; y++)
    {
        for (x = 0; x < w; x++)
        {
            dst [y * 16 + x] = byteclip((hpel_lpf(src + (y - 2) * src_stride + x, src_stride) + 16) >> 5);
        }
    }
}

static void average_16x16_unalign(uint8_t *dst, const uint8_t *src1, int src1_stride)
{
    int x, y;
    for (y = 0; y < 16; y++)
    {
        for (x = 0; x < 16; x++)
        {
            dst[y * 16 + x] = (uint8_t)(((uint32_t)dst [y * 16 + x] + src1[y*src1_stride + x] + 1) >> 1);
        }
    }
}

static void h264e_qpel_average_wh_align(const uint8_t *src0, const uint8_t *src1, uint8_t *dst, point_t wh)
{
    int w = wh.s.x;
    int h = wh.s.y;
    int x, y;
    for (y = 0; y < h; y++)
    {
        for (x = 0; x < w; x++)
        {
            dst[y * 16 + x] = (uint8_t)((src0[y * 16 + x] + src1[y * 16 + x] + 1) >> 1);
        }
    }
}

static void h264e_qpel_interpolate_luma(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, point_t wh, point_t dxdy)
{
    ALIGN(16) uint8_t scratch[16*16] ALIGN2(16);
    //  src += ((dx + 1) >> 2) + ((dy + 1) >> 2)*src_stride;            // dx == 3 ? next row; dy == 3 ? next line
    //  dxdy              actions: Horizontal, Vertical, Diagonal, Average
    //  0 1 2 3 +1        -   ha    h    ha+
    //  1                 va  hva   hda  hv+a
    //  2                 v   vda   d    v+da
    //  3                 va+ h+va h+da  h+v+a
    //  +stride
    int32_t pos = 1 << (dxdy.s.x + 4*dxdy.s.y);
    int dstused = 0;

    if (pos == 1)
    {
        copy_wh(src, src_stride, dst, wh.s.x, wh.s.y);
        return;
    }
    if (pos & 0xe0ee)// 1110 0000 1110 1110
    {
        hpel_lpf_hor(src + ((pos & 0xe000) ? src_stride : 0), src_stride, dst, wh.s.x, wh.s.y);
        dstused++;
    }
    if (pos & 0xbbb0)// 1011 1011 1011 0000
    {
        hpel_lpf_ver(src + ((pos & 0x8880) ? 1 : 0), src_stride, dstused ? scratch : dst, wh.s.x, wh.s.y);
        dstused++;
    }
    if (pos & 0x4e40)// 0100 1110 0100 0000
    {
        hpel_lpf_diag(src, src_stride, dstused ? scratch : dst, wh.s.x, wh.s.y);
        dstused++;
    }
    if (pos & 0xfafa)// 1111 1010 1111 1010
    {
        assert(wh.s.x == 16 && wh.s.y == 16);
        if (dstused == 2)
        {
            point_t p;

            src = scratch;
            src_stride = 16;
            p.u32 = 16 + (16<<16);

            h264e_qpel_average_wh_align(src, dst, dst, p);
            return;
        } else
        {
            src += ((dxdy.s.x + 1) >> 2) + ((dxdy.s.y + 1) >> 2)*src_stride;
        }
        average_16x16_unalign(dst, src, src_stride);
    }
}

static void h264e_qpel_interpolate_chroma(const uint8_t *src, int src_stride, uint8_t *h264e_restrict dst, point_t wh, point_t dxdy)
{
    /* if fractionl mv is not (0, 0) */
    if (dxdy.u32)
    {
        int a = (8 - dxdy.s.x) * (8 - dxdy.s.y);
        int b = dxdy.s.x * (8 - dxdy.s.y);
        int c = (8 - dxdy.s.x) * dxdy.s.y;
        int d = dxdy.s.x * dxdy.s.y;
        int h = wh.s.y;
        do
        {
            int x;
            for (x = 0; x < wh.s.x; x++)
            {
                dst[x] = (uint8_t)((
                   a * src[             x] + b * src[             x + 1] +
                   c * src[src_stride + x] + d * src[src_stride + x + 1] +
                   32) >> 6);
            }
            dst += 16;
            src += src_stride;
        } while (--h);
    } else
    {
        copy_wh(src, src_stride, dst, wh.s.x, wh.s.y);
    }
}

static int sad_block(const pix_t *a, int a_stride, const pix_t *b, int b_stride, int w, int h)
{
    int r, c, sad = 0;
    for (r = 0; r < h; r++)
    {
        for (c = 0; c < w; c++)
        {
            int d = a[c] - b[c];
            sad += ABS(d);
        }
        a += a_stride;
        b += b_stride;
    }
    return sad;
}

static int h264e_sad_mb_unlaign_8x8(const pix_t *a, int a_stride, const pix_t *b, int sad[4])
{
    sad[0] = sad_block(a,     a_stride, b,     16, 8, 8);
    sad[1] = sad_block(a + 8, a_stride, b + 8, 16, 8, 8);
    a += 8*a_stride;
    b += 8*16;
    sad[2] = sad_block(a,     a_stride, b,     16, 8, 8);
    sad[3] = sad_block(a + 8, a_stride, b + 8, 16, 8, 8);
    return sad[0] + sad[1] + sad[2] + sad[3];
}

static int h264e_sad_mb_unlaign_wh(const pix_t *a, int a_stride, const pix_t *b, point_t wh)
{
    return sad_block(a, a_stride, b, 16, wh.s.x, wh.s.y);
}

static void h264e_copy_8x8(pix_t *d, int d_stride, const pix_t *s)
{
    int cloop = 8;
    assert(IS_ALIGNED(d, 8));
    assert(IS_ALIGNED(s, 8));
    do
    {
        int a = ((const int*)s)[0];
        int b = ((const int*)s)[1];
        ((int*)d)[0] = a;
        ((int*)d)[1] = b;
        s += 16;
        d += d_stride;
    } while(--cloop);
}

static void h264e_copy_16x16(pix_t *d, int d_stride, const pix_t *s, int s_stride)
{
    int cloop = 16;
    assert(IS_ALIGNED(d, 8));
    assert(IS_ALIGNED(s, 8));
    do
    {
        int a = ((const int*)s)[0];
        int b = ((const int*)s)[1];
        int x = ((const int*)s)[2];
        int y = ((const int*)s)[3];
        ((int*)d)[0] = a;
        ((int*)d)[1] = b;
        ((int*)d)[2] = x;
        ((int*)d)[3] = y;
        s += s_stride;
        d += d_stride;
    } while(--cloop);
}
#endif /* H264E_ENABLE_PLAIN_C */

#if H264E_ENABLE_PLAIN_C || (H264E_ENABLE_NEON && !defined(MINIH264_ASM))
static void h264e_copy_borders(unsigned char *pic, int w, int h, int guard)
{
    int r, rowbytes = w + 2*guard;
    unsigned char *d = pic - guard;
    for (r = 0; r < h; r++, d += rowbytes)
    {
        memset(d, d[guard], guard);
        memset(d + rowbytes - guard, d[rowbytes - guard - 1], guard);
    }
    d = pic - guard - guard*rowbytes;
    for (r = 0; r < guard; r++)
    {
        memcpy(d, pic - guard, rowbytes);
        memcpy(d + (guard + h)*rowbytes, pic - guard + (h - 1)*rowbytes, rowbytes);
        d += rowbytes;
    }
}
#endif /* H264E_ENABLE_PLAIN_C || (H264E_ENABLE_NEON && !defined(MINIH264_ASM)) */

#if H264E_ENABLE_PLAIN_C
#undef TRANSPOSE_BLOCK
#define TRANSPOSE_BLOCK     1
#define UNZIGSAG_IN_QUANT   0
#define SUM_DIF(a, b) { int t = a + b; b = a - b; a = t; }

static int clip_byte(int x)
{
    if (x > 255)
    {
        x = 255;
    } else if (x < 0)
    {
        x = 0;
    }
    return x;
}

static void hadamar4_2d(int16_t *x)
{
    int s = 1;
    int sback = 1;
    int16_t tmp[16];
    int16_t *out = tmp;
    int16_t *p = x;
    do
    {
        int cloop = 4;
        do
        {
            int a, b, c, d;
            a = *p; p += 4;//s;
            b = *p; p += 4;//s;
            c = *p; p += 4;//s;
            d = *p; p -= 11;//sback;
            SUM_DIF(a, c);
            SUM_DIF(b, d);
            SUM_DIF(a, b);
            SUM_DIF(c, d);

            *out = (int16_t)a; out += s;
            *out = (int16_t)c; out += s;
            *out = (int16_t)d; out += s;
            *out = (int16_t)b; out += sback;
        } while (--cloop);
        s = 5 - s;
        sback = -11;
        out = x;
        p = tmp;
    } while (s != 1);
}

static void dequant_dc(quant_t *q, int16_t *qval, int dequant, int n)
{
    do q++->dq[0] = (int16_t)(*qval++ * (int16_t)dequant); while (--n);
}

static void quant_dc(int16_t *qval, int16_t *deq, int16_t quant, int n, int round_q18)
{
#if UNZIGSAG_IN_QUANT
    int r_minus =  (1 << 18) - round_q18;
    static const uint8_t iscan16[16] = {0, 1, 5, 6, 2, 4, 7, 12, 3, 8, 11, 13, 9, 10, 14, 15};
    static const uint8_t iscan4[4] = {0, 1, 2, 3};
    const uint8_t *scan = n == 4 ? iscan4 : iscan16;
    do
    {
        int v = *qval;
        int r = v < 0 ? r_minus : round_q18;
        deq[*scan++] = *qval++ = (v * quant + r) >> 18;
    } while (--n);
#else
    int r_minus =  (1<<18) - round_q18;
    do
    {
        int v = *qval;
        int r = v < 0 ? r_minus : round_q18;
        *deq++ = *qval++ = (v * quant + r) >> 18;
    } while (--n);
#endif
}

static void hadamar2_2d(int16_t *x)
{
    int a = x[0];
    int b = x[1];
    int c = x[2];
    int d = x[3];
    x[0] = (int16_t)(a + b + c + d);
    x[1] = (int16_t)(a - b + c - d);
    x[2] = (int16_t)(a + b - c - d);
    x[3] = (int16_t)(a - b - c + d);
}

static void h264e_quant_luma_dc(quant_t *q, int16_t *deq, const uint16_t *qdat)
{
    int16_t *tmp = ((int16_t*)q) - 16;
    hadamar4_2d(tmp);
    quant_dc(tmp, deq, qdat[0], 16, 0x20000);//0x15555);
    hadamar4_2d(tmp);
    assert(!(qdat[1] & 3));
    // dirty trick here: shift w/o rounding, since it have no effect  for qp >= 10 (or, to be precise, for qp => 9)
    dequant_dc(q, tmp, qdat[1] >> 2, 16);
}

static int h264e_quant_chroma_dc(quant_t *q, int16_t *deq, const uint16_t *qdat)
{
    // q point to scratch->qu or scratch->qv
    int16_t *tmp = ((int16_t*)q) - 16; // dcu/dcv
    hadamar2_2d(tmp);
    // deq: quant_dc_u/quant_dc_v
    quant_dc(tmp, deq, (int16_t)(qdat[0] << 1), 4, 0xAAAA);
    hadamar2_2d(tmp);
    assert(!(qdat[1] & 1));
    dequant_dc(q, tmp, qdat[1] >> 1, 4);
    return !!(tmp[0] | tmp[1] | tmp[2] | tmp[3]);
}

static const uint8_t g_idx2quant[16] =
{
    0, 2, 0, 2,
    2, 4, 2, 4,
    0, 2, 0, 2,
    2, 4, 2, 4
};

#define TRANSFORM(x0, x1, x2, x3, p, s) { \
    int t0 = x0 + x3;                     \
    int t1 = x0 - x3;                     \
    int t2 = x1 + x2;                     \
    int t3 = x1 - x2;                     \
    (p)[  0] = (int16_t)(t0 + t2);        \
    (p)[  s] = (int16_t)(t1*2 + t3);      \
    (p)[2*s] = (int16_t)(t0 - t2);        \
    (p)[3*s] = (int16_t)(t1 - t3*2);      \
}

static void FwdTransformResidual4x42(const uint8_t *inp, const uint8_t *pred,
    uint32_t inp_stride, int16_t *out)
{
    int i;
    int16_t tmp[16];

// For plain c, TRANSPOSE_BLOCK is always 1
// !!! The rersult has been transposed !!!
#if TRANSPOSE_BLOCK
    // Transform columns
    for (i = 0; i < 4; i++, pred++, inp++)
    {
        int f0 = inp[0] - pred[0];
        int f1 = inp[1*inp_stride] - pred[1*16];
        int f2 = inp[2*inp_stride] - pred[2*16];
        int f3 = inp[3*inp_stride] - pred[3*16];
        TRANSFORM(f0, f1, f2, f3, tmp + i*4, 1);
    }
    // Transform rows
    for (i = 0; i < 4; i++)
    {
        int d0 = tmp[i + 0];
        int d1 = tmp[i + 4];
        int d2 = tmp[i + 8];
        int d3 = tmp[i + 12];
        TRANSFORM(d0, d1, d2, d3, out + i, 4);
    }

#else
    /* Transform rows */
    for (i = 0; i < 16; i += 4)
    {
        int d0 = inp[0] - pred[0];
        int d1 = inp[1] - pred[1];
        int d2 = inp[2] - pred[2];
        int d3 = inp[3] - pred[3];
        TRANSFORM(d0, d1, d2, d3, tmp + i, 1);
        pred += 16;
        inp += inp_stride;
   }

    /* Transform columns */
    for (i = 0; i < 4; i++)
    {
        int f0 = tmp[i + 0];
        int f1 = tmp[i + 4];
        int f2 = tmp[i + 8];
        int f3 = tmp[i + 12];
        TRANSFORM(f0, f1, f2, f3, out + i, 4);
    }
#endif
}

static void TransformResidual4x4(int16_t *pSrc)
{
    int i;
    int16_t tmp[16];

    /* Transform rows */
    for (i = 0; i < 16; i += 4)
    {
// For plain c, UNZIGSAG_IN_QUANT is always 0, TRANSPOSE_BLOCK is always 1
#if TRANSPOSE_BLOCK
        int d0 = pSrc[(i >> 2) + 0];
        int d1 = pSrc[(i >> 2) + 4];
        int d2 = pSrc[(i >> 2) + 8];
        int d3 = pSrc[(i >> 2) + 12];
#else
        int d0 = pSrc[i + 0];
        int d1 = pSrc[i + 1];
        int d2 = pSrc[i + 2];
        int d3 = pSrc[i + 3];
#endif
        int e0 = d0 + d2;
        int e1 = d0 - d2;
        int e2 = (d1 >> 1) - d3;
        int e3 = d1 + (d3 >> 1);
        int f0 = e0 + e3;
        int f1 = e1 + e2;
        int f2 = e1 - e2;
        int f3 = e0 - e3;
        tmp[i + 0] = (int16_t)f0;
        tmp[i + 1] = (int16_t)f1;
        tmp[i + 2] = (int16_t)f2;
        tmp[i + 3] = (int16_t)f3;
    }

    /* Transform columns */
    for (i = 0; i < 4; i++)
    {
        int f0 = tmp[i + 0];
        int f1 = tmp[i + 4];
        int f2 = tmp[i + 8];
        int f3 = tmp[i + 12];
        int g0 = f0 + f2;
        int g1 = f0 - f2;
        int g2 = (f1 >> 1) - f3;
        int g3 = f1 + (f3 >> 1);
        int h0 = g0 + g3;
        int h1 = g1 + g2;
        int h2 = g1 - g2;
        int h3 = g0 - g3;
        pSrc[i + 0] = (int16_t)((h0 + 32) >> 6);
        pSrc[i + 4] = (int16_t)((h1 + 32) >> 6);
        pSrc[i + 8] = (int16_t)((h2 + 32) >> 6);
        pSrc[i + 12] = (int16_t)((h3 + 32) >> 6);
    }
}

static int is_zero(const int16_t *dat, int i0, const uint16_t *thr)
{
    int i;
    for (i = i0; i < 16; i++)
    {
        if ((unsigned)(dat[i] + thr[i & 7]) > (unsigned)2*thr[i & 7])
        {
            return 0;
        }
    }
    return 1;
}

static int is_zero4(const quant_t *q, int i0, const uint16_t *thr)
{
    return is_zero(q[0].dq, i0, thr) &&
           is_zero(q[1].dq, i0, thr) &&
           is_zero(q[4].dq, i0, thr) &&
           is_zero(q[5].dq, i0, thr);
}

static int zero_smallq(quant_t *q, int mode, const uint16_t *qdat)
{
    int zmask = 0;
    // If mode = QDQ_MODE_CHROMA, then i0=1, n=2
    int i, i0 = mode & 1, n = mode >> 1;
    if (mode == QDQ_MODE_INTER || mode == QDQ_MODE_CHROMA)
    {
        for (i = 0; i < n*n; i++)
        {
            // i0=1,跳过DC系数
            if (is_zero(q[i].dq, i0, qdat + OFFS_THR_1_OFF /* 10 */))
            {
                zmask |= (1 << i); //9.19
            }
        }
        if (mode == QDQ_MODE_INTER)   //8.27
        {
            if ((~zmask & 0x0033) && is_zero4(q +  0, i0, qdat + OFFS_THR_2_OFF)) zmask |= 0x33;
            if ((~zmask & 0x00CC) && is_zero4(q +  2, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 2);
            if ((~zmask & 0x3300) && is_zero4(q +  8, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 8);
            if ((~zmask & 0xCC00) && is_zero4(q + 10, i0, qdat + OFFS_THR_2_OFF)) zmask |= (0x33 << 10);
        }
    }
    return zmask;
}

static int quantize(quant_t *q, int mode, const uint16_t *qdat, int zmask)
{
    // For plain c, UNZIGSAG_IN_QUANT is always 0, TRANSPOSE_BLOCK is always 1
#if UNZIGSAG_IN_QUANT
#if TRANSPOSE_BLOCK
    // ; Zig-zag scan      Transposed zig-zag
    // ;    0 1 5 6        0 2 3 9
    // ;    2 4 7 C        1 4 8 A
    // ;    3 8 B D        5 7 B E
    // ;    9 A E F        6 C D F
    static const unsigned char iscan16[16] = { 0, 2, 3, 9, 1, 4, 8, 10, 5, 7, 11, 14, 6, 12, 13, 15 };
#else
    static const unsigned char iscan16[16] = { 0, 1, 5, 6, 2, 4, 7, 12, 3, 8, 11, 13, 9, 10, 14, 15 };
#endif
#endif
    int i, i0 = mode & 1, ccol, crow; // mode & 1，避免对chroma宏块的DC进行量化（索引从1开始）
    int nz_block_mask = 0;
    ccol = mode >> 1;
    crow = ccol;
    do
    {
        do
        {
            int nz_mask = 0;

            if (zmask & 1)
            {
                int32_t *p = (int32_t *)q->qv;
                *p++ = 0; *p++ = 0; *p++ = 0; *p++ = 0;
                *p++ = 0; *p++ = 0; *p++ = 0; *p++ = 0;
            } else
            {
                // For chroma mb, i = i0 = 1 means do not perform
                // quantization on DC
                for (i = i0; i < 16; i++)
                {   /*  0, 2, 0, 2,
                        2, 4, 2, 4,
                        0, 2, 0, 2,
                        2, 4, 2, 4
                    */
                    int off = g_idx2quant[i];
                    int v, round = qdat[OFFS_RND_INTER /* 6 */];

                    if (q->dq[i] < 0) round = 0xFFFF - round;

                    v = (q->dq[i]*qdat[off] /* MF */ + round) >> 16;
#if UNZIGSAG_IN_QUANT
                    if (v)
                        nz_mask |= 1 << iscan16[i];
                    q->qv[iscan16[i]] = (int16_t)v;
#else
                    if (v)
                        nz_mask |= 1 << i;
                    q->qv[i] = (int16_t)v;
#endif
                    q->dq[i] = (int16_t)(v*qdat[off + 1]);
                }
            }

            zmask >>= 1;
            nz_block_mask <<= 1;
            if (nz_mask)
                nz_block_mask |= 1;
            q++;
        } while (--ccol);
        ccol = mode >> 1;
    } while (--crow);
    return nz_block_mask;
}

static void transform(const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q)
{
    int crow = mode >> 1;
    int ccol = crow;

    do
    {
        do
        {
            FwdTransformResidual4x42(inp, pred, inp_stride, q->dq);
            q++;
            inp += 4;
            pred += 4;
        } while (--ccol);
        ccol = mode >> 1;
        inp += 4*(inp_stride - ccol);
        pred += 4*(16 - ccol);
    } while (--crow);
}

// luma :
// inp_stride = 16
// mode = QDQ_MODE_INTRA_4 = 2
// chroma:
// mode = QDQ_MODE_CHROMA = 5
static int h264e_transform_sub_quant_dequant(const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat)
{
    int zmask;
    transform(inp, pred, inp_stride, mode, q);
    if (mode & 1) // QDQ_MODE_INTRA_16 || QDQ_MODE_CHROMA
    {
        // For CHROMA, we need get DC of U/V,
        // store in dcu and dcv
        int cloop = (mode >> 1)*(mode >> 1);
        short *dc = ((short *)q) - 16; // Get address of dcu/dcv
        quant_t *pq = q;
        do
        {
            *dc++ = pq->dq[0];
            pq++;
        } while (--cloop);
    }
    // if mode = QDQ_MODE_INTRA_4, return 0 (no zero blocks)
    // 判断AC系数是否为0
    // zmask 4 bits； 1: AC全0
    zmask = zero_smallq(q, mode, qdat);
    return quantize(q, mode, qdat, zmask);
}

// For luma intra4x4 side = 1, mask = -1
static void h264e_transform_add(pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask)
{
    int crow = side;
    int ccol = crow;

    assert(IS_ALIGNED(out, 4));
    assert(IS_ALIGNED(pred, 4));
    assert(!(out_stride % 4));

    do
    {
        do
        {
            if (mask >= 0)
            {
                // copy 4x4
                pix_t *dst = out;
                *(uint32_t*)dst = *(uint32_t*)(pred + 0 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 1 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 2 * 16); dst += out_stride;
                *(uint32_t*)dst = *(uint32_t*)(pred + 3 * 16);
            } else
            {
                int i, j;
                // Inverse Transform？
                TransformResidual4x4(q->dq);
                for (j = 0; j < 4; j++)
                {
                    for (i = 0; i < 4; i++)
                    {
                        int Value = q->dq[j * 4 + i] + pred[j * 16 + i];
                        out[j * out_stride + i] = (pix_t)clip_byte(Value);
                    }
                }
            }
            mask = (uint32_t)mask << 1;
            q++;
            out += 4;
            pred += 4;
        } while (--ccol);
        ccol = side;
        out += 4*(out_stride - ccol);
        pred += 4*(16 - ccol);
    } while (--crow);
}
#endif /* H264E_ENABLE_PLAIN_C */

#if H264E_ENABLE_PLAIN_C || (H264E_ENABLE_NEON && !defined(MINIH264_ASM))

#define BS_BITS 32

static void h264e_bs_put_bits(bs_t *bs, unsigned n, unsigned val)
{
    assert(!(val >> n));
    bs->shift -= n;
    assert((unsigned)n <= 32);
    if (bs->shift < 0)
    {
        assert(-bs->shift < 32);
        bs->cache |= val >> -bs->shift;
        *bs->buf++ = SWAP32(bs->cache);
        bs->shift = 32 + bs->shift;
        bs->cache = 0;
    }
    bs->cache |= val << bs->shift;
}

static void h264e_bs_flush(bs_t *bs)
{
    *bs->buf = SWAP32(bs->cache);
}

static unsigned h264e_bs_get_pos_bits(const bs_t *bs)
{
    unsigned pos_bits = (unsigned)((bs->buf - bs->origin)*BS_BITS);
    pos_bits += BS_BITS - bs->shift;
    assert((int)pos_bits >= 0);
    return pos_bits;
}

static unsigned h264e_bs_byte_align(bs_t *bs)
{
    int pos = h264e_bs_get_pos_bits(bs);
    h264e_bs_put_bits(bs, -pos & 7, 0);
    return pos + (-pos & 7);
}

/**
*   Golomb code
*   0 => 1
*   1 => 01 0
*   2 => 01 1
*   3 => 001 00
*   4 => 001 01
*
*   [0]     => 1
*   [1..2]  => 01x
*   [3..6]  => 001xx
*   [7..14] => 0001xxx
*
*/
static void h264e_bs_put_golomb(bs_t *bs, unsigned val)
{
#ifdef __arm__
    int size = 32 - __clz(val + 1);
#else
    int size = 0;
    unsigned t = val + 1;
    do
    {
        size++;
    } while (t >>= 1);
#endif
    h264e_bs_put_bits(bs, 2*size - 1, val + 1);
}

/**
*   signed Golomb code.
*   mapping to unsigned code:
*       0 => 0
*       1 => 1
*      -1 => 2
*       2 => 3
*      -2 => 4
*       3 => 5
*      -3 => 6
*/
static void h264e_bs_put_sgolomb(bs_t *bs, int val)
{
    val = 2*val - 1;
    val ^= val >> 31;
    h264e_bs_put_golomb(bs, val);
}

static void h264e_bs_init_bits(bs_t *bs, void *data)
{
    bs->origin = data;
    bs->buf = bs->origin;
    bs->shift = BS_BITS;
    bs->cache = 0;
}

static void h264e_vlc_encode(bs_t *bs, int16_t *quant, int maxNumCoeff, uint8_t *nz_ctx)
{
    int nnz_context, nlevels, nnz; // nnz = nlevels + trailing_ones
    int trailing_ones = 0;
    int trailing_ones_sign = 0;
    uint8_t runs[16];
    uint8_t *prun = runs;
    int16_t *levels;
    int cloop = maxNumCoeff;
    BS_OPEN(bs)

#if H264E_ENABLE_SSE2 || (H264E_ENABLE_PLAIN_C && !H264E_ENABLE_NEON)
    // this branch used with SSE + C configuration
    int16_t zzquant[16];
    levels = zzquant + ((maxNumCoeff == 4) ? 4 : 16);
    if (maxNumCoeff != 4)
    {
        int v;
        if (maxNumCoeff == 16)
        {
            v = quant[15]*2; if (v) *--levels = (int16_t)v, *prun++ = 16;
            v = quant[11]*2; if (v) *--levels = (int16_t)v, *prun++ = 15;
            v = quant[14]*2; if (v) *--levels = (int16_t)v, *prun++ = 14;
            v = quant[13]*2; if (v) *--levels = (int16_t)v, *prun++ = 13;
            v = quant[10]*2; if (v) *--levels = (int16_t)v, *prun++ = 12;
            v = quant[ 7]*2; if (v) *--levels = (int16_t)v, *prun++ = 11;
            v = quant[ 3]*2; if (v) *--levels = (int16_t)v, *prun++ = 10;
            v = quant[ 6]*2; if (v) *--levels = (int16_t)v, *prun++ =  9;
            v = quant[ 9]*2; if (v) *--levels = (int16_t)v, *prun++ =  8;
            v = quant[12]*2; if (v) *--levels = (int16_t)v, *prun++ =  7;
            v = quant[ 8]*2; if (v) *--levels = (int16_t)v, *prun++ =  6;
            v = quant[ 5]*2; if (v) *--levels = (int16_t)v, *prun++ =  5;
            v = quant[ 2]*2; if (v) *--levels = (int16_t)v, *prun++ =  4;
            v = quant[ 1]*2; if (v) *--levels = (int16_t)v, *prun++ =  3;
            v = quant[ 4]*2; if (v) *--levels = (int16_t)v, *prun++ =  2;
            v = quant[ 0]*2; if (v) *--levels = (int16_t)v, *prun++ =  1;
        } else
        {
            v = quant[15]*2; if (v) *--levels = (int16_t)v, *prun++ = 15;
            v = quant[11]*2; if (v) *--levels = (int16_t)v, *prun++ = 14;
            v = quant[14]*2; if (v) *--levels = (int16_t)v, *prun++ = 13;
            v = quant[13]*2; if (v) *--levels = (int16_t)v, *prun++ = 12;
            v = quant[10]*2; if (v) *--levels = (int16_t)v, *prun++ = 11;
            v = quant[ 7]*2; if (v) *--levels = (int16_t)v, *prun++ = 10;
            v = quant[ 3]*2; if (v) *--levels = (int16_t)v, *prun++ =  9;
            v = quant[ 6]*2; if (v) *--levels = (int16_t)v, *prun++ =  8;
            v = quant[ 9]*2; if (v) *--levels = (int16_t)v, *prun++ =  7;
            v = quant[12]*2; if (v) *--levels = (int16_t)v, *prun++ =  6;
            v = quant[ 8]*2; if (v) *--levels = (int16_t)v, *prun++ =  5;
            v = quant[ 5]*2; if (v) *--levels = (int16_t)v, *prun++ =  4;
            v = quant[ 2]*2; if (v) *--levels = (int16_t)v, *prun++ =  3;
            v = quant[ 1]*2; if (v) *--levels = (int16_t)v, *prun++ =  2;
            v = quant[ 4]*2; if (v) *--levels = (int16_t)v, *prun++ =  1;
        }
    } else
    {
        int v;
        v = quant[ 3]*2; if (v) *--levels = (int16_t)v, *prun++ = 4;
        v = quant[ 2]*2; if (v) *--levels = (int16_t)v, *prun++ = 3;
        v = quant[ 1]*2; if (v) *--levels = (int16_t)v, *prun++ = 2;
        v = quant[ 0]*2; if (v) *--levels = (int16_t)v, *prun++ = 1;
    }
    quant = zzquant + ((maxNumCoeff == 4) ? 4 : 16);
    nnz = (int)(quant - levels);
#else
    quant += (maxNumCoeff == 4) ? 4 : 16;
    levels = quant;
    do
    {
        int v = *--quant;
        if (v)
        {
            *--levels = v*2;
            *prun++ = cloop; // RunBefore
        }
    } while (--cloop);
    quant += maxNumCoeff;
    nnz = quant - levels;
#endif

    if (nnz)
    {
        cloop = MIN(3, nnz); // 拖尾系数数目不超过三个
        levels = quant - 1;
        do
        {
            if ((unsigned)(*levels + 2) > 4u)
            {
                break;
            }
            trailing_ones_sign = (trailing_ones_sign << 1) | (*levels-- < 0);
            trailing_ones++;
        } while (--cloop);
    }
    nlevels = nnz - trailing_ones; // 除了拖尾系数之外的非零系数个数

    nnz_context = nz_ctx[-1] + nz_ctx[1];

    nz_ctx[0] = (uint8_t)nnz; // 更新当前块非零系数的数目
    // 获取NC
    if (nnz_context <= 34)
    {
        nnz_context = (nnz_context + 1) >> 1;
    }
    nnz_context &= 31;

    // 9.2.1 Parsing process for total number of transform coefficient levels and trailing ones
    {
        int off = h264e_g_coeff_token[nnz_context]; // 选择非零系数数目和拖尾系数数目的编码表格
        int n = 6, val = h264e_g_coeff_token[off + trailing_ones + 4*nlevels];
        if (off != 230) // 非定长表格
        {
            n = (val & 15) + 1;
            val >>= 4;
        }
        BS_PUT(n, val);
    }

    if (nnz)
    {
        if (trailing_ones)
        {
            // 对每个拖尾系数的符号进行编码
            BS_PUT(trailing_ones, trailing_ones_sign);
        }
        if (nlevels)
        {
            // sym：幅值从有符号变为无符号
            // vlcnum: 编码后缀长度
            // prefix_len: 前缀
            int vlcnum = 1; // 初始化后缀长度为1,由于是直接跳入loop_enter，绕过第一次更新
            int sym_len, prefix_len;

            // levels指向是非零系数，已经乘2
            // 将非零系数从有符号变为无符号
            int sym = *levels-- - 2;
            if (sym < 0) sym = -3 - sym;

            if (sym >= 6) vlcnum++; // 更新后缀长度为2, 不应该放在这里
            if (trailing_ones < 3)
            {
                sym -= 2;  // 为什么会有一个减2的操作
                if (nnz > 10)
                {   // 此时suffixLength初始化为1, 此时编码前缀长度是幅值右移一位，原因suffixLength初始化为1
                    sym_len = 1;
                    prefix_len = sym >> 1;
                    if (prefix_len >= 15)
                    {
                        // or vlcnum = 1;  goto escape;
                        prefix_len = 15;
                        sym_len = 12;  // levelSuffixsSize=level_prefix - 3;
                    }
                    sym -= prefix_len << 1;
                    // bypass vlcnum advance due to sym -= 2; above
                    goto loop_enter;
                }
            }
            // 编码前缀长度等于幅值，并且编码前缀范围0~15
            if (sym < 14)
            {
                prefix_len = sym;
                sym = 0; // to avoid side effect in bitbuf
                sym_len = 0;
            } else if (sym < 15 /* 30 */) // 编码前缀长度等于14 ,修订
            {
                prefix_len = 14;
                sym_len = 4;
                sym -= 14; // 必须为0
            } else
            {
                vlcnum = 1;
                goto escape;
            }
            goto loop_enter;

            for (;;)
            {
                // vlc是当前的suffixLength
                sym_len = vlcnum;
                prefix_len = sym >> vlcnum; // 编码前缀长度
                if (prefix_len >= 15) // 编码前缀范围0~15
                {
escape:
                    prefix_len = 15;
                    sym_len = 12;
                }
                sym -= prefix_len << vlcnum; // 测试是恒为0，也必须恒为0

                if (prefix_len >= 3 && vlcnum < 6)
                    vlcnum++; // 更新后缀长度
loop_enter:
                // 此时 sym_len是levelSuffixsSize
                sym |= 1 << sym_len; // 此时是1加上编码后缀0的位数. 1左移后缀长度
                // 此时sym_len是整个码流的长度
                sym_len += prefix_len + 1; // 后缀长度 + 前缀长度 + 1（1的位置）
                BS_PUT(sym_len, sym); // 写入码流长度，补上前缀长度0的数目
                if (!--nlevels) break;
                sym = *levels-- - 2;
                if (sym < 0) sym = -3 - sym;
            }
        }

        if (nnz < maxNumCoeff)
        {
            const uint8_t *vlc = (maxNumCoeff == 4) ? h264e_g_total_zeros_cr_2x2 : h264e_g_total_zeros;
            uint8_t *run = runs;
            int run_prev = *run++; // 最后一个非零系数的索引（从1开始）
            int nzeros = run_prev - nnz; // 最后一个非零系数前零的数目
            int zeros_left = 2*nzeros /*- 1*/; // 不应该执行-1，修订;除非是码表有特殊的设计
            int ctx = nnz - 1;
            run[nnz - 1] = (uint8_t)maxNumCoeff; // terminator,终止条件
            for (;;)
            {
                int t;

                int val = vlc[vlc[ctx] + nzeros];
                int n = val & 15;
                val >>= 4;
                BS_PUT(n, val);// 第一次进入编码的是最后一个非0系数前0的数目, 后续编码RunBefore和ZerosLeft,且在反向扫描下最后一个非0系数的RunBefore和ZerosLeft无需编码；

                zeros_left -= nzeros; // 第一次时zeros_left=nzeros
                if (zeros_left < 0)
                {
                    break;
                }

                t = *run++; // 6
                nzeros = run_prev - t - 1; // RunBefore
                if (nzeros < 0)
                {
                    break;
                }
                run_prev = t;
                assert(zeros_left < 14);
                vlc = h264e_g_run_before; // run_before编码表
                ctx = zeros_left; // Zerosleft
            }
        }
    }
    BS_CLOSE(bs);
}
#endif /* H264E_ENABLE_PLAIN_C || (H264E_ENABLE_NEON && !defined(MINIH264_ASM)) */

#if H264E_SVC_API
static uint32_t udiv32(uint32_t n, uint32_t d)
{
    uint32_t q = 0, r = n, N = 16;
    do
    {
        N--;
        if ((r >> N) >= d)
        {
            r -= (d << N);
            q += (1 << N);
        }
    } while (N);
    return q;
}

static void h264e_copy_8x8_s(pix_t *d, int d_stride, const pix_t *s, int s_stride)
{
    int cloop = 8;
    assert(!((unsigned)(uintptr_t)d & 7));
    assert(!((unsigned)(uintptr_t)s & 7));
    do
    {
        int a = ((const int*)s)[0];
        int b = ((const int*)s)[1];
        ((int*)d)[0] = a;
        ((int*)d)[1] = b;
        s += s_stride;
        d += d_stride;
    } while(--cloop);
}

static void h264e_frame_downsampling(uint8_t *out, int wo, int ho,
    const uint8_t *src, int wi, int hi, int wo_Crop, int ho_Crop, int wi_Crop, int hi_Crop)
{
#define Q_BILIN 12
#define ONE_BILIN (1<<Q_BILIN)
    int r, c;
    int scaleh = udiv32(hi_Crop<<Q_BILIN, ho_Crop);
    int scalew = udiv32(wi_Crop<<Q_BILIN, wo_Crop);

    for (r = 0; r < ho_Crop; r++)
    {
        int dy = r*scaleh + (scaleh >> 2);
        int y = dy >> Q_BILIN;
        dy = dy & (ONE_BILIN - 1);

        for (c = 0; c < wo_Crop; c++)
        {
            int dx = c*scalew + (scalew >> 2);
            //          int dx = c*scalew;
            int x = dx >> Q_BILIN;
            const uint8_t *s0, *s1;
            uint8_t s00, s01, s10, s11;
            dx &= (ONE_BILIN - 1);


            s1 = s0 = src + x + y*wi;
            if (y < hi - 1)
            {
                s1 = s0 + wi;
            }

            s00 = s01 = s0[0];
            s10 = s11 = s1[0];
            if (x < wi - 1)
            {
                s01 = s0[1];
                s11 = s1[1];
            }

            *out++ =(uint8_t) ((((s11*dx + s10*(ONE_BILIN - dx)) >> (Q_BILIN - 1))*dy +
                ((s01*dx + s00*(ONE_BILIN - dx)) >> (Q_BILIN - 1))*(ONE_BILIN - dy) + (1 << (Q_BILIN + 1 - 1))) >> (Q_BILIN + 1));
        }
        if (wo > wo_Crop) //copy border
        {
            int cloop = wo - wo_Crop;
            uint8_t border = out[-1];
            do
            {
                *out++ = border;
            } while(--cloop);
        }
    }

    // copy bottom
    {
        int cloop = (ho - ho_Crop) * wo;
        if (cloop > 0)
        {
            do
            {
                *out = out[-wo];
                out++;
            } while(--cloop);
        }
    }
}

static int clip(int val, int max)
{
    if (val < 0) return 0;
    if (val > max) return max;
    return val;
}

static const int8_t g_filter16_luma[16][4] =
{
    {  0, 32,  0,  0 },
    { -1, 32,  2, -1 },
    { -2, 31,  4, -1 },
    { -3, 30,  6, -1 },
    { -3, 28,  8, -1 },
    { -4, 26, 11, -1 },
    { -4, 24, 14, -2 },
    { -3, 22, 16, -3 },
    { -3, 19, 19, -3 },
    { -3, 16, 22, -3 },
    { -2, 14, 24, -4 },
    { -1, 11, 26, -4 },
    { -1,  8, 28, -3 },
    { -1,  6, 30, -3 },
    { -1,  4, 31, -2 },
    { -1,  2, 32, -1 }
};

static void h264e_intra_upsampling(int srcw, int srch, int dstw, int dsth, int is_chroma,
    const uint8_t *arg_src, int src_stride, uint8_t *arg_dst, int dst_stride)
{
    int i, j;
    //===== set position calculation parameters =====
    int shift_x = 16;//(m_iLevelIdc <= 30 ? 16 : 31 - CeilLog2(iBaseW));
    int shift_y = 16;//(m_iLevelIdc <= 30 ? 16 : 31 - CeilLog2(iBaseH));
    int step_x  = udiv32(((unsigned int)srcw << shift_x) + (dstw >> 1), dstw);
    int step_y  = udiv32(((unsigned int)srch << shift_y) + (dsth >> 1), dsth);
    int start_x = udiv32((srcw << (shift_x - 1 - is_chroma)) + (dstw >> 1), dstw) + (1 << (shift_x - 5));
    int start_y = udiv32((srch << (shift_y - 1 - is_chroma)) + (dsth >> 1), dsth) + (1 << (shift_y - 5));
    int16_t *temp16 = (short*)(arg_dst + dst_stride*dsth) + 4;  // malloc(( iBaseH )*sizeof(short)); //ref frame have border =1 mb

    if (is_chroma)
    {
        int xpos = start_x - (4 << 12);
        for (i = 0; i < dstw; i++, xpos += step_x)
        {
            const uint8_t* src = arg_src;
            int xfrac  = (xpos >> 12) & 15;
            int xint = xpos >> 16;
            int m0 = clip(xint + 0, srcw - 1);
            int m1 = clip(xint + 1, srcw - 1);
            for( j = 0; j < srch ; j++ )
            {
                temp16[j] = (int16_t)(src[m1]*xfrac + src[m0]*(16 - xfrac));
                src += src_stride;
            }
            temp16[-1] = temp16[0];
            temp16[srch] = temp16[srch-1];

            //========== vertical upsampling ===========
            {
                int16_t* src16 = temp16;
                uint8_t* dst = arg_dst + i;
                int ypos = start_y - (4 << 12);
                for (j = 0; j < dsth; j++)
                {
                    int yfrac = (ypos >> 12) & 15;
                    int yint  = (ypos >> 16);
                    int acc = yfrac*src16[yint + 1] + (16 - yfrac)*src16[yint + 0];
                    acc = (acc + 128) >> 8;
                    *dst = (int8_t)acc;
                    dst += dst_stride;
                    ypos += step_y;
                }
            }
        }
    } else
    {
        int xpos = start_x - (8 << 12);
        for (i = 0; i < dstw; i++, xpos += step_x)
        {
            const uint8_t *src = arg_src;
            int xfrac    = (xpos >> 12) & 15;
            int xint   = xpos >> 16;
            int m0 = clip(xint - 1, srcw - 1);
            int m1 = clip(xint    , srcw - 1);
            int m2 = clip(xint + 1, srcw - 1);
            int m3 = clip(xint + 2, srcw - 1);
            //========== horizontal upsampling ===========
            for( j = 0; j < srch ; j++ )
            {
                int acc = 0;
                acc += g_filter16_luma[xfrac][0] * src[m0];
                acc += g_filter16_luma[xfrac][1] * src[m1];
                acc += g_filter16_luma[xfrac][2] * src[m2];
                acc += g_filter16_luma[xfrac][3] * src[m3];
                temp16[j] = (int16_t)acc;
                src += src_stride;
            }
            temp16[-2] = temp16[-1] = temp16[0];
            temp16[srch + 1] = temp16[srch] = temp16[srch - 1];

            //========== vertical upsampling ===========
            {
                int16_t *src16 = temp16;
                uint8_t *dst = arg_dst + i;
                int ypos = start_y - (8 << 12);

                for (j = 0; j < dsth; j++)
                {
                    int yfrac = (ypos >> 12) & 15;
                    int yint = ypos >> 16;
                    int acc = 512;
                    acc += g_filter16_luma[yfrac][0] * src16[yint + 0 - 1];
                    acc += g_filter16_luma[yfrac][1] * src16[yint + 1 - 1];
                    acc += g_filter16_luma[yfrac][2] * src16[yint + 2 - 1];
                    acc += g_filter16_luma[yfrac][3] * src16[yint + 3 - 1];
                    acc >>= 10;
                    if (acc < 0)
                    {
                        acc = 0;
                    }
                    if (acc > 255)
                    {
                        acc = 255;
                    }
                    *dst = (int8_t)acc;
                    dst += dst_stride;
                    ypos += step_y;
                }
            }
        }
    }
}
#endif /* H264E_SVC_API */

// Experimental code branch:
// Rate-control takes into account that long-term references compresses worser than short-term
#define H264E_RATE_CONTROL_GOLDEN_FRAMES 1

/************************************************************************/
/*      Constants (can't be changed)                                    */
/************************************************************************/

#define MIN_QP          10   // Minimum QP

#define MVPRED_MEDIAN   1
#define MVPRED_L        2
#define MVPRED_U        3
#define MVPRED_UR       4
#define MV_NA           0x8000
#define AVAIL(mv)       ((mv).u32 != MV_NA)

#define SLICE_TYPE_P    0
#define SLICE_TYPE_I    2

#define NNZ_NA          64

#define MAX_MV_CAND     20

#define STARTCODE_4BYTES 4

#define SCALABLE_BASELINE 83

/************************************************************************/
/*      Hardcoded params (can be changed at compile time)               */
/************************************************************************/
#define ALPHA_OFS       0       // Deblock alpha offset
#define BETA_OFS        0       // Deblock beta offset
#define DQP_CHROMA      0       // chroma delta QP

#define MV_RANGE        32      // Motion vector search range, pixels
#define MV_GUARD        14      // Out-of-frame MV's restriction, pixels

/************************************************************************/
/*      Code shortcuts                                                  */
/************************************************************************/
#define U(n,v) h264e_bs_put_bits(enc->bs, n, v)
#define U1(v)  h264e_bs_put_bits(enc->bs, 1, v)
#define UE(v)  h264e_bs_put_golomb(enc->bs, v)
#define SE(v)  h264e_bs_put_sgolomb(enc->bs, v)
#define SWAP(datatype, a, b) { datatype _ = a; a = b; b = _; }
#define SQR(x) ((x)*(x))
#define SQRP(pnt) SQR(pnt.s.x) + SQR(pnt.s.y)
#define SMOOTH(smth, p) smth.s.x = (63*smth.s.x + p.s.x + 32) >> 6;  smth.s.y = (63*smth.s.y + p.s.y + 32) >> 6;
#define MUL_LAMBDA(x, lambda) ((x)*(lambda) >> 4)

/************************************************************************/
/*      Optimized code fallback                                         */
/************************************************************************/

#if defined(MINIH264_ASM)
#include "asm/minih264e_asm.h"
#endif
#if H264E_ENABLE_NEON && defined(MINIH264_ASM)
#   define h264e_bs_put_bits_neon      h264e_bs_put_bits_arm11
#   define h264e_bs_flush_neon         h264e_bs_flush_arm11
#   define h264e_bs_get_pos_bits_neon  h264e_bs_get_pos_bits_arm11
#   define h264e_bs_byte_align_neon    h264e_bs_byte_align_arm11
#   define h264e_bs_put_golomb_neon    h264e_bs_put_golomb_arm11
#   define h264e_bs_put_sgolomb_neon   h264e_bs_put_sgolomb_arm11
#   define h264e_bs_init_bits_neon     h264e_bs_init_bits_arm11
#   define h264e_vlc_encode_neon       h264e_vlc_encode_arm11
#elif H264E_ENABLE_NEON
#   define h264e_bs_put_bits_neon      h264e_bs_put_bits
#   define h264e_bs_flush_neon         h264e_bs_flush
#   define h264e_bs_get_pos_bits_neon  h264e_bs_get_pos_bits
#   define h264e_bs_byte_align_neon    h264e_bs_byte_align
#   define h264e_bs_put_golomb_neon    h264e_bs_put_golomb
#   define h264e_bs_put_sgolomb_neon   h264e_bs_put_sgolomb
#   define h264e_bs_init_bits_neon     h264e_bs_init_bits
#   define h264e_vlc_encode_neon       h264e_vlc_encode
#   define h264e_copy_borders_neon     h264e_copy_borders
#endif

/************************************************************************/
/*      Declare exported functions for each configuration               */
/************************************************************************/
#if !H264E_CONFIGS_COUNT
#   error no build configuration defined
#elif H264E_CONFIGS_COUNT == 1
//  Exactly one configuration: append config suffix to exported names
#   if H264E_ENABLE_NEON
#       define MAP_NAME(name) name##_neon
#   endif
#   if H264E_ENABLE_SSE2
#       define MAP_NAME(name) name##_sse2
#   endif
#else //if H264E_CONFIGS_COUNT > 1
//  Several configurations: use Virtual Functions Table (VFT)
typedef struct
{
#   define  H264E_API(type, name, args) type (*name) args;
// h264e_qpel
H264E_API(void, h264e_qpel_interpolate_chroma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_interpolate_luma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_average_wh_align, (const uint8_t *p0, const uint8_t *p1, uint8_t *h264e_restrict d, point_t wh))
// h264e_deblock
H264E_API(void, h264e_deblock_chroma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
H264E_API(void, h264e_deblock_luma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
// h264e_intra
H264E_API(void, h264e_intra_predict_chroma,  (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(void, h264e_intra_predict_16x16, (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(int,  h264e_intra_choose_4x4, (const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty))
// h264e_cavlc
H264E_API(void,     h264e_bs_put_bits, (bs_t *bs, unsigned n, unsigned val))
H264E_API(void,     h264e_bs_flush, (bs_t *bs))
H264E_API(unsigned, h264e_bs_get_pos_bits, (const bs_t *bs))
H264E_API(unsigned, h264e_bs_byte_align, (bs_t *bs))
H264E_API(void,     h264e_bs_put_golomb, (bs_t *bs, unsigned val))
H264E_API(void,     h264e_bs_put_sgolomb, (bs_t *bs, int val))
H264E_API(void,     h264e_bs_init_bits, (bs_t *bs, void *data))
H264E_API(void,     h264e_vlc_encode, (bs_t *bs, int16_t *quant, int maxNumCoeff, uint8_t *nz_ctx))
// h264e_sad
H264E_API(int,  h264e_sad_mb_unlaign_8x8, (const pix_t *a, int a_stride, const pix_t *b, int sad[4]))
H264E_API(int,  h264e_sad_mb_unlaign_wh, (const pix_t *a, int a_stride, const pix_t *b, point_t wh))
H264E_API(void, h264e_copy_8x8, (pix_t *d, int d_stride, const pix_t *s))
H264E_API(void, h264e_copy_16x16, (pix_t *d, int d_stride, const pix_t *s, int s_stride))
H264E_API(void, h264e_copy_borders, (unsigned char *pic, int w, int h, int guard))
// h264e_transform
H264E_API(void, h264e_transform_add, (pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask))
H264E_API(int,  h264e_transform_sub_quant_dequant, (const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat))
H264E_API(void, h264e_quant_luma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
H264E_API(int,  h264e_quant_chroma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
// h264e_denoise
H264E_API(void, h264e_denoise_run, (unsigned char *frm, unsigned char *frmprev, int w, int h, int stride_frm, int stride_frmprev))
#   undef H264E_API
} vft_t;

// non-const VFT, run-time initialized
static const vft_t *g_vft;

// const VFT for each supported build config
#if H264E_ENABLE_PLAIN_C
static const vft_t g_vft_plain_c =
{
#define  H264E_API(type, name, args) name,
// h264e_qpel
H264E_API(void, h264e_qpel_interpolate_chroma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_interpolate_luma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_average_wh_align, (const uint8_t *p0, const uint8_t *p1, uint8_t *h264e_restrict d, point_t wh))
// h264e_deblock
H264E_API(void, h264e_deblock_chroma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
H264E_API(void, h264e_deblock_luma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
// h264e_intra
H264E_API(void, h264e_intra_predict_chroma,  (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(void, h264e_intra_predict_16x16, (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(int,  h264e_intra_choose_4x4, (const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty))
// h264e_cavlc
H264E_API(void,     h264e_bs_put_bits, (bs_t *bs, unsigned n, unsigned val))
H264E_API(void,     h264e_bs_flush, (bs_t *bs))
H264E_API(unsigned, h264e_bs_get_pos_bits, (const bs_t *bs))
H264E_API(unsigned, h264e_bs_byte_align, (bs_t *bs))
H264E_API(void,     h264e_bs_put_golomb, (bs_t *bs, unsigned val))
H264E_API(void,     h264e_bs_put_sgolomb, (bs_t *bs, int val))
H264E_API(void,     h264e_bs_init_bits, (bs_t *bs, void *data))
H264E_API(void,     h264e_vlc_encode, (bs_t *bs, int16_t *quant, int maxNumCoeff, uint8_t *nz_ctx))
// h264e_sad
H264E_API(int,  h264e_sad_mb_unlaign_8x8, (const pix_t *a, int a_stride, const pix_t *b, int sad[4]))
H264E_API(int,  h264e_sad_mb_unlaign_wh, (const pix_t *a, int a_stride, const pix_t *b, point_t wh))
H264E_API(void, h264e_copy_8x8, (pix_t *d, int d_stride, const pix_t *s))
H264E_API(void, h264e_copy_16x16, (pix_t *d, int d_stride, const pix_t *s, int s_stride))
H264E_API(void, h264e_copy_borders, (unsigned char *pic, int w, int h, int guard))
// h264e_transform
H264E_API(void, h264e_transform_add, (pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask))
H264E_API(int,  h264e_transform_sub_quant_dequant, (const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat))
H264E_API(void, h264e_quant_luma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
H264E_API(int,  h264e_quant_chroma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
// h264e_denoise
H264E_API(void, h264e_denoise_run, (unsigned char *frm, unsigned char *frmprev, int w, int h, int stride_frm, int stride_frmprev))
#undef H264E_API
};
#endif
#if H264E_ENABLE_NEON
static const vft_t g_vft_neon =
{
#define  H264E_API(type, name, args) name##_neon,
// h264e_qpel
H264E_API(void, h264e_qpel_interpolate_chroma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_interpolate_luma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_average_wh_align, (const uint8_t *p0, const uint8_t *p1, uint8_t *h264e_restrict d, point_t wh))
// h264e_deblock
H264E_API(void, h264e_deblock_chroma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
H264E_API(void, h264e_deblock_luma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
// h264e_intra
H264E_API(void, h264e_intra_predict_chroma,  (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(void, h264e_intra_predict_16x16, (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(int,  h264e_intra_choose_4x4, (const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty))
// h264e_cavlc
H264E_API(void,     h264e_bs_put_bits, (bs_t *bs, unsigned n, unsigned val))
H264E_API(void,     h264e_bs_flush, (bs_t *bs))
H264E_API(unsigned, h264e_bs_get_pos_bits, (const bs_t *bs))
H264E_API(unsigned, h264e_bs_byte_align, (bs_t *bs))
H264E_API(void,     h264e_bs_put_golomb, (bs_t *bs, unsigned val))
H264E_API(void,     h264e_bs_put_sgolomb, (bs_t *bs, int val))
H264E_API(void,     h264e_bs_init_bits, (bs_t *bs, void *data))
H264E_API(void,     h264e_vlc_encode, (bs_t *bs, int16_t *quant, int maxNumCoeff, uint8_t *nz_ctx))
// h264e_sad
H264E_API(int,  h264e_sad_mb_unlaign_8x8, (const pix_t *a, int a_stride, const pix_t *b, int sad[4]))
H264E_API(int,  h264e_sad_mb_unlaign_wh, (const pix_t *a, int a_stride, const pix_t *b, point_t wh))
H264E_API(void, h264e_copy_8x8, (pix_t *d, int d_stride, const pix_t *s))
H264E_API(void, h264e_copy_16x16, (pix_t *d, int d_stride, const pix_t *s, int s_stride))
H264E_API(void, h264e_copy_borders, (unsigned char *pic, int w, int h, int guard))
// h264e_transform
H264E_API(void, h264e_transform_add, (pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask))
H264E_API(int,  h264e_transform_sub_quant_dequant, (const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat))
H264E_API(void, h264e_quant_luma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
H264E_API(int,  h264e_quant_chroma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
// h264e_denoise
H264E_API(void, h264e_denoise_run, (unsigned char *frm, unsigned char *frmprev, int w, int h, int stride_frm, int stride_frmprev))
#undef H264E_API
};
#endif
#if H264E_ENABLE_SSE2
static const vft_t g_vft_sse2 =
{
#define  H264E_API(type, name, args) name##_sse2,
// h264e_qpel
H264E_API(void, h264e_qpel_interpolate_chroma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_interpolate_luma, (const uint8_t *src,int src_stride, uint8_t *h264e_restrict dst,point_t wh, point_t dxdy))
H264E_API(void, h264e_qpel_average_wh_align, (const uint8_t *p0, const uint8_t *p1, uint8_t *h264e_restrict d, point_t wh))
// h264e_deblock
H264E_API(void, h264e_deblock_chroma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
H264E_API(void, h264e_deblock_luma, (uint8_t *pSrcDst, int32_t srcdstStep, const deblock_params_t *par))
// h264e_intra
H264E_API(void, h264e_intra_predict_chroma,  (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(void, h264e_intra_predict_16x16, (pix_t *predict, const pix_t *left, const pix_t *top, int mode))
H264E_API(int,  h264e_intra_choose_4x4, (const pix_t *blockin, pix_t *blockpred, int avail, const pix_t *edge, int mpred, int penalty))
// h264e_cavlc
H264E_API(void,     h264e_bs_put_bits, (bs_t *bs, unsigned n, unsigned val))
H264E_API(void,     h264e_bs_flush, (bs_t *bs))
H264E_API(unsigned, h264e_bs_get_pos_bits, (const bs_t *bs))
H264E_API(unsigned, h264e_bs_byte_align, (bs_t *bs))
H264E_API(void,     h264e_bs_put_golomb, (bs_t *bs, unsigned val))
H264E_API(void,     h264e_bs_put_sgolomb, (bs_t *bs, int val))
H264E_API(void,     h264e_bs_init_bits, (bs_t *bs, void *data))
H264E_API(void,     h264e_vlc_encode, (bs_t *bs, int16_t *quant, int maxNumCoeff, uint8_t *nz_ctx))
// h264e_sad
H264E_API(int,  h264e_sad_mb_unlaign_8x8, (const pix_t *a, int a_stride, const pix_t *b, int sad[4]))
H264E_API(int,  h264e_sad_mb_unlaign_wh, (const pix_t *a, int a_stride, const pix_t *b, point_t wh))
H264E_API(void, h264e_copy_8x8, (pix_t *d, int d_stride, const pix_t *s))
H264E_API(void, h264e_copy_16x16, (pix_t *d, int d_stride, const pix_t *s, int s_stride))
H264E_API(void, h264e_copy_borders, (unsigned char *pic, int w, int h, int guard))
// h264e_transform
H264E_API(void, h264e_transform_add, (pix_t *out, int out_stride, const pix_t *pred, quant_t *q, int side, int32_t mask))
H264E_API(int,  h264e_transform_sub_quant_dequant, (const pix_t *inp, const pix_t *pred, int inp_stride, int mode, quant_t *q, const uint16_t *qdat))
H264E_API(void, h264e_quant_luma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
H264E_API(int,  h264e_quant_chroma_dc, (quant_t *q, int16_t *deq, const uint16_t *qdat))
// h264e_denoise
H264E_API(void, h264e_denoise_run, (unsigned char *frm, unsigned char *frmprev, int w, int h, int stride_frm, int stride_frmprev))
#undef H264E_API
};
#endif

/************************************************************************/
/*      Code to detect CPU features and init VFT                        */
/************************************************************************/

#if H264E_ENABLE_SSE2
#if defined(_MSC_VER)
#define minih264_cpuid __cpuid
#else
static __inline__ __attribute__((always_inline)) void minih264_cpuid(int CPUInfo[], const int InfoType)
{
#if defined(__PIC__)
    __asm__ __volatile__(
#if defined(__x86_64__)
        "push %%rbx\n"
        "cpuid\n"
        "xchgl %%ebx, %1\n"
        "pop  %%rbx\n"
#else /* defined(__x86_64__) */
        "xchgl %%ebx, %1\n"
        "cpuid\n"
        "xchgl %%ebx, %1\n"
#endif /* defined(__x86_64__) */
        : "=a" (CPUInfo[0]), "=r" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3])
        : "a" (InfoType));
#else /* defined(__PIC__) */
    __asm__ __volatile__(
        "cpuid"
        : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3])
        : "a" (InfoType));
#endif /* defined(__PIC__)*/
}
#endif /* defined(_MSC_VER) */

static int CPU_have_SSE2()
{
    int CPUInfo[4];
    minih264_cpuid(CPUInfo, 0);
    if (CPUInfo[0] > 0)
    {
        minih264_cpuid(CPUInfo, 1);
        if (CPUInfo[3] & (1 << 26))
            return 1;
    }
    return 0;
}
#endif

static void init_vft(int enableNEON)
{
#if H264E_ENABLE_PLAIN_C
    g_vft = &g_vft_plain_c;
#endif
    (void)enableNEON;
#if H264E_ENABLE_NEON
    if (enableNEON)
        g_vft = &g_vft_neon;
    else
        g_vft = &g_vft_plain_c;
#endif
#if H264E_ENABLE_SSE2
    if (CPU_have_SSE2())
    {
        g_vft = &g_vft_sse2;
    }
#endif
}

#define MAP_NAME(name) g_vft->name

#endif

#ifdef MAP_NAME
#   define h264e_qpel_interpolate_chroma     MAP_NAME(h264e_qpel_interpolate_chroma)
#   define h264e_qpel_interpolate_luma       MAP_NAME(h264e_qpel_interpolate_luma)
#   define h264e_qpel_average_wh_align       MAP_NAME(h264e_qpel_average_wh_align)
#   define h264e_deblock_chroma              MAP_NAME(h264e_deblock_chroma)
#   define h264e_deblock_luma                MAP_NAME(h264e_deblock_luma)
#   define h264e_intra_predict_chroma        MAP_NAME(h264e_intra_predict_chroma)
#   define h264e_intra_predict_16x16         MAP_NAME(h264e_intra_predict_16x16)
#   define h264e_intra_choose_4x4            MAP_NAME(h264e_intra_choose_4x4)
#   define h264e_bs_put_bits                 MAP_NAME(h264e_bs_put_bits)
#   define h264e_bs_flush                    MAP_NAME(h264e_bs_flush)
#   define h264e_bs_get_pos_bits             MAP_NAME(h264e_bs_get_pos_bits)
#   define h264e_bs_byte_align               MAP_NAME(h264e_bs_byte_align)
#   define h264e_bs_put_golomb               MAP_NAME(h264e_bs_put_golomb)
#   define h264e_bs_put_sgolomb              MAP_NAME(h264e_bs_put_sgolomb)
#   define h264e_bs_init_bits                MAP_NAME(h264e_bs_init_bits)
#   define h264e_vlc_encode                  MAP_NAME(h264e_vlc_encode)
#   define h264e_sad_mb_unlaign_8x8          MAP_NAME(h264e_sad_mb_unlaign_8x8)
#   define h264e_sad_mb_unlaign_wh           MAP_NAME(h264e_sad_mb_unlaign_wh)
#   define h264e_copy_8x8                    MAP_NAME(h264e_copy_8x8)
#   define h264e_copy_16x16                  MAP_NAME(h264e_copy_16x16)
#   define h264e_copy_borders                MAP_NAME(h264e_copy_borders)
#   define h264e_transform_add               MAP_NAME(h264e_transform_add)
#   define h264e_transform_sub_quant_dequant MAP_NAME(h264e_transform_sub_quant_dequant)
#   define h264e_quant_luma_dc               MAP_NAME(h264e_quant_luma_dc)
#   define h264e_quant_chroma_dc             MAP_NAME(h264e_quant_chroma_dc)
#   define h264e_denoise_run                 MAP_NAME(h264e_denoise_run)
#endif

/************************************************************************/
/*      Arithmetics                                                     */
/************************************************************************/

#ifndef __arm__
/**
*   Count of leading zeroes
*/
static unsigned __clz(unsigned v)
{
#if defined(_MSC_VER)
    unsigned long nbit;
    _BitScanReverse(&nbit, v);
    return 31 - nbit;
#elif defined(__GNUC__) || defined(__clang__) || defined(__aarch64__)
    return __builtin_clz(v);
#else
    unsigned clz = 32;
    assert(v);
    do
    {
        clz--;
    } while (v >>= 1);
    return clz;
#endif
}
#endif

/**
*   Size of unsigned Golomb code
*   ue: 使用于宏块类型和参考帧index等
*/
static int bitsize_ue(int v)
{
    return 2*(32 - __clz(v + 1)) - 1;
}

/**
*   Size of signed Golomb code
*/
static int bits_se(int v)
{
    v = 2*v - 1;
    v ^= v >> 31;
    return bitsize_ue(v);
}

/**
*   Multiply 32x32 Q16
*/
static uint32_t mul32x32shr16(uint32_t x, uint32_t y)
{
    uint32_t r = (x >> 16) * (y & 0xFFFFu) + x * (y >> 16) + ((y & 0xFFFFu) * (x & 0xFFFFu) >> 16);
    //assert(r == (uint32_t)((__int64)x*y>>16));
    return r;
}

/**
*   Integer division, producing Q16 output
*/
static uint32_t div_q16(uint32_t numer, uint32_t denum)
{
    unsigned f = 1 << __clz(denum);
    do
    {
        denum = denum * f >> 16;
        numer = mul32x32shr16(numer, f);
        f = ((1 << 17) - denum);
    } while (denum  != 0xffff);
    return numer;
}

/************************************************************************/
/*      Motion Vector arithmetics                                       */
/************************************************************************/

static point_t point(int x, int y)
{
    point_t p;
    p.u32 = ((unsigned)y << 16) | ((unsigned)x & 0xFFFF);    // assumes little-endian
    return p;
}

static int mv_is_zero(point_t p)
{
    return !p.u32;
}

static int mv_equal(point_t p0, point_t p1)
{
    return (p0.u32 == p1.u32);
}

/**
*   check that difference between given MV's components is greater than 3
*/
static int mv_differs3(point_t p0, point_t p1)
{
    return ABS(p0.s.x - p1.s.x) > 3 || ABS(p0.s.y - p1.s.y) > 3;
}

static point_t mv_add(point_t a, point_t b)
{
#if defined(__arm__)
    a.u32 = __sadd16(a.u32, b.u32);
#elif H264E_ENABLE_SSE2 && (H264E_CONFIGS_COUNT == 1)
    a.u32 = _mm_cvtsi128_si32(_mm_add_epi16(_mm_cvtsi32_si128(a.u32), _mm_cvtsi32_si128(b.u32)));
#else
    a.s.x += b.s.x;
    a.s.y += b.s.y;
#endif
    return a;
}

static point_t mv_sub(point_t a, point_t b)
{
#if defined(__arm__)
    a.u32 = __ssub16(a.u32, b.u32);
#elif H264E_ENABLE_SSE2 && (H264E_CONFIGS_COUNT == 1)
    a.u32 = _mm_cvtsi128_si32(_mm_sub_epi16(_mm_cvtsi32_si128(a.u32), _mm_cvtsi32_si128(b.u32)));
#else
    a.s.x -= b.s.x;
    a.s.y -= b.s.y;
#endif
    return a;
}

static void mv_clip(point_t *h264e_restrict p, const rectangle_t *range)
{
    p->s.x = MAX(p->s.x, range->tl.s.x);
    p->s.x = MIN(p->s.x, range->br.s.x);
    p->s.y = MAX(p->s.y, range->tl.s.y);
    p->s.y = MIN(p->s.y, range->br.s.y);
}

static int mv_in_rect(point_t p, const rectangle_t *r)
{
    return (p.s.y >= r->tl.s.y && p.s.y <= r->br.s.y && p.s.x >= r->tl.s.x && p.s.x <= r->br.s.x);
}

static point_t mv_round_qpel(point_t p)
{
    return point((p.s.x + 1) & ~3, (p.s.y + 1) & ~3);
}

/************************************************************************/
/*      Misc macroblock helper functions                                */
/************************************************************************/
/**
*   @return current macroblock input luma pixels
*/
static pix_t *mb_input_luma(h264e_enc_t *enc)
{
    return enc->inp.yuv[0] + (enc->mb.x + enc->mb.y*enc->inp.stride[0])*16;
}

/**
*   @return current macroblock input chroma pixels
*/
static pix_t *mb_input_chroma(h264e_enc_t *enc, int uv)
{
    return enc->inp.yuv[uv] + (enc->mb.x + enc->mb.y*enc->inp.stride[uv])*8;
}

/**
*   @return absolute MV for current macroblock for given MV
*/
static point_t mb_abs_mv(h264e_enc_t *enc, point_t mv)
{
    return mv_add(mv, point(enc->mb.x*64, enc->mb.y*64));
}

/************************************************************************/
/*      Pixel copy functions                                            */
/************************************************************************/
/**
*   Copy incomplete (cropped) macroblock pixels with borders extension
*/
static void pix_copy_cropped_mb(pix_t *d, int d_stride, const pix_t *s, int s_stride, int w, int h)
{
    int nbottom = d_stride - h; // assume dst = square d_strideXd_stride
    s_stride -= w;
    do
    {
        int cloop = w;
        pix_t last = 0;
        do
        {
            last = *s++;
            *d++ = last;
        } while (--cloop);
        cloop = d_stride - w;
        if (cloop) do
        {
            *d++ = last;    // extend row
        } while (--cloop);
        s += s_stride;
    } while (--h);
    s = d - d_stride;
    if (nbottom) do
    {
        memcpy(d, s, d_stride);  // extend columns
        d += d_stride;
    } while (--nbottom);
}

/**
*   Copy one image component
*/
static void pix_copy_pic(pix_t *dst, int dst_stride, pix_t *src, int src_stride, int w, int h)
{
    do
    {
        memcpy(dst, src, w);
        dst += dst_stride;
        src += src_stride;
    } while (--h);
}

/**
*   Copy reconstructed frame to reference buffer, with borders extensionn
*/
static void pix_copy_recon_pic_to_ref(h264e_enc_t *enc)
{
    int c, h = enc->frame.h, w = enc->frame.w, guard = 16;
    for (c = 0; c < 3; c++)
    {
        if (enc->param.const_input_flag)
        {
            SWAP(pix_t*, enc->ref.yuv[c], enc->dec.yuv[c]);
        } else
        {
            pix_copy_pic(enc->ref.yuv[c], w + 2*guard, enc->dec.yuv[c], w, w, h);
        }

        h264e_copy_borders(enc->ref.yuv[c], w, h, guard);
        if (!c) guard >>= 1, w >>= 1, h >>= 1;
    }
}

/************************************************************************/
/*      Median MV predictor                                             */
/************************************************************************/

/**
*   @return neighbors availability flags for current macroblock
*/
static int mb_avail_flag(const h264e_enc_t *enc)
{
    int nmb = enc->mb.num;
    // flag = 1 if AVAIL_T avail
    int flag = nmb >= enc->slice.start_mb_num + enc->frame.nmbx;
    if (nmb >= enc->slice.start_mb_num + enc->frame.nmbx - 1 && enc->mb.x != enc->frame.nmbx-1)
    {
        flag += AVAIL_TR;
    }
    if (nmb != enc->slice.start_mb_num && enc->mb.x)
    {
        flag += AVAIL_L;
    }
    if (nmb > enc->slice.start_mb_num + enc->frame.nmbx && enc->mb.x)
    {
        flag += AVAIL_TL;
    }
    return flag;
}

/**
*   @param n: Current 4x4 num in 16x16
*   @return neighbors availability flags for current 4x4 macroblock
*/
static int mb4x4_avail_flag(const h264e_enc_t *enc, int n) {
    /**
     *  The lower 4 bits of the flag indicate which neighboring properties
     *  the current 4x4 macroblock can inherit from the parent 16x16 macroblock.
     *  block2avail[0] is 0x07, The flag indicates that if the parent 16x16 has L, T, and TL adjacent data,
     *  then the 0th 4x4 macroblock must also have them.
     *  The upper 4 bits of the flag indicate the current attributes of the 4x4 macroblock itself
     *  (confined within the 16x16 macroblock). The 0th macroblock definitely does not have neighboring
     *  data from the top (T), left (L), top-left (TL), and top-right (TR)
    */
    static const uint8_t block2avail[16] = {
        0x07, 0x23, 0x23, 0x2b, 0x9b, 0x77, 0xff, 0x77, 0x9b, 0xff, 0xff, 0x77, 0x9b, 0x77, 0xff, 0x77,
    };
    int a, avail = mb_avail_flag(enc);
    a = avail;
    a &= block2avail[n];
    a |= block2avail[n] >> 4;

    if (!(block2avail[n] & AVAIL_TL)) // TL replace
    {
        if ((n <= 2 && (avail & AVAIL_T)) ||
            (n  > 2 && (avail & AVAIL_L)))
        {
            a |= AVAIL_TL;
        }
    }
    if (n < 2 && (avail & AVAIL_T))
    {
        a |= AVAIL_TR;
    }
}


/**
*   @return median of 3 given integers
*/
#if !(H264E_ENABLE_SSE2 && (H264E_CONFIGS_COUNT == 1))
static int me_median_of_3(int a, int b, int c)
{
    return MAX(MIN(MAX(a, b), c), MIN(a, b));
}
#endif

/**
*   @return median of 3 given motion vectors
*/
static point_t point_median_of_3(point_t a, point_t b, point_t c)
{
#if H264E_ENABLE_SSE2 && (H264E_CONFIGS_COUNT == 1)
    __m128i a2 = _mm_cvtsi32_si128(a.u32);
    __m128i b2 = _mm_cvtsi32_si128(b.u32);
    point_t med;
    med.u32 = _mm_cvtsi128_si32(_mm_max_epi16(_mm_min_epi16(_mm_max_epi16(a2, b2), _mm_cvtsi32_si128(c.u32)), _mm_min_epi16(a2, b2)));
    return med;
#else
    return point(me_median_of_3(a.s.x, b.s.x, c.s.x),
                 me_median_of_3(a.s.y, b.s.y, c.s.y));
#endif
}

/**
*   Save state of the MV predictor
*/
static void me_mv_medianpredictor_save_ctx(h264e_enc_t *enc, point_t *ctx)
{
    int i;
    point_t *mvtop = enc->mv_pred + 8 + enc->mb.x*4;
    for (i = 0; i < 4; i++)
    {
        *ctx++ = enc->mv_pred[i];
        *ctx++ = enc->mv_pred[4 + i];
        *ctx++ = mvtop[i];
    }
}

/**
*   Restore state of the MV predictor
*/
static void me_mv_medianpredictor_restore_ctx(h264e_enc_t *enc, const point_t *ctx)
{
    int i;
    point_t *mvtop = enc->mv_pred + 8 + enc->mb.x*4;
    for (i = 0; i < 4; i++)
    {
        enc->mv_pred[i] = *ctx++;
        enc->mv_pred[4 + i] = *ctx++;
        mvtop[i] = *ctx++;
    }
}

/**
*   Put motion vector to the deblock filter matrix.
*   x,y,w,h refers to 4x4 blocks within 16x16 macroblock, and should be in the range [0,4]
*/
static void me_mv_dfmatrix_put(point_t *dfmv, int x, int y, int w, int h, point_t mv)
{
    int i;
    assert(y < 4 && x < 4);

    dfmv += y*5 + x + 5;   // 5x5 matrix without left-top cell
    do
    {
        for (i = 0; i < w; i++)
        {
            dfmv[i] = mv;
        }
        dfmv += 5;
    } while (--h);
}

/**
*   Use given motion vector for prediction
*/
static void me_mv_medianpredictor_put(h264e_enc_t *enc, int x, int y, int w, int h, point_t mv)
{
    int i;
    point_t *mvtop = enc->mv_pred + 8 + enc->mb.x*4;
    assert(y < 4 && x < 4);

    enc->mv_pred[4 + y] = mvtop[x + w-1]; // top-left corner = top-right corner
    for (i = 1; i < h; i++)
    {
        enc->mv_pred[4 + y + i] = mv;     // top-left corner(s) for next row(s) = this
    }
    for (i = 0; i < h; i++)
    {
        enc->mv_pred[y + i] = mv;         // left = this
    }
    for (i = 0; i < w; i++)
    {
        mvtop[x + i] = mv;                // top = this
    }
}

/**
*   Motion vector median predictor for non-skip macroblock, as defined in the standard
*/
static point_t me_mv_medianpredictor_get(const h264e_enc_t *enc, point_t xy, point_t wh)
{
    int x = xy.s.x >> 2;
    int y = xy.s.y >> 2;
    int w = wh.s.x >> 2;
    int h = wh.s.y >> 2;
    int mvPredType = MVPRED_MEDIAN;
    point_t a, b, c, d, ret = point(0, 0);
    point_t *mvtop = enc->mv_pred + 8 + enc->mb.x*4;
    int flag = enc->mb.avail;

    assert(y < 4);
    assert(x < 4);
    assert(w <= 4);
    assert(h <= 4);

    a = enc->mv_pred[y];
    b = mvtop[x];
    c = mvtop[x + w];
    d = enc->mv_pred[4 + y];

    if (!x)
    {
        if (!(flag & AVAIL_L))
        {
            a.u32 = MV_NA;
        }
        if (!(flag & AVAIL_TL))
        {
            d.u32 = MV_NA;
        }
    }
    if (!y)
    {
        if (!(flag & AVAIL_T))
        {
            b.u32 = MV_NA;
            if (x + w < 4)
            {
                c.u32 = MV_NA;
            }
            if (x > 0)
            {
                d.u32 = MV_NA;
            }
        }
        if (!(flag & AVAIL_TL) && !x)
        {
            d.u32 = MV_NA;
        }
        if (!(flag & AVAIL_TR) && x + w == 4)
        {
            c.u32 = MV_NA;
        }
    }

    if (x + w == 4 && (!(flag & AVAIL_TR) || y))
    {
        c = d;
    }

    if (AVAIL(a) && !AVAIL(b) && !AVAIL(c))
    {
        mvPredType = MVPRED_L;
    } else if (!AVAIL(a) && AVAIL(b) && !AVAIL(c))
    {
        mvPredType = MVPRED_U;
    } else if (!AVAIL(a) && !AVAIL(b) && AVAIL(c))
    {
        mvPredType = MVPRED_UR;
    }

    // Directional predictions
    if (w == 2 && h == 4)
    {
        if (x == 0)
        {
            if (AVAIL(a))
            {
                mvPredType = MVPRED_L;
            }
        } else
        {
            if (AVAIL(c))
            {
                mvPredType = MVPRED_UR;
            }
        }
    } else if (w == 4 && h == 2)
    {
        if (y == 0)
        {
            if (AVAIL(b))
            {
                mvPredType = MVPRED_U;
            }
        } else
        {
            if (AVAIL(a))
            {
                mvPredType = MVPRED_L;
            }
        }
    }

    switch(mvPredType)
    {
    default:
    case MVPRED_MEDIAN:
        if (!(AVAIL(b) || AVAIL(c)))
        {
            if (AVAIL(a))
            {
                ret = a;
            }
        } else
        {
            if (!AVAIL(a))
            {
                a = ret;
            }
            if (!AVAIL(b))
            {
                b = ret;
            }
            if (!AVAIL(c))
            {
                c = ret;
            }
            ret = point_median_of_3(a, b, c);
        }
        break;
    case MVPRED_L:
        if (AVAIL(a))
        {
            ret = a;
        }
        break;
    case MVPRED_U:
        if (AVAIL(b))
        {
            ret = b;
        }
        break;
    case MVPRED_UR:
        if (AVAIL(c))
        {
            ret = c;
        }
        break;
    }
    return ret;
}

/**
*   Motion vector median predictor for skip macroblock
*/
static point_t me_mv_medianpredictor_get_skip(h264e_enc_t *enc)
{
    point_t pred_16x16 = me_mv_medianpredictor_get(enc, point(0, 0),  point(16, 16));
    enc->mb.mv_skip_pred = point(0, 0);
    if (!(~enc->mb.avail & (AVAIL_L | AVAIL_T)))
    {
        point_t *mvtop = enc->mv_pred + 8 + enc->mb.x*4;
        if (!mv_is_zero(enc->mv_pred[0]) && !mv_is_zero(mvtop[0]))
        {
            enc->mb.mv_skip_pred = pred_16x16;
        }
    }
    return pred_16x16;
}

/**
*   Get starting points candidates for MV search
*/
static int me_mv_medianpredictor_get_cand(const h264e_enc_t *enc, point_t *mv)
{
    point_t *mv0 = mv;
    point_t *mvtop = enc->mv_pred + 8 + enc->mb.x*4;
    int flag = enc->mb.avail;
    *mv++ = point(0, 0);
    if ((flag & AVAIL_L) && AVAIL(enc->mv_pred[0]))
    {
        *mv++ = enc->mv_pred[0];
    }
    if ((flag & AVAIL_T) && AVAIL(mvtop[0]))
    {
        *mv++ = mvtop[0];
    }
    if ((flag & AVAIL_TR) && AVAIL(mvtop[4]))
    {
        *mv++ = mvtop[4];
    }
    return (int)(mv - mv0);
}


/************************************************************************/
/*      NAL encoding                                                    */
/************************************************************************/

/**
*   Count ## of escapes, i.e. binary strings 0000 0000  0000 0000  0000 00xx
*   P(escape) = 2^-22
*   E(run_between_escapes) = 2^21 ~= 2 MB
*/
static int nal_count_esc(const uint8_t *s, int n)
{
    int i, cnt_esc = 0, cntz = 0;
    for (i = 0; i < n; i++)
    {
        uint8_t byte = *s++;
        if (cntz == 2 && byte <= 3)
        {
            cnt_esc++;
            cntz = 0;
        }

        if (byte)
        {
            cntz = 0;
        } else
        {
            cntz++;
        }
    }
    return cnt_esc;
}

/**
*   Put NAL escape codes to the output bitstream
*/
static int nal_put_esc(uint8_t *d, const uint8_t *s, int n)
{
    int i, j = 0, cntz = 0;
    for (i = 0; i < n; i++)
    {
        uint8_t byte = *s++;
        if (cntz == 2 && byte <= 3)
        {
            d[j++] = 3;
            cntz = 0;
        }

        if (byte)
        {
            cntz = 0;
        } else
        {
            cntz++;
        }
        d[j++] = byte;
    }
    assert(d + j <= s);
    return j;
}

/**
*   Init NAL encoding
*/
static void nal_start(h264e_enc_t *enc, int nal_hdr)
{
    uint8_t *d = enc->out + enc->out_pos;
    d[0] = d[1] = d[2] = 0; d[3] = 1; // start code
    enc->out_pos += STARTCODE_4BYTES;
    d += STARTCODE_4BYTES + (-(int)enc->out_pos & 3);   // 4-bytes align for bitbuffer
    assert(IS_ALIGNED(d, 4));
    h264e_bs_init_bits(enc->bs, d);
    U(8, nal_hdr);
}

/**
*   Finalize NAL encoding
*/
static void nal_end(h264e_enc_t *enc)
{
    int cnt_esc, bs_bytes;
    uint8_t *nal = enc->out + enc->out_pos;

    U1(1); // stop bit
    bs_bytes = h264e_bs_byte_align(enc->bs) >> 3;
    h264e_bs_flush(enc->bs);

    // count # of escape bytes to insert
    cnt_esc = nal_count_esc((unsigned char*)enc->bs->origin, bs_bytes);

    if ((uint8_t *)enc->bs->origin != nal + cnt_esc)
    {
        // make free space for escapes and remove align bytes
        memmove(nal + cnt_esc, enc->bs->origin, bs_bytes);
    }
    if (cnt_esc)
    {
        // insert escape bytes
        bs_bytes = nal_put_esc(nal, nal + cnt_esc, bs_bytes);
    }
    if (enc->run_param.nalu_callback)
    {
        // Call application-supplied callback
        enc->run_param.nalu_callback(nal, bs_bytes, enc->run_param.nalu_callback_token);
    }
    enc->out_pos += bs_bytes;
}


/************************************************************************/
/*      Top-level syntax elements (SPS,PPS,Slice)                       */
/************************************************************************/

/**
*   Encode Sequence Parameter Set (SPS)
*   ref: [1] 7.3.2.1.1
*/

//temp global
#define dependency_id 1
#define quality_id 0
#define default_base_mode_flag 0
#define log2_max_frame_num_minus4 1

static void encode_sps(h264e_enc_t *enc, int profile_idc)
{
    struct limit_t
    {
        uint8_t level;
        uint8_t constrains;
        uint16_t max_fs;
        uint16_t max_vbvdiv5;
        uint32_t max_dpb;
    };
    static const struct limit_t limit [] = {
        {10, 0xE0, 99,    175/5, 396},
        {10, 0xF0, 99,    350/5, 396},
        {11, 0xE0, 396,   500/5, 900},
        {12, 0xE0, 396,   1000/5, 2376},
        {13, 0xE0, 396,   2000/5, 2376},
        {20, 0xE0, 396,   2000/5, 2376},
        {21, 0xE0, 792,   4000/5, 4752},
        {22, 0xE0, 1620,  4000/5, 8100},
        {30, 0xE0, 1620,  10000/5, 8100},
        {31, 0xE0, 3600,  14000/5, 18000},
        {32, 0xE0, 5120,  20000/5, 20480},
        {40, 0xE0, 8192,  25000/5, 32768},
        {41, 0xE0, 8192,  62500/5, 32768},
        {42, 0xE0, 8704,  62500/5, 34816},
        {50, 0xE0, 22080, 135000/5, 110400},
        {51, 0xE0, 36864, 240000/5, 184320}
    };
    const struct limit_t *plim = limit;

    while (plim->level < 51 && (enc->frame.nmb > plim->max_fs ||
        enc->param.vbv_size_bytes > plim->max_vbvdiv5*(5*1000/8) ||
        (unsigned)(enc->frame.nmb*(enc->param.max_long_term_reference_frames + 1)) > plim->max_dpb))
    {
        plim++;
    }

    nal_start(enc, 0x67 | (profile_idc == SCALABLE_BASELINE)*8);
    U(8, profile_idc);  // profile, 66 = baseline
    U(8, plim->constrains & ((profile_idc!= SCALABLE_BASELINE)*4));     // no constrains
    U(8, plim->level);
    //U(5, 0x1B);       // sps_id|log2_max_frame_num_minus4|pic_order_cnt_type
    //UE(0);  // sps_id 1
    UE(enc->param.sps_id);

#if H264E_SVC_API
    if(profile_idc== SCALABLE_BASELINE)
    {
        UE(1); //chroma_format_idc
        UE(0); //bit_depth_luma_minus8
        UE(0); //bit_depth_chroma_minus8)
        U1(0); //qpprime_y_zero_transform_bypass_flag
        U1(0); //seq_scaling_matrix_present_flag
    }
#endif
    UE(log2_max_frame_num_minus4);  // log2_max_frame_num_minus4  1 UE(0);  // log2_max_frame_num_minus4  1
    UE(2);  // pic_order_cnt_type         011
    UE(1 + enc->param.max_long_term_reference_frames);  // num_ref_frames
    U1(0);                                      // gaps_in_frame_num_value_allowed_flag);
    UE(((enc->param.width + 15) >> 4) - 1);     // pic_width_in_mbs_minus1
    UE(((enc->param.height + 15) >> 4) - 1);    // pic_height_in_map_units_minus1
    U(3, 6 + enc->frame.cropping_flag);         // frame_mbs_only_flag|direct_8x8_inference_flag|frame_cropping_flag
//    U1(1);  // frame_mbs_only_flag
//    U1(1);  // direct_8x8_inference_flag
//    U1(frame_cropping_flag);  // frame_cropping_flag
    if (enc->frame.cropping_flag)
    {
        UE(0);                                          // frame_crop_left_offset
        UE((enc->frame.w - enc->param.width) >> 1);     // frame_crop_right_offset
        UE(0);                                          // frame_crop_top_offset
        UE((enc->frame.h - enc->param.height) >> 1);    // frame_crop_bottom_offset
    }
    U1(0);      // vui_parameters_present_flag

#if H264E_SVC_API
    if(profile_idc == SCALABLE_BASELINE)
    {
        U1(1);  //(inter_layer_deblocking_filter_control_present_flag); //inter_layer_deblocking_filter_control_present_flag
        U(2,0); //extended_spatial_scalability
        U1(0);  //chroma_phase_x_plus1_flag
        U(2,0); //chroma_phase_y_plus1

    /*    if( sps->sps_ext.extended_spatial_scalability == 1 )
        {
            //if( ChromaArrayType > 0 )
            {
                put_bits( s, 1,0);
                put_bits( s, 2,0); ///
            }
            put_bits_se( s, sps->sps_ext.seq_scaled_ref_layer_left_offset );
            put_bits_se( s, sps->sps_ext.seq_scaled_ref_layer_top_offset );
            put_bits_se( s, sps->sps_ext.seq_scaled_ref_layer_right_offset );
            put_bits_se( s, sps->sps_ext.seq_scaled_ref_layer_bottom_offset );
        }*/
        U1(0); //seq_tcoeff_level_prediction_flag
        U1(1); //slice_header_restriction_flag
        U1(0); //svc_vui_parameters_present_flag
        U1(0); //additional_extension2_flag
    }
#endif
    nal_end(enc);
}

/**
*   Encode Picture Parameter Set (SPS)
*   ref: [1] 7.3.2.2
*/
static void encode_pps(h264e_enc_t *enc, int pps_id)
{
    nal_start(enc, 0x68);
 //   U(10, 0x338);       // constant shortcut:
    UE(enc->param.sps_id*4 + pps_id);  // pic_parameter_set_id         1
    UE(enc->param.sps_id);  // seq_parameter_set_id         1
    U1(0);  // entropy_coding_mode_flag     0
    U1(0);  // pic_order_present_flag       0
    UE(0);  // num_slice_groups_minus1      1
    UE(0);  // num_ref_idx_l0_active_minus1 1
    UE(0);  // num_ref_idx_l1_active_minus1 1
    U1(0);  // weighted_pred_flag           0
    U(2,0); // weighted_bipred_idc          00
    SE(enc->sps.pic_init_qp - 26);  // pic_init_qp_minus26
#if DQP_CHROMA
    SE(0);  // pic_init_qs_minus26                    1
    SE(DQP_CHROMA);  // chroma_qp_index_offset        1
    U1(1);  // deblocking_filter_control_present_flag 1
    U1(0);  // constrained_intra_pred_flag            0
    U1(0);  // redundant_pic_cnt_present_flag         0
#else
    U(5, 0x1C);         // constant shortcut:
//     SE(0);  // pic_init_qs_minus26                    1
//     SE(0);  // chroma_qp_index_offset                 1
//     U1(1);  // deblocking_filter_control_present_flag 1
//     U1(0);  // constrained_intra_pred_flag            0
//     U1(0);  // redundant_pic_cnt_present_flag         0
#endif
    nal_end(enc);
}

/**
*   Encode Slice Header
*   ref: [1] 7.3.3
*/
static void encode_slice_header(h264e_enc_t *enc, int frame_type, int long_term_idx_use, int long_term_idx_update, int pps_id, int enc_type)
{
    // slice reset
    enc->slice.start_mb_num = enc->mb.num;
    enc->mb.skip_run = 0;
    memset(enc->i4x4mode, -1, (enc->frame.nmbx + 1)*4);
    memset(enc->nnz, NNZ_NA, (enc->frame.nmbx + 1)*8);    // DF ignore slice borders, but uses it's own nnz's

    if (enc_type == 0)
    {
#if H264E_SVC_API
        if (enc->param.num_layers > 1)
        {
            //need prefix nal for compatibility base layer with h264
            nal_start(enc, 14 | 0x40);
            //if((nal_unit_type == NAL_UNIT_TYPE_PREFIX_SCALABLE_EXT ) ||nal_unit_type == NAL_UNIT_TYPE_RBSP_SCALABLE_EXT))
            {
                //reserved_one_bit = 1    idr_flag                    priority_id
                U(8, (1 << 7) | ((frame_type == H264E_FRAME_TYPE_KEY) << 6) | 0);
                U1(1);   //no_inter_layer_pred_flag
                U(3, 0); //dependency_id
                U(4, quality_id); //quality_id
                //reserved_three_2bits = 3!
                U(3, 0); //temporal_id
                U1(1); //use_ref_base_pic_flag
                U1(0); //discardable_flag
                U1(1); //output_flag
                U(2, 3);

                U1(0); //store_ref_base_pic_flag
                if (!(frame_type == H264E_FRAME_TYPE_KEY))
                {
                    U1(0); //adaptive_ref_base_pic_marking_mode_flag  u(1)
                }

                U1(0); //prefix_nal_unit_additional_extension_flag 2 u(1)

                //put_bits_rbsp_trailing( s );
            }
            nal_end(enc);
        }
#endif //#if H264E_SVC_API
        nal_start(enc, (frame_type == H264E_FRAME_TYPE_KEY ? 5 : 1) | (long_term_idx_update >= 0 ? 0x60 : 0));
    }
#if H264E_SVC_API
    else
    {
        nal_start(enc, (20 | (long_term_idx_update >= 0 ? 0x60 : 0)));  //RBSP_SCALABLE_EXT = 20
        //nal_unit_type 20 or 14
        {
            //reserved_one_bit = 1    idr_flag                    priority_id
            U(8, (1 << 7) | ((frame_type == H264E_FRAME_TYPE_KEY) << 6) | 0);
            U1(!enc->param.inter_layer_pred_flag); //no_inter_layer_pred_flag
            U(3, dependency_id); //dependency_id
            U(4, quality_id);    //quality_id
            //reserved_three_2bits = 3!!!
            U(3, 0); //temporal_id
            U1(0); //use_ref_base_pic_flag
            U1(1); //discardable_flag
            U1(1); //output_flag
            U(2, 3);
        }
    }
#endif

    UE(enc->slice.start_mb_num);        // first_mb_in_slice
    UE(enc->slice.type);                // slice_type
    //U(1+4, 16 + (enc->frame.num&15));   // pic_parameter_set_id | frame_num
    UE(pps_id);                           // pic_parameter_set_id
    U(4 + log2_max_frame_num_minus4, enc->frame.num & ((1 << (log2_max_frame_num_minus4 + 4)) - 1)); // frame_num U(4, enc->frame.num&15);            // frame_num
    if (frame_type == H264E_FRAME_TYPE_KEY)
    {
        UE(enc->next_idr_pic_id);       // idr_pic_id
    }
    //!!!  if !quality_id && enc->slice.type == SLICE_TYPE_P  put_bit(s, 0); // num_ref_idx_active_override_flag = 0
    if(!quality_id)
    {
        if (((enc_type != 0)) && enc->slice.type == SLICE_TYPE_P)
        {
            //U1(0);
        }
        if (enc->slice.type == SLICE_TYPE_P)// if( slice_type == P  | |  slice_type ==  SP  | |  slice_type  = =  B )
        {
            int ref_pic_list_modification_flag_l0 = long_term_idx_use > 0;
            //U1(0);                      // num_ref_idx_active_override_flag
            // ref_pic_list_modification()
            U(2, ref_pic_list_modification_flag_l0); // num_ref_idx_active_override_flag | ref_pic_list_modification_flag_l0
            if (ref_pic_list_modification_flag_l0)
            {
                // Table 7-7
                UE(2);      // long_term_pic_num is present and specifies the long-term picture number for a reference picture
                UE(long_term_idx_use - 1); // long_term_pic_num
                UE(3);      // End loop
            }
        }

        if (long_term_idx_update >= 0)
        {
            //dec_ref_pic_marking( )
            if (frame_type == H264E_FRAME_TYPE_KEY)
            {
                //U1(0);                                      // no_output_of_prior_pics_flag
                //U1(enc->param.enable_golden_frames_flag);   // long_term_reference_flag
                U(2, enc->param.max_long_term_reference_frames > 0);   // no_output_of_prior_pics_flag | long_term_reference_flag
            } else
            {
                int adaptive_ref_pic_marking_mode_flag = long_term_idx_update > 0;//(frame_type == H264E_FRAME_TYPE_GOLDEN);
                U1(adaptive_ref_pic_marking_mode_flag);
                if (adaptive_ref_pic_marking_mode_flag)
                {
                    // Table 7-9
                    if (enc->short_term_used)
                    {
                        UE(1);  // unmark short
                        UE(0);  // unmark short
                    }
                    if (enc->lt_used[long_term_idx_update - 1])
                    {
                        UE(2);  // Mark a long-term reference picture as "unused for reference"
                        UE(long_term_idx_update - 1); // index
                    } else
                    {
                        UE(4);  // Specify the maximum long-term frame index
                        UE(enc->param.max_long_term_reference_frames);    // [0,max-1]+1
                    }
                    UE(6);  // Mark the current picture as "used for long-term reference"
                    UE(long_term_idx_update - 1);   // index
                    UE(0);  // End loop
                }
            }
        }
    }
    SE(enc->rc.prev_qp - enc->sps.pic_init_qp);     // slice_qp_delta
#if H264E_MAX_THREADS
    if (enc->param.max_threads > 1)
    {
        UE(enc->speed.disable_deblock ? 1 : 2);
    } else
#endif
    {
        UE(enc->speed.disable_deblock);             // disable deblock
    }

    if (enc->speed.disable_deblock != 1)
    {
#if ALPHA_OFS || BETA_OFS
        SE(ALPHA_OFS/2);                            // slice_alpha_c0_offset_div2
        SE(BETA_OFS/2);                             // slice_beta_offset_div2
#else
        U(2, 3);
#endif
    }

#if H264E_SVC_API
    if (enc_type != 0)
    {
        enc->adaptive_base_mode_flag = enc->param.inter_layer_pred_flag;
        if (enc->param.inter_layer_pred_flag && !quality_id)
        {
            UE(16*(dependency_id - 1));
            //if(1)//(inter_layer_deblocking_filter_control_present_flag)
            {
                UE(0);//disable_inter_layer_deblocking_filter_idc
                UE(0);
                UE(0);
            }
            /*if( sh->disable_inter_layer_deblocking_filter_idc != 1 )
            {
                put_bits_se(s, sh->slice_alpha_c0_offset_div2);
                put_bits_se(s, sh->slice_beta_offset_div2);
            }*/
            U1(0); // constrained_intra_resampling_flag 2 u(1)
        }
        if (enc->param.inter_layer_pred_flag)
        {
            U1(0); //slice_skip_flag u(1)
            {
                U1(enc->adaptive_base_mode_flag); // 2 u(1)
                if (!enc->adaptive_base_mode_flag)
                    U1(default_base_mode_flag); // 2 u(1)
                if (!default_base_mode_flag)
                {
                    U1(0); //adaptive_motion_prediction_flag) // 2 u(1)
                    U1(0); //sh->default_motion_prediction_flag// 2 u(1)
                }
                U1(0); //adaptive_residual_prediction_flag // 2 u(1)
                U1(0); //default_residual_prediction_flag // 2 u(1)
            }
        }
    }
#endif // #if H264E_SVC_API
}

/**
*   Macroblock transform, quantization and bitstream encoding
*   If luma mb is intra4x4, then only chroma mb
*   For intra4x4, enc_type = base_mode = 0
*/
static void mb_write(h264e_enc_t *enc, int enc_type, int base_mode)
{
    int i, uv, mb_type, cbpc, cbpl, cbp;
    scratch_t *qv = enc->scratch;
    //int base_mode = enc_type > 0 ? 1 : 0;
    int mb_type_svc = base_mode ? -2 : enc->mb.type; /* intra4x4 5; intra16x16 6; */
    int intra16x16_flag = mb_type_svc >= 6;// && !base_mode;
    uint8_t nz[9];
    /*nmbx*8 + 8*/
    uint8_t *nnz_top = enc->nnz + 8 + enc->mb.x*8;
    uint8_t *nnz_left = enc->nnz;

    if (enc->mb.type != 5)
    {
        enc->i4x4mode[0] = enc->i4x4mode[enc->mb.x + 1] = 0x02020202;
    }

    enc->df.nzflag = ((enc->df.nzflag >> 4) & 0x84210) | enc->df.df_nzflag[enc->mb.x];
    for (i = 0; i < 4; i++)
    {
        nz[5 + i] = nnz_top[i];
        nnz_top[i] = 0;
        nz[3 - i] = nnz_left[i];
        nnz_left[i] = 0;
    }

l_skip:
    if (enc->mb.type == -1)
    {
        // encode skip macroblock
        assert(enc->slice.type != SLICE_TYPE_I);

        // Increment run count
        enc->mb.skip_run++;

        // Update predictors
        *(uint32_t*)(nnz_top + 4) = *(uint32_t*)(nnz_left + 4) = 0; // set chroma NNZ to 0
        me_mv_medianpredictor_put(enc, 0, 0, 4, 4, enc->mb.mv[0]);
        me_mv_dfmatrix_put(enc->df.df_mv, 0, 0, 4, 4, enc->mb.mv[0]);

        // Update reference with reconstructed pixels
        h264e_copy_16x16(enc->dec.yuv[0], enc->dec.stride[0], enc->pbest, 16);
        h264e_copy_8x8(enc->dec.yuv[1], enc->dec.stride[1], enc->ptest);
        h264e_copy_8x8(enc->dec.yuv[2], enc->dec.stride[2], enc->ptest + 8);
    } else
    {
        if (enc->mb.type != 5)
        {
            unsigned nz_mask;
            nz_mask = h264e_transform_sub_quant_dequant(qv->mb_pix_inp, enc->pbest, 16, intra16x16_flag ? QDQ_MODE_INTRA_16 : QDQ_MODE_INTER, qv->qy, enc->rc.qdat[0]);
            enc->scratch->nz_mask = (uint16_t)nz_mask;
            if (intra16x16_flag)
            {
                h264e_quant_luma_dc(qv->qy, qv->quant_dc, enc->rc.qdat[0]);
                nz_mask = 0xFFFF;
            }
            h264e_transform_add(enc->dec.yuv[0], enc->dec.stride[0], enc->pbest, qv->qy, 4, nz_mask << 16);
        }

        // Coded Block Pattern for luma
        cbpl = 0;
        if (enc->scratch->nz_mask & 0xCC00) cbpl |= 1;
        if (enc->scratch->nz_mask & 0x3300) cbpl |= 2;
        if (enc->scratch->nz_mask & 0x00CC) cbpl |= 4;
        if (enc->scratch->nz_mask & 0x0033) cbpl |= 8;

        // Coded Block Pattern for chroma
        cbpc = 0;
        for (uv = 1; uv < 3; uv++) // Process UV separately
        {
            pix_t *pred = enc->ptest + (uv - 1)*8;
            pix_t *pix_mb_uv = mb_input_chroma(enc, uv);
            int dc_flag, inp_stride = enc->inp.stride[uv];
            unsigned nz_mask;
            quant_t *pquv = (uv == 1) ? qv->qu : qv->qv;

            if (enc->frame.cropping_flag && ((enc->mb.x + 1)*16  > enc->param.width || (enc->mb.y + 1)*16  > enc->param.height))
            {
                pix_copy_cropped_mb(enc->scratch->mb_pix_inp, 8, pix_mb_uv, enc->inp.stride[uv],
                    MIN(8, enc->param.width/2  - enc->mb.x*8),
                    MIN(8, enc->param.height/2 - enc->mb.y*8)
                    );
                pix_mb_uv = enc->scratch->mb_pix_inp;
                inp_stride = 8;
            }

            // nz_mask: x x x x, 4 bits
            // 根据移位，nz_mask的第三位对应第0个宏块，第0位对应第三个宏块；
            nz_mask = h264e_transform_sub_quant_dequant(pix_mb_uv, pred, inp_stride, QDQ_MODE_CHROMA, pquv, enc->rc.qdat[1]);

            if (nz_mask)
            {
                cbpc = 2; // 所有残差系数都被传送
            }
            // Process the chroma block DC coefficients separately.
            // Store dc quant coefficients in quant_dc_u/quant_dc_v
            // Store dc dequant coefficients in qu->dq/qv->dq
            // qv is scratch_t
            //  quant_t *pquv = (uv == 1) ? qv->qu : qv->qv;
            cbpc |= dc_flag = h264e_quant_chroma_dc(pquv, uv == 1 ? qv->quant_dc_u : qv->quant_dc_v, enc->rc.qdat[1]);

            if (!(dc_flag | nz_mask))
            {
                h264e_copy_8x8(enc->dec.yuv[uv], enc->dec.stride[uv], pred);
            } else
            {
                if (dc_flag)
                {
                    for (i = 0; i < 4; i++)
                    {
                        // If ac quant coefficients is zero, memset ac dequant coefficients as zero directly
                        if (~nz_mask & (8 >> i))
                        {
                            memset(pquv[i].dq + 1, 0, (16 - 1)*sizeof(int16_t));
                        }
                    }
                    nz_mask = 15;
                }
                // Reconstructed chroma mb in enc->dec.yuv[uv], ac Inverse Transform in qu->dq/qv->dq?
                // if dc_flag > 0, nz_mask = 15, then nz_mask << 28 = -1;
                h264e_transform_add(enc->dec.yuv[uv], enc->dec.stride[uv], pred, pquv, 2, nz_mask << 28);
            }
        }
        cbpc = MIN(cbpc, 2); // 只有0, 1, 2有意义

        // Rollback to skip
        // 判断是否跳过宏块编码，优化编码效率。
        if (!(enc->mb.type | cbpl | cbpc) && // Inter prediction, all-zero after quantization
            mv_equal(enc->mb.mv[0], enc->mb.mv_skip_pred)) // MV == MV preditor for skip
        {
            enc->mb.type = -1;
            goto l_skip;
        }

        mb_type = enc->mb.type;
        if (mb_type_svc >= 6)   // intra 16x16
        {
            if (cbpl)
            {
                cbpl = 15;
            }
            mb_type += enc->mb.i16.pred_mode_luma + cbpc*4 + (cbpl ? 12 : 0); // I_16x16_x_y_z
        }
        if (mb_type >= 5 && enc->slice.type == SLICE_TYPE_I)    // Intra in I slice
        {
            mb_type -= 5;
        }

        if (enc->slice.type != SLICE_TYPE_I)
        {
            UE(enc->mb.skip_run);
            enc->mb.skip_run = 0;
        }

        (void)enc_type;
#if H264E_SVC_API
        if (enc->adaptive_base_mode_flag && enc_type > 0)
            U1(base_mode);
#endif

        if (!base_mode)
            UE(mb_type); // for intra_4x4 write 0, intra_16x16 write I_16x16_x_y_z

        // -1 = skip, 0 = P16x16, 1 = P16x8, 2=P8x16, 3 = P8x8, 5 = I4x4, >=6 = I16x16
        if (enc->mb.type == 3) // P8x8
        {
            for (i = 0; i < 4; i++)
            {
                UE(0);
            }
            // 0 = 8x8
            // 1 = 8x4
            // 2 = 4x8
            // 3 = 4x4
        }

        if (!base_mode)
        {
            if (enc->mb.type >= 5)   // intra
            {
                int pred_mode_chroma;
                if (enc->mb.type == 5)  // intra 4x4
                {
                    for (i = 0; i < 16; i++)
                    {
                        // ZigZag scan
                        int m = enc->mb.i4x4_mode[decode_block_scan[i]];
                        int nbits =  4;
                        if (m < 0)
                        {
                            m = nbits = 1;
                        }
                        U(nbits, m); // intra 4x4 预测模式
                    }
                }
                pred_mode_chroma = enc->mb.i16.pred_mode_luma;
                if (!(pred_mode_chroma&1))
                {
                    pred_mode_chroma ^= 2;
                }
                UE(pred_mode_chroma); // chroma 8x8 预测模式
                me_mv_medianpredictor_put(enc, 0, 0, 4, 4, point(MV_NA,0)); // skip
            } else
            {
                int part, x = 0, y = 0;
                int dx = (enc->mb.type & 2) ? 2 : 4;
                int dy = (enc->mb.type & 1) ? 2 : 4;
                for (part = 0;;part++)
                {
                    SE(enc->mb.mvd[part].s.x);
                    SE(enc->mb.mvd[part].s.y);
                    me_mv_medianpredictor_put(enc, x, y, dx, dy, enc->mb.mv[part]);
                    me_mv_dfmatrix_put(enc->df.df_mv, x, y, dx, dy, enc->mb.mv[part]);
                    x = (x + dx) & 3;
                    if (!x)
                    {
                        y = (y + dy) & 3;
                        if (!y)
                        {
                            break;
                        }
                    }
                }
            }
        }
        cbp = cbpl + (cbpc << 4);
        /*temp for test up-sample filter*/
        /*if(base_mode)
        {
            cbp = 0;
            cbpl=0;
            cbpc = 0;
        }*/
        // 对于非intra_16x16的宏块类型CBP不在mb_type字段需要单独指定
        if (mb_type_svc < 6) /* intra_4x4 5; intra_16x16 6*/
        {
            // encode cbp 9.1.2 Mapping process for coded block pattern
            static const uint8_t cbp2code[2][48] = {
                {3, 29, 30, 17, 31, 18, 37,  8, 32, 38, 19,  9, 20, 10, 11,  2, 16, 33, 34, 21, 35, 22, 39,  4,
                36, 40, 23,  5, 24,  6,  7,  1, 41, 42, 43, 25, 44, 26, 46, 12, 45, 47, 27, 13, 28, 14, 15,  0},
                {0,  2,  3,  7,  4,  8, 17, 13,  5, 18,  9, 14, 10, 15, 16, 11,  1, 32, 33, 36, 34, 37, 44, 40,
                35, 45, 38, 41, 39, 42, 43, 19,  6, 24, 25, 20, 26, 21, 46, 28, 27, 47, 22, 29, 23, 30, 31, 12}
            };
            UE(cbp2code[mb_type_svc < 5][cbp]);
        }

        // 宏块层量化参数的偏移值
        if (cbp || (mb_type_svc >= 6))
        {
            SE(enc->rc.qp - enc->rc.prev_qp);
            enc->rc.prev_qp = enc->rc.qp;
        }

        // *** Huffman encoding ***

        // 1. Encode Luma DC (intra 16x16 only)
        if (intra16x16_flag)
        {
            h264e_vlc_encode(enc->bs, qv->quant_dc, 16, nz + 4);
        }

        // 2. Encode luma residual (only if CBP non-zero)
        if (cbpl)
        {
            for (i = 0; i < 16; i++)
            {
                /* 0 1 4 5
                 * 2 3 6 7
                 * 8 9 C D
                 * A B E F
                 */
                int j = decode_block_scan[i];
                if (cbp & (1 << (i >> 2)))
                {
                    uint8_t *pnz = nz + 4 + (j & 3) - (j >> 2);
                    h264e_vlc_encode(enc->bs, qv->qy[j].qv, 16 - intra16x16_flag, pnz);
                    if (*pnz)
                    {
                        enc->df.nzflag |= 1 << (5 + (j & 3) + 5*(j >> 2));
                    }
                } else
                {
                    nz[4 + (j & 3) - (j >> 2)] = 0;
                }
            }
            // 更新nnz
            for (i = 0; i < 4; i++)
            {
                nnz_top[i] = nz[1 + i];
                nnz_left[i] = nz[7 - i];
            }
        }

        // 2. Encode chroma
        if (cbpc)
        {
            uint8_t nzcdc[3];
            nzcdc[0] = nzcdc[2] = 17;   // dummy neighbors, indicating chroma DC
            // 2.1. Encode chroma DC
            for (uv = 1; uv < 3; uv++)
            {
                h264e_vlc_encode(enc->bs, uv == 1 ? qv->quant_dc_u : qv->quant_dc_v, 4, nzcdc + 1);
            }

            // 2.2. Encode chroma residual
            if (cbpc > 1)
            {
                for (uv = 1; uv < 3; uv++)
                {
                    uint8_t nzc[5];
                    int nnz_off = (uv == 1 ? 4 : 6);
                    quant_t *pquv = uv == 1 ? qv->qu : qv->qv;
                    for (i = 0; i < 2; i++)
                    {
                        nzc[3 + i] = nnz_top[nnz_off + i] ;
                        nzc[1 - i] = nnz_left[nnz_off + i];
                    }
                    for (i = 0; i < 4; i++)
                    {
                        int k = 2 + (i & 1) - (i >> 1);
                        h264e_vlc_encode(enc->bs, pquv[i].qv, 15, nzc + k);
                    }
                    for (i = 0; i < 2; i++)
                    {
                        nnz_top[nnz_off + i]  = nzc[1 + i];
                        nnz_left[nnz_off + i] = nzc[3 - i];
                    }
                }
            }
        }
        if (cbpc !=2)
        {
            *(uint32_t*)(nnz_top+4) = *(uint32_t*)(nnz_left+4) = 0; // set chroma NNZ to 0
        }
    }

    // Save top & left lines
    for (uv = 0; uv < 3; uv++)
    {
        int off = 0, n = uv ? 8 : 16;
        pix_t *top = enc->top_line + 48 + enc->mb.x*32;
        pix_t *left = enc->top_line;
        pix_t *mb = enc->dec.yuv[uv];

        if (uv)
        {
            off = 8 + uv*8;
        }
        top  += off;
        left += off;

        enc->top_line[32 + uv] = top[n - 1];
        for (i = 0; i < n; i++)
        {
            left[i] = mb[n - 1 + i*enc->dec.stride[uv]];
            top[i] = mb[(n - 1)*enc->dec.stride[uv] + i];
        }
    }
}

/************************************************************************/
/*      Intra mode encoding                                             */
/************************************************************************/
/**
*   Estimate cost of 4x4 intra predictor
*/
static void intra_choose_4x4(h264e_enc_t *enc)
{
    int i, n, a, nz_mask = 0;
    scratch_t *qv = enc->scratch;
    // Reconstruction current MB
    pix_t *mb_dec = enc->dec.yuv[0];
    pix_t *dec = enc->ptest;
    // Init cost according to qp
    int cost =  g_lambda_i4_q4[enc->rc.qp];// + MUL_LAMBDA(16, g_lambda_q4[enc->rc.qp]);    // 4x4 cost: at least 16 bits + penalty

    // 48 bytes of edge data
    uint32_t edge_store[(3 + 16 + 1 + 16 + 4)/4 + 2]; // pad for SSE
    pix_t *edge = ((pix_t*)edge_store) + 3 + 16 + 1;
    uint32_t *edge32 = (uint32_t *)edge;              // alias
    const uint32_t *top32 = (const uint32_t*)(enc->top_line + 48 + enc->mb.x*32);
    pix_t *left = enc->top_line;

    edge[-1] = enc->top_line[32]; // Luma UL
    for (i = 0; i < 16; i++)
    {
        edge[-2 - i] = left[i];
    }
    for (i = 0; i < 4; i++)
    {
        edge32[i] = top32[i];
    }
    edge32[4] = top32[8]; // 下一个宏块16x16的U0

    for (n = 0; n < 16; n++)
    {
        //#define AVAIL_TR    8
        //#define AVAIL_TL    4
        //#define AVAIL_L     2
        //#define AVAIL_T     1
#if 0
        static const uint8_t block2avail[16] = {
            0x07, 0x23, 0x23, 0x2b, 0x9b, 0x77, 0xff, 0x77, 0x9b, 0xff, 0xff, 0x77, 0x9b, 0x77, 0xff, 0x77,
        };

        // TODO: Maybe We should supply mb3x4_avail_flag()?
        a = avail;
        a &= block1avail[n];
        a |= block1avail[n] >> 4;

        if (!(block1avail[n] & AVAIL_TL)) // TL replace
        {
            if ((n <= 2 && (avail & AVAIL_T)) ||
                (n  > 2 && (avail & AVAIL_L)))
            {
                a |= AVAIL_TL;
            }
        }
        if (n < 2 && (avail & AVAIL_T))
        {
            a |= AVAIL_TR;
        }
#endif
        a = mb4x4_avail_flag(enc, n);
        pix_t *block;
        pix_t *blockin;
        int sad, mpred, mode;
        int r = n >> 2;
        int c = n & 3;
        int8_t *ctx_l = (int8_t *)enc->i4x4mode + r;
        int8_t *ctx_t = (int8_t *)enc->i4x4mode + 4 + enc->mb.x*4 + c;
        edge = ((pix_t*)edge_store) + 3 + 16 + 1 + 4*c - 4*r;

        // Input MB, size 16x16
        blockin = enc->scratch->mb_pix_inp + (c + r*16)*4;
        block = dec + (c + r*16)*4;

        mpred = MIN(*ctx_l, *ctx_t); // Default = -1, 预测模式中的较小者
        if (mpred < 0)
        {
            mpred = 2;
        }

        // Using Rate-Distortion optimization choose best mode
        // Return sad = best_m + (best_sad << 4);
        sad = h264e_intra_choose_4x4(blockin, block, a, edge, mpred, MUL_LAMBDA(3, g_lambda_q4[enc->rc.qp]));
        mode = sad & 15;
        sad >>= 4;

        *ctx_l = *ctx_t = (int8_t)mode;
        if (mode == mpred)
        {
            mode = -1; // 意味着当前4x4宏块的预测模式不需要编码。
        } else if (mode > mpred)
        {
            mode--;
        }
        enc->mb.i4x4_mode[n] = (int8_t)mode;

        nz_mask <<= 1;
        if (sad > g_skip_thr_i4x4[enc->rc.qp])
        {
            //  skip transform on low SAD gains just about 2% for all-intra coding at QP40,
            //  for other QP gain is minimal, so SAD check do not used
            //  transform quant and dequant
            //  nz_mask 用来存储16个4x4宏块量化后有非0元素宏块的位置信息
            nz_mask |= h264e_transform_sub_quant_dequant(blockin, block, 16, QDQ_MODE_INTRA_4, qv->qy + n, enc->rc.qdat[0]);

            // nz_mask最低位用来表示当前4x4宏块对残差量化后是否有非0值
            // 如果有的话则要对当前4x4宏块反量化（在前一步中已经完成）==>反变换==>重建原始数据，结果存在block中(该结构体之前存储宏块预测值)
            if (nz_mask & 1)
            {
                // Update qy->dq and block, block is the reconstructed block
                // block store pred value, qy->dq store dequant value
                h264e_transform_add(block, 16, block, qv->qy + n, 1, ~0);
            }
        } else
        {
            memset((qv->qy+n), 0, sizeof(qv->qy[0]));
        }

        // Accumulate cost of 4x4 blocks
        cost += sad;

        // Update edge data
        edge[2] = block[3];
        edge[1] = block[3 + 16];
        edge[0] = block[3 + 16*2];
        *(uint32_t*)&edge[-4] = *(uint32_t*)&block[16*3];
    }
    enc->scratch->nz_mask = (uint16_t)nz_mask;

    // If 4x4 is better than 16x16, restore 16x16 reference
    if (cost < enc->mb.cost)
    {
        enc->mb.cost = cost;
        enc->mb.type = 5;   // intra 4x4
        h264e_copy_16x16(mb_dec, enc->dec.stride[0], dec/* 重建后的数据 */, 16);  // restore reference
    }
}

/**
*   Choose 16x16 prediction mode, most suitable for given gradient
*/
static int intra_estimate_16x16(pix_t *p, int s, int avail, int qp)
{
    //#define AVAIL_TR    8
    //#define AVAIL_TL    4
    //#define AVAIL_L     2
    //#define AVAIL_T     1
    static const uint8_t mode_i16x16_valid[8] = { 4, 5, 6, 7, 4, 5, 6, 15 };
    pix_t p00 = p[0];
    pix_t p01 = p[15];
    pix_t p10 = p[15*s + 0];
    pix_t p11 = p[15*s + 15];
    int v = mode_i16x16_valid[avail & (AVAIL_T + AVAIL_L + AVAIL_TL)];
    // better than above on low bitrates
    // 水平差值
    int dx = ABS(p00 - p01) + ABS(p10 - p11) + ABS(p[8*s] - p[8*s + 15]);
    // 垂直差值
    int dy = ABS(p00 - p10) + ABS(p01 - p11) + ABS(p[8] - p[15*s + 8]);

    // 如果水平差值远大于垂直差值并且垂直差值足够小，采用编码模式0
    if ((dx > 30 + 3*dy && dy < (100 + 50 - qp)
        //|| (/*dx < 50 &&*/ dy <= 12)
        ) && (v & 1))
        return 0;
    else if (dy > 30 + 3*dx && dx < (100 + 50 - qp) && (v & (1 << 1)))
        return 1;
    else
    // 对于模式2(DC)，不需要top和left像素
        return 2;
}

/**
*   Estimate cost of 16x16 intra predictor
*
*   for foreman@qp10
*
*   12928 - [0-3], [0]
*   12963 - [0-2], [0]
*   12868 - [0-2], [0-3]
*   12878 - [0-2], [0-2]
*   12834 - [0-3], [0-3]
*sad
*   13182
*heuristic
*   13063
*
*/
static void intra_choose_16x16(h264e_enc_t *enc, pix_t *left, pix_t *top, int avail)
{
    int sad, sad4[4];
    // heuristic(启发式) mode decision
    enc->mb.i16.pred_mode_luma = intra_estimate_16x16(enc->scratch->mb_pix_inp, 16, avail, enc->rc.qp);

    // run chosen predictor
    h264e_intra_predict_16x16(enc->ptest, left, top, enc->mb.i16.pred_mode_luma);

    // coding cost
    // Calculate Rate-Distortion for each 16x16 block
    // g_lambda_q4 它在编码过程中用于平衡视频质量和比特率之间的权衡。使用量化系数作为索引
    sad = h264e_sad_mb_unlaign_8x8(enc->scratch->mb_pix_inp, 16, enc->ptest, sad4)        // SAD(绝对误差和)
        + MUL_LAMBDA(bitsize_ue(enc->mb.i16.pred_mode_luma + 1)/*预测模式ue编码长度*/, g_lambda_q4[enc->rc.qp]) // side-info penalty
        + g_lambda_i16_q4[enc->rc.qp];                                                    // block kind penalty

    if (sad < enc->mb.cost)
    {
        enc->mb.cost = sad;
        enc->mb.type = 6;
        /**
         * @brief Store the best 16x16 intra mode
         *
         */
        SWAP(pix_t*, enc->pbest, enc->ptest);
    }
}

/************************************************************************/
/*      Inter mode encoding                                             */
/************************************************************************/

/**
*   Sub-pel luma interpolation
*/
static void interpolate_luma(const pix_t *ref, int stride, point_t mv, point_t wh, pix_t *dst)
{
    ref += (mv.s.y >> 2) * stride + (mv.s.x >> 2);
    mv.u32 &= 0x000030003;
    h264e_qpel_interpolate_luma(ref, stride, dst, wh, mv);
}

/**
*   Sub-pel chroma interpolation
*/
static void interpolate_chroma(h264e_enc_t *enc, point_t mv)
{
    int i;
    for (i = 1; i < 3; i++)
    {
        point_t wh;
        int part = 0, x = 0, y = 0;
        wh.s.x = (enc->mb.type & 2) ? 4 : 8;
        wh.s.y = (enc->mb.type & 1) ? 4 : 8;
        if (enc->mb.type == -1) // skip
        {
            wh.s.x = wh.s.y = 8;
        }

        for (;;part++)
        {
            pix_t *ref;
            mv = mb_abs_mv(enc, enc->mb.mv[part]);
            ref = enc->ref.yuv[i] + ((mv.s.y >> 3) + y)*enc->ref.stride[i] + (mv.s.x >> 3) + x;
            mv.u32 &= 0x00070007;
            h264e_qpel_interpolate_chroma(ref, enc->ref.stride[i], enc->ptest + (i - 1)*8 + 16*y + x, wh, mv);
            x = (x + wh.s.x) & 7;
            if (!x)
            {
                y = (y + wh.s.y) & 7;
                if (!y)
                {
                    break;
                }
            }
        }
    }
}

/**
*   RD cost of given MV
*/
static int me_mv_cost(point_t mv, point_t mv_pred, int qp)
{
    int nb = bits_se(mv.s.x - mv_pred.s.x) + bits_se(mv.s.y - mv_pred.s.y);
    return MUL_LAMBDA(nb, g_lambda_mv_q4[qp]);
}

/**
*   RD cost of given MV candidate (TODO)
*/
#define me_mv_cand_cost me_mv_cost
//static int me_mv_cand_cost(point_t mv, point_t mv_pred, int qp)
//{
//    int nb = bits_se(mv.s.x - mv_pred.s.x) + bits_se(mv.s.y - mv_pred.s.y);
//    return MUL_LAMBDA(nb, g_lambda_mv_q4[qp]);
//}


/**
*   Modified full-pel motion search with small diamond algorithm
*   note: diamond implemented with small modifications, trading speed for precision
*/
static int me_search_diamond(h264e_enc_t *enc, const pix_t *ref, const pix_t *b, int rowbytes, point_t *mv,
    const rectangle_t *range, int qp, point_t mv_pred, int min_sad, point_t wh, pix_t *scratch, pix_t **ppbest, int store_bytes)
{
    // cache map           cache moves
    //      3              0   x->1
    //      *              1   x->0
    //  1 * x * 0          2   x->3
    //      *              3   x->2
    //      2                   ^1

    //   cache double moves:
    //           prev               prev
    //      x ->   0   ->   3   ==>   3   =>   1
    //      x ->   0   ->   2   ==>   2   =>   1
    //      x ->   0   ->   0   ==>   0   =>   1
    //      x ->   0   ->   1   - impossible
    //   prev SAD(n) is (n+4)
    //

    static const point_t dir2mv[] = {{{4, 0}},{{-4, 0}},{{0, 4}},{{0, -4}}};
    union
    {
        uint16_t cache[8];
        uint32_t cache32[4];
    } sad;

    int dir, cloop, dir_prev, cost;
    point_t v;

    assert(mv_in_rect(*mv, range));

restart:
    dir = 0;                // start gradient descend with direction dir2mv[0]
    cloop = 4;              // try 4 directions
    dir_prev = -1;          // not yet moved

    // reset SAD cache
    sad.cache32[0] = sad.cache32[1] = sad.cache32[2] = sad.cache32[3] = ~0u;

    // 1. Full-pel ME with small diamond modification:
    // center point moved immediately as soon as new minimum found
    do
    {
        assert(dir >= 0 && dir < 4);

        // Try next point. Avoid out-of-range moves
        v = mv_add(*mv, dir2mv[dir]);
        //if (mv_in_rect(v, range) && sad.cache[dir] == (uint16_t)~0u)
        if (mv_in_rect(v, range) && sad.cache[dir] == 0xffffu)
        {
            cost = h264e_sad_mb_unlaign_wh(ref + ((v.s.y*rowbytes + v.s.x) >> 2), rowbytes, b, wh);
            //cost += me_mv_cost(*mv, mv_pred, qp);
            cost += me_mv_cost(v, mv_pred, qp);
            sad.cache[dir] = (uint16_t)cost;
            if (cost < min_sad)
            {
                // This point is better than center: move this point to center and continue
                int corner = ~0;
                if (dir_prev >= 0)                      // have previous move
                {                                       // save cache point, which can be used in next iteration
                    corner = sad.cache[4 + dir];        // see "cache double moves" above
                }
                sad.cache32[2] = sad.cache32[0];        // save current cache to 'previous'
                sad.cache32[3] = sad.cache32[1];
                sad.cache32[0] = sad.cache32[1] = ~0u;  // reset current cache
                if (dir_prev >= 0)                      // but if have previous move
                {                                       // one cache point can be reused from previous iteration
                    sad.cache[dir_prev^1] = (uint16_t)corner; // see "cache double moves" above
                }
                sad.cache[dir^1] = (uint16_t)min_sad;   // previous center become a neighbor's
                dir_prev = dir;                         // save this direction
                dir--;                                  // start next iteration with the same direction
                cloop = 4 + 1;                          // and try 4 directions (+1 for do-while loop)
                *mv = v;                                // Save best point found
                min_sad = cost;                         // and it's SAD
            }
        }
        dir = (dir + 1) & 3;                            // cycle search directions
    } while(--cloop);

    // 2. Optional: Try diagonal step
    //if (1)
    {
        int primary_dir   = sad.cache[3] >= sad.cache[2] ? 2 : 3;
        int secondary_dir = sad.cache[1] >= sad.cache[0] ? 0 : 1;
        if (sad.cache[primary_dir] < sad.cache[secondary_dir])
        {
            SWAP(int, secondary_dir, primary_dir);
        }

        v = mv_add(dir2mv[secondary_dir], dir2mv[primary_dir]);
        v = mv_add(*mv, v);
        //cost = (uint16_t)~0u;
        if (mv_in_rect(v, range))
        {
            cost = h264e_sad_mb_unlaign_wh(ref + ((v.s.y*rowbytes + v.s.x) >> 2), rowbytes, b, wh);
            cost += me_mv_cost(v, mv_pred, qp);
            if (cost < min_sad)
            {
                *mv = v;//mv_add(*mv, v);
                min_sad = cost;
                goto restart;
            }
        }
    }

    interpolate_luma(ref, rowbytes, *mv, wh, scratch);    // Plain NxM copy can be used
    *ppbest = scratch;

    // 3. Fractional pel search
    if (enc->run_param.encode_speed < 9 && mv_in_rect(*mv, &enc->frame.mv_qpel_limit))
    {
        point_t vbest = *mv;
        pix_t *pbest = scratch;
        pix_t *hpel  = scratch + store_bytes;
        pix_t *hpel1 = scratch + ((store_bytes == 8) ? 256 : 2*store_bytes);
        pix_t *hpel2 = hpel1 + store_bytes;

        int i, sad_test;
        point_t primary_qpel, secondary_qpel, vdiag;

        unsigned minsad1 = sad.cache[1];
        unsigned minsad2 = sad.cache[3];
        secondary_qpel = point(-1, 0);
        primary_qpel = point(0, -1);
        if (sad.cache[3] >= sad.cache[2])
            primary_qpel = point(0, 1), minsad2 = sad.cache[2];
        if (sad.cache[1] >= sad.cache[0])
            secondary_qpel = point(1, 0), minsad1 = sad.cache[0];

        if (minsad2 > minsad1)
        {
            SWAP(point_t, secondary_qpel, primary_qpel);
        }

        //     ============> primary
        //     |00 01 02
        //     |10 11 12
        //     |20    22
        //     V
        //     secondary
        vdiag = mv_add(primary_qpel, secondary_qpel);

        for (i = 0; i < 7; i++)
        {
            pix_t *ptest;
            switch(i)
            {
            case 0:
                // 02 = interpolate primary half-pel
                v = mv_add(*mv, mv_add(primary_qpel, primary_qpel));
                interpolate_luma(ref, rowbytes, v, wh, ptest = hpel1);
                break;
            case 1:
                // 01 q-pel = (00 + 02)/2
                v = mv_add(*mv, primary_qpel);
                h264e_qpel_average_wh_align(scratch, hpel1, ptest = hpel, wh);
                break;
            case 2:
                // 20 = interpolate secondary half-pel
                v = mv_add(*mv, mv_add(secondary_qpel, secondary_qpel));
                interpolate_luma(ref, rowbytes, v, wh, ptest = hpel2);
                break;
            case 3:
                // 10 q-pel = (00 + 20)/2
                hpel  = scratch + store_bytes; if (pbest == hpel) hpel = scratch;
                v = mv_add(*mv, secondary_qpel);
                h264e_qpel_average_wh_align(scratch, hpel2, ptest = hpel, wh);
                break;
            case 4:
                // 11 q-pel = (02 + 20)/2
                hpel  = scratch + store_bytes; if (pbest == hpel) hpel = scratch;
                v = mv_add(*mv, vdiag);
                h264e_qpel_average_wh_align(hpel1, hpel2, ptest = hpel, wh);
                break;
            case 5:
                // 22 = interpolate center half-pel
                if (pbest == hpel2) hpel2 = scratch, hpel = scratch + store_bytes;
                v = mv_add(*mv, mv_add(vdiag, vdiag));
                interpolate_luma(ref, rowbytes, v, wh, ptest = hpel2);
                break;
            case 6:
            default:
                // 12 q-pel = (02 + 22)/2
                hpel  = scratch + store_bytes; if (pbest == hpel) hpel = scratch;
                v = mv_add(*mv, mv_add(primary_qpel, vdiag));
                h264e_qpel_average_wh_align(hpel2, hpel1, ptest = hpel, wh);
                break;
            }

            sad_test = h264e_sad_mb_unlaign_wh(ptest, 16, b, wh) + me_mv_cost(v, mv_pred, qp);
            if (sad_test < min_sad)
            {
                min_sad = sad_test;
                vbest = v;
                pbest = ptest;
            }
        }

        *mv = vbest;
        *ppbest = pbest;
    }
    return min_sad;
}

/**
*   Set range for MV search
*/
static void me_mv_set_range(point_t *pnt, rectangle_t *range, const rectangle_t *mv_limit, int mby)
{
    // clip start point
    rectangle_t r = *mv_limit;
    r.tl.s.y = (int16_t)(MAX(r.tl.s.y, mby - 63*4));
    r.br.s.y = (int16_t)(MIN(r.br.s.y, mby + 63*4));
    mv_clip(pnt, &r);
    range->tl = mv_add(*pnt, point(-MV_RANGE*4, -MV_RANGE*4));
    range->br = mv_add(*pnt, point(+MV_RANGE*4, +MV_RANGE*4));
    // clip search range
    mv_clip(&range->tl, &r);
    mv_clip(&range->br, &r);
}

/**
*   Remove duplicates from MV candidates list
*/
static int me_mv_refine_cand(point_t *p, int n)
{
    int i, j, k;
    p[0] = mv_round_qpel(p[0]);
    for (j = 1, k = 1; j < n; j++)
    {
        point_t mv = mv_round_qpel(p[j]);
        for (i = 0; i < k; i++)
        {
            // TODO
            //if (!mv_differs3(mv, p[i], 3*4))
            //if (!mv_differs3(mv, p[i], 1*4))
            //if (!mv_differs3(mv, p[i], 3))
            if (mv_equal(mv, p[i]))
                break;
        }
        if (i == k)
            p[k++] = mv;
    }
    return k;
}

/**
*   Choose candidates for inter MB partitioning (16x8,8x16 or 8x8),
*   using SAD's for 8x8 sub-blocks
*/
static void mb_inter_partition(/*const */int sad[4], int mode[4])
{
/*
    slope
        |[ 1  1]| _ |[ 1 -1]|
        |[-1 -1]|   |[ 1 -1]|
        indicates v/h gradient: big negative = vertical prediction; big positive = horizontal

    skew
        |[ 1  0]| _ |[ 0 -1]|
        |[ 0 -1]|   |[ 1  0]|
        indicates diagonal gradient: big negative = diagonal down right
*/
    int p00 = sad[0];
    int p01 = sad[1];
    int p10 = sad[2];
    int p11 = sad[3];
    int sum = p00 + p01 + p10 + p11;
    int slope = ABS((p00 - p10) + (p01 - p11)) - ABS((p00 - p01) + (p10 - p11));
    int skew = ABS(p11 - p00) - ABS(p10 - p01);

    if (slope >  (sum >> 4))
    {
        mode[1] = 1;    // try 8x16 partition
    }
    if (slope < -(sum >> 4))
    {
        mode[2] = 1;    // try 16x8 partition
    }
    if (ABS(skew) > (sum >> 4) && ABS(slope) <= (sum >> 4))
    {
        mode[3] = 1;    // try 8x8 partition
    }
}

/**
*   Online MV clustering to "long" and "short" clusters
*   Estimate mean "long" and "short" vectors
*/
static void mv_clusters_update(h264e_enc_t *enc, point_t mv)
{
    int mv_norm = SQRP(mv);
    int n0 = SQRP(enc->mv_clusters[0]);
    int n1 = SQRP(enc->mv_clusters[1]);
    if (mv_norm < n1)
    {
        // "short" is shorter than "long"
        SMOOTH(enc->mv_clusters[0], mv);
    }
    if (mv_norm >= n0)
    {
        // "long" is longer than "short"
        SMOOTH(enc->mv_clusters[1], mv);
    }
}

/**
*   Choose inter mode: skip/coded, ME partition, find MV
*/
static void inter_choose_mode(h264e_enc_t *enc)
{
    int prefered_modes[4] = { 1, 0, 0, 0 };
    point_t mv_skip, mv_skip_a, mv_cand[MAX_MV_CAND];
    point_t mv_pred_16x16 = me_mv_medianpredictor_get_skip(enc);
    point_t mv_best = point(MV_NA, 0); // avoid warning

    int sad, sad_skip = 0x7FFFFFFF, sad_best = 0x7FFFFFFF;
    int off, i, j = 0, ncand = 0;
    int cand_sad4[MAX_MV_CAND][4];
    const pix_t *ref_yuv = enc->ref.yuv[0];
    int ref_stride = enc->ref.stride[0];
    int mv_cand_cost_best = 0;
    mv_skip = enc->mb.mv_skip_pred;
    mv_skip_a = mb_abs_mv(enc, mv_skip);

    for (i = 0; i < 4; i++)
    {
        enc->df.df_mv[4 + 5*i].u32 = enc->mv_pred[i].u32;
        enc->df.df_mv[i].u32       = enc->mv_pred[8 + 4*enc->mb.x + i].u32;
    }

    // Try skip mode
    if (mv_in_rect(mv_skip_a, &enc->frame.mv_qpel_limit))
    {
        int *sad4 = cand_sad4[0];
        interpolate_luma(ref_yuv, ref_stride, mv_skip_a, point(16, 16), enc->ptest);
        sad_skip = h264e_sad_mb_unlaign_8x8(enc->scratch->mb_pix_inp, 16, enc->ptest, sad4);

        if (MAX(MAX(sad4[0], sad4[1]), MAX(sad4[2], sad4[3])) < g_skip_thr_inter[enc->rc.qp])
        {
            int uv, sad_uv;

            SWAP(pix_t*, enc->pbest, enc->ptest);
            enc->mb.type = -1;
            enc->mb.mv[0] = mv_skip;
            enc->mb.cost = 0;
            interpolate_chroma(enc, mv_skip_a);

            // Check that chroma SAD is not too big for the skip
            for (uv = 1; uv <= 2; uv++)
            {
                pix_t *pred = enc->ptest + (uv - 1)*8;
                pix_t *pix_mb_uv = mb_input_chroma(enc, uv);
                int inp_stride = enc->inp.stride[uv];

                if (enc->frame.cropping_flag && ((enc->mb.x + 1)*16  > enc->param.width || (enc->mb.y + 1)*16  > enc->param.height))
                {
                    // Speculative read beyond frame borders: make local copy of the macroblock.
                    // TODO: same code used in mb_write() and mb_encode()
                    pix_copy_cropped_mb(enc->scratch->mb_pix_store, 8, pix_mb_uv, enc->inp.stride[uv],
                        MIN(8, enc->param.width/2  - enc->mb.x*8),
                        MIN(8, enc->param.height/2 - enc->mb.y*8));
                    pix_mb_uv = enc->scratch->mb_pix_store;
                    inp_stride = 8;
                }

                sad_uv = h264e_sad_mb_unlaign_wh(pix_mb_uv, inp_stride, pred, point(8, 8));
                if (sad_uv >= g_skip_thr_inter[enc->rc.qp])
                {
                    break;
                }
            }
            if (uv == 3)
            {
                return;
            }
        }

        if (enc->run_param.encode_speed < 1) // enable 8x16, 16x8 and 8x8 partitions
        {
            mb_inter_partition(sad4, prefered_modes);
        }

        //sad_skip += me_mv_cost(mv_skip, mv_pred_16x16, enc->rc.qp);

        // Too big skip SAD. Use skip predictor as a diamond start point candidate
        mv_best = mv_cand[ncand++] = mv_round_qpel(mv_skip);
        if (!((mv_skip.s.x | mv_skip.s.y) & 3))
        {
            sad_best = sad_skip;//+ me_mv_cost(mv_best, mv_pred_16x16, enc->rc.qp)
            mv_cand_cost_best = me_mv_cand_cost(mv_skip, mv_pred_16x16, enc->rc.qp);
            //mv_cand_cost_best = me_mv_cand_cost(mv_skip, point(0,0), enc->rc.qp);
            j = 1;
        }
    }

    mv_cand[ncand++] = mv_pred_16x16;
    ncand += me_mv_medianpredictor_get_cand(enc, mv_cand + ncand);

    if (enc->mb.x <= 0)
    {
        mv_cand[ncand++] = point(8*4, 0);
    }
    if (enc->mb.y <= 0)
    {
        mv_cand[ncand++] = point(0, 8*4);
    }

    mv_cand[ncand++] = enc->mv_clusters[0];
    mv_cand[ncand++] = enc->mv_clusters[1];

    assert(ncand <= MAX_MV_CAND);
    ncand = me_mv_refine_cand(mv_cand, ncand);

    for (/*j = 0*/; j < ncand; j++)
    {
        point_t mv = mb_abs_mv(enc, mv_cand[j]);
        if (mv_in_rect(mv, &enc->frame.mv_limit))
        {
            int mv_cand_cost = me_mv_cand_cost(mv_cand[j], mv_pred_16x16, enc->rc.qp);

            int *sad4 = cand_sad4[j];
            off = ((mv.s.y + 0) >> 2)*ref_stride + ((mv.s.x + 0) >> 2);
            sad = h264e_sad_mb_unlaign_8x8(ref_yuv + off, ref_stride, enc->scratch->mb_pix_inp, sad4);

            if (enc->run_param.encode_speed < 1) // enable 8x16, 16x8 and 8x8 partitions
            {
                mb_inter_partition(sad4, prefered_modes);
            }

            if (sad + mv_cand_cost < sad_best + mv_cand_cost_best)
            //if (sad < sad_best)
            {
                mv_cand_cost_best = mv_cand_cost;
                sad_best = sad;
                mv_best = mv_cand[j];
            }
        }
    }

    sad_best += me_mv_cost(mv_best, mv_pred_16x16, enc->rc.qp);

    {
        int mb_type;
        point_t wh, part, mvpred_ctx[12], part_mv[4][16], part_mvd[4][16];
        pix_t *store = enc->scratch->mb_pix_store;
        pix_t *pred_best = store, *pred_test = store + 256;

#define MAX8X8_MODES 4
        me_mv_medianpredictor_save_ctx(enc, mvpred_ctx);
        enc->mb.cost = 0xffffff;
        for (mb_type = 0; mb_type < MAX8X8_MODES; mb_type++)
        {
            static const int nbits[4] = { 1, 4, 4, 12 };
            int imv = 0;
            int part_sad = MUL_LAMBDA(nbits[mb_type], g_lambda_q4[enc->rc.qp]);

            if (!prefered_modes[mb_type]) continue;

            wh.s.x = (mb_type & 2) ? 8 : 16;
            wh.s.y = (mb_type & 1) ? 8 : 16;
            part = point(0, 0);
            for (;;)
            {
                rectangle_t range;
                pix_t *diamond_out;
                point_t mv, mv_pred, mvabs = mb_abs_mv(enc, mv_best);
                me_mv_set_range(&mvabs, &range, &enc->frame.mv_limit, enc->mb.y*16*4 + part.s.y*4);

                mv_pred = me_mv_medianpredictor_get(enc, part, wh);

                if (mb_type)
                {
                    mvabs = mv_round_qpel(mb_abs_mv(enc, mv_pred));
                    me_mv_set_range(&mvabs, &range, &enc->frame.mv_limit, enc->mb.y*16*4 + part.s.y*4);
                    off = ((mvabs.s.y >> 2) + part.s.y)*ref_stride + ((mvabs.s.x >> 2) + part.s.x);
                    sad_best = h264e_sad_mb_unlaign_wh(ref_yuv + off, ref_stride, enc->scratch->mb_pix_inp + part.s.y*16 + part.s.x, wh)
                        + me_mv_cost(mvabs,
                        //mv_pred,
                        mb_abs_mv(enc, mv_pred),
                        enc->rc.qp);
                }

                part_sad += me_search_diamond(enc, ref_yuv + part.s.y*ref_stride + part.s.x,
                    enc->scratch->mb_pix_inp + part.s.y*16 + part.s.x, ref_stride, &mvabs, &range, enc->rc.qp,
                    mb_abs_mv(enc, mv_pred), sad_best, wh,
                    store, &diamond_out, mb_type ? (mb_type == 2 ? 8 : 128) : 256);

                if (!mb_type)
                {
                    pred_test = diamond_out;
                    if (pred_test < store + 2*256)
                    {
                        pred_best = (pred_test == store ? store + 256 : store);
                        store += 2*256;
                    } else
                    {
                        pred_best = (pred_test == (store + 512) ? store + 512 + 256 : store + 512);
                    }
                } else
                {
                    h264e_copy_8x8(pred_test + part.s.y*16 + part.s.x, 16, diamond_out);
                    if (mb_type < 3)
                    {
                        int part_off = (wh.s.x >> 4)*8 + (wh.s.y >> 4)*8*16;
                        h264e_copy_8x8(pred_test + part_off + part.s.y*16 + part.s.x, 16, diamond_out + part_off);
                    }
                }

                mv = mv_sub(mvabs, point(enc->mb.x*16*4, enc->mb.y*16*4));

                part_mvd[mb_type][imv] = mv_sub(mv, mv_pred);
                part_mv[mb_type][imv++] = mv;

                me_mv_medianpredictor_put(enc, part.s.x >> 2, part.s.y >> 2, wh.s.x >> 2, wh.s.y >> 2, mv);

                part.s.x = (part.s.x + wh.s.x) & 15;
                if (!part.s.x)
                {
                    part.s.y = (part.s.y + wh.s.y) & 15;
                    if (!part.s.y) break;
                }
            }

            me_mv_medianpredictor_restore_ctx(enc, mvpred_ctx);

            if (part_sad < enc->mb.cost)
            {
                SWAP(pix_t*, pred_best, pred_test);
                enc->mb.cost = part_sad;
                enc->mb.type = mb_type;
            }
        }
        enc->pbest = pred_best;
        enc->ptest = pred_test;
        memcpy(enc->mb.mv,  part_mv [enc->mb.type], 16*sizeof(point_t));
        memcpy(enc->mb.mvd, part_mvd[enc->mb.type], 16*sizeof(point_t));

        if (enc->mb.cost > sad_skip)
        {
            enc->mb.type = 0;
            enc->mb.cost = sad_skip + me_mv_cand_cost(mv_skip, mv_pred_16x16, enc->rc.qp);
            enc->mb.mv [0] = mv_skip;
            enc->mb.mvd[0] = mv_sub(mv_skip, mv_pred_16x16);

            assert(mv_in_rect(mv_skip_a, &enc->frame.mv_qpel_limit)) ;
            interpolate_luma(ref_yuv, ref_stride, mv_skip_a, point(16, 16), enc->pbest);
            interpolate_chroma(enc, mv_skip_a);
        }
    }
}

/************************************************************************/
/*      Deblock filter                                                  */
/************************************************************************/
#define MB_FLAG_SVC_INTRA 1
#define MB_FLAG_SLICE_START_DEBLOCK_2 2

/**
*   Set deblock filter strength
*/
static void df_strength(deblock_filter_t *df, int mb_type, int mbx, uint8_t *strength, int IntraBLFlag)
{
    uint8_t *sv = strength;
    uint8_t *sh = strength + 16;
    int flag = df->nzflag;
    df->df_nzflag[mbx] = (uint8_t)(flag >> 20);
    /*
        nzflag represents macroblock and it's neighbors with 24 bit flags:
        0 1 2 3
      4 5 6 7 8
      A B C D E
      F G H I J
      K L K N O
    */
    (void)IntraBLFlag;
#if H264E_SVC_API
    if (IntraBLFlag & MB_FLAG_SVC_INTRA)
    {
        int ccloop = 4;
        do
        {
            int cloop = 4;
            do
            {
                int v = 0;
                if (flag & 3 << 4)
                {
                    v = 1;
                }

                *sv = (uint8_t)v; sv += 4;

                v = 0;
                if (flag & 33)
                {
                    v = 1;
                }

                *sh++ = (uint8_t)v;

                flag >>= 1;

            } while(--cloop);
            flag >>= 1;
            sv -= 15;

        } while(--ccloop);
    } else
#endif
    {
        if (mb_type < 5)
        {
            int ccloop = 4;
            point_t *mv = df->df_mv;
            do
            {
                int cloop = 4;
                do
                {
                    int v = 0;
                    if (flag & 3 << 4)
                    {
                        v = 2;
                    } else if (mv_differs3(mv[4], mv[5]))
                    {
                        v = 1;
                    }
                    *sv = (uint8_t)v; sv += 4;

                    v = 0;
                    if (flag & 33)
                    {
                        v = 2;
                    } else if (mv_differs3(mv[0], mv[5]))
                    {
                        v = 1;
                    }
                    *sh++ = (uint8_t)v;

                    flag >>= 1;
                    mv++;
                } while(--cloop);
                flag >>= 1;
                sv -= 15;
                mv++;
            } while(--ccloop);
        } else
        {
            // Deblock mode #3 (intra)
            ((uint32_t*)(sv))[1] = ((uint32_t*)(sv))[2] = ((uint32_t*)(sv))[3] =             // for inner columns
            ((uint32_t*)(sh))[1] = ((uint32_t*)(sh))[2] = ((uint32_t*)(sh))[3] = 0x03030303; // for inner rows
        }
        if ((mb_type >= 5 || df->mb_type[mbx - 1] >= 5)) // speculative read
        {
            ((uint32_t*)(strength))[0] = 0x04040404;    // Deblock mode #4 (strong intra) for left column
        }
        if ((mb_type >= 5 || df->mb_type[mbx    ] >= 5))
        {
            ((uint32_t*)(strength))[4] = 0x04040404;    // Deblock mode #4 (strong intra) for top row
        }
    }
    df->mb_type[mbx] = (int8_t)mb_type;
}

/**
*   Run deblock for current macroblock
*/
static void mb_deblock(deblock_filter_t *df, int mb_type, int qp_this, int mbx, int mby, H264E_io_yuv_t *mbyuv, int IntraBLFlag)
{
    int i, cr, qp, qp_left, qp_top;
    deblock_params_t par;
    uint8_t *alpha = par.alpha; //[2*2];
    uint8_t *beta  = par.beta;  //[2*2];
    uint32_t *strength32  = par.strength32; //[4*2]; // == uint8_t strength[16*2];
    uint8_t *strength = (uint8_t *)strength32;
    uint8_t *tc0 = par.tc0; //[16*2];

    df_strength(df, mb_type, mbx, strength, IntraBLFlag);
    if (!mbx || (IntraBLFlag & MB_FLAG_SLICE_START_DEBLOCK_2))
    {
        strength32[0] = 0;
    }

    if (!mby)
    {
        strength32[4] = 0;
    }

    qp_top = df->df_qp[mbx];
    qp_left = df->df_qp[mbx - 1];
    df->df_qp[mbx] = (uint8_t)qp_this;

    cr = 0;
    for (;;)
    {
        const uint8_t *lut;
        if (*((uint32_t*)strength))
        {
            qp = (qp_left + qp_this + 1) >> 1;
            lut = g_a_tc0_b[-10 + qp + ALPHA_OFS];
            alpha[0] = lut[0];
            beta[0]  = lut[4 + (BETA_OFS - ALPHA_OFS)*5];
            for (i = 0; i < 4; i++) tc0[i] = lut[strength[i]];
        }
        if (*((uint32_t*)(strength + 16)))
        {
            qp = (qp_top + qp_this + 1) >> 1;
            lut = g_a_tc0_b[-10 + qp + ALPHA_OFS];

            alpha[2]  = lut[0];
            beta[2] = lut[4 + (BETA_OFS - ALPHA_OFS)*5];
            for (i = 0; i < 4; i++) tc0[16 + i] = lut[strength[16 + i]];
        }

        lut = g_a_tc0_b[-10 + qp_this + ALPHA_OFS];
        alpha[3] = alpha[1] = lut[0];
        beta[3] = beta[1] = lut[4 + (BETA_OFS - ALPHA_OFS)*5];
        for (i = 4; i < 16; i++)
        {
            tc0[i] = lut[strength[i]];
            tc0[16 + i] = lut[strength[16 + i]];
        }
        if (cr)
        {
            int *t = (int *)tc0;
            t[1] = t[2];         // TODO: need only for OMX
            t[5] = t[6];
            i = 2;
            do
            {
                h264e_deblock_chroma(mbyuv->yuv[i], mbyuv->stride[i], &par);
            } while (--i);
            break;
        }
        h264e_deblock_luma(mbyuv->yuv[0], mbyuv->stride[0], &par);

        qp_this = qpy2qpc[qp_this + DQP_CHROMA];
        qp_left = qpy2qpc[qp_left + DQP_CHROMA];
        qp_top = qpy2qpc[qp_top + DQP_CHROMA];
        cr++;
    }
}

/************************************************************************/
/*      Macroblock encoding                                             */
/************************************************************************/
/**
*   Macroblock encoding
*/
static void mb_encode(h264e_enc_t *enc, int enc_type)
{

    pix_t *top = enc->top_line + 48 + enc->mb.x*32;
    pix_t *left = enc->top_line;
    int avail = enc->mb.avail = mb_avail_flag(enc);
    int base_mode = 0;

    if (enc->frame.cropping_flag && ((enc->mb.x + 1)*16 > enc->param.width || (enc->mb.y + 1)*16 > enc->param.height))
    {
        pix_copy_cropped_mb(enc->scratch->mb_pix_inp, 16, mb_input_luma(enc), enc->inp.stride[0],
             MIN(16, enc->param.width  - enc->mb.x*16),
             MIN(16, enc->param.height - enc->mb.y*16));
    } else
    {
        // cache input luma macroblock
        h264e_copy_16x16(enc->scratch->mb_pix_inp, 16, mb_input_luma(enc), enc->inp.stride[0]);
    }

    if (!(avail & AVAIL_L)) left = NULL;
    if (!(avail & AVAIL_T)) top  = NULL;

    // Store 16x16 Macroblock best prediction (Global)
    // Size of mb_pix_store is 16x16x4
    enc->pbest = enc->scratch->mb_pix_store;
    // Store 16x16 Macroblock tmp prediction
    enc->ptest = enc->pbest + 256;
    enc->mb.type = 0;
    enc->mb.cost = 0x7FFFFFFF;

    if (enc->slice.type == SLICE_TYPE_P)
    {
        inter_choose_mode(enc);
    }
#if H264E_SVC_API
    else if (enc_type > 0 && enc->param.inter_layer_pred_flag)
    {
        base_mode = 1;
        enc->mb.type = 6;
        h264e_copy_16x16(enc->pbest, 16, (enc->ref.yuv[0] + (enc->mb.x + enc->mb.y*enc->ref.stride[0])*16), enc->ref.stride[0]);
        h264e_copy_8x8_s(enc->ptest, 16, (enc->ref.yuv[1] + (enc->mb.x + enc->mb.y*enc->ref.stride[1])*8), enc->ref.stride[1]);
        h264e_copy_8x8_s(enc->ptest + 8, 16, (enc->ref.yuv[2] + (enc->mb.x + enc->mb.y*enc->ref.stride[2])*8), enc->ref.stride[2]);

        goto _WRITE_MB;
    }
#endif

    if (enc->mb.type >= 0)
    {
        /**
         * luma intra prediction
         */
        intra_choose_16x16(enc, left, top, avail);
        if (enc->run_param.encode_speed < 2 || enc->slice.type != SLICE_TYPE_P) // enable intra4x4 on P slices
        {
            intra_choose_4x4(enc);
        }
    }

    // if enc->mb.type == 6, then we have intra 16x16 prediction
    // if enc->mb.type == 5, then we have intra 4x4 prediction
    if (enc->mb.type < 5)
    {
        mv_clusters_update(enc, enc->mb.mv[0]);
    }

    if (enc->mb.type >= 5)
    {
        pix_t *pred = enc->ptest;
        h264e_intra_predict_chroma(pred, left + 16, top + 16, enc->mb.i16.pred_mode_luma);
    } else
    {
        interpolate_chroma(enc, mb_abs_mv(enc, enc->mb.mv[0]));
    }

#if H264E_SVC_API
_WRITE_MB:
#endif
    mb_write(enc, enc_type, base_mode);

    if (!enc->speed.disable_deblock)
    {
        int mbx = enc->mb.x;
        int mby = enc->mb.y;
#if H264E_MAX_THREADS
        if (enc->param.max_threads > 1)
        {   // Avoid deblock across slice border
            if (enc->mb.num < enc->slice.start_mb_num + enc->frame.nmbx)
                mby = 0;
            if (enc->mb.num == enc->slice.start_mb_num)
            {
                base_mode |= MB_FLAG_SLICE_START_DEBLOCK_2;
            }
        }
#endif
        // 去滤波技术：减少编码后视频中图像边界的锐利度，以减少由于压缩引起的视觉失真
        mb_deblock(&enc->df, enc->mb.type, enc->rc.prev_qp, mbx, mby, &enc->dec, base_mode);
    }
}


/************************************************************************/
/*      Rate-control                                                    */
/************************************************************************/

/**
*   @return zero threshold for given rounding offset
*/
static uint16_t rc_rnd2thr(int round, int q)
{
    int b, thr = 0;
    for (b = 0x8000; b; b >>= 1)
    {
        int t = (thr | b)*q;
        if (t <= 0x10000 - round)  // TODO: error: < !!!!!!!
        {
            thr |= b;
        }
    }
    return (uint16_t)thr;
}

/**
*   Set quantizer constants (deadzone and rounding) for given QP
*/
static void rc_set_qp(h264e_enc_t *enc, int qp)
{
    qp = MIN(qp, enc->run_param.qp_max);
    qp = MAX(qp, enc->run_param.qp_min);
    qp = MIN(qp, 51);   // avoid VC2010 static analyzer warning

    if (enc->rc.qp != qp)
    {
        static const int16_t g_quant_coeff[6*6] =
        {
            //    0         2         1
            13107, 10, 8066, 13, 5243, 16,
            11916, 11, 7490, 14, 4660, 18,
            10082, 13, 6554, 16, 4194, 20,
             9362, 14, 5825, 18, 3647, 23,
             8192, 16, 5243, 20, 3355, 25,
             7282, 18, 4559, 23, 2893, 29
            // 0 2 0 2
            // 2 1 2 1
            // 0 2 0 2
            // 2 1 2 1
        };

        int cloop = 2;
        enc->rc.qp = qp;

        do
        {
            uint16_t *qdat0 = enc->rc.qdat[2 - cloop];
            uint16_t *qdat  = enc->rc.qdat[2 - cloop];
            // qbits
            int qp_div6 = qp*86 >> 9;
            int qp_mod6 = qp - qp_div6*6;
            const int16_t *quant_coeff = g_quant_coeff + qp_mod6*6; // TODO: need calculate qp%6*6
            int i = 3;

            // Quant/dequant multiplier
            do
            {
                // Quant MF >> qbits
                *qdat++ = *quant_coeff++ << 1 >> qp_div6;
                // Dequant MF << qbits?
                *qdat++ = *quant_coeff++ << qp_div6;
            } while(--i);

            // quantizer deadzone for P & chroma
            *qdat++ = enc->slice.type == SLICE_TYPE_P ? g_rnd_inter[qp] : g_deadzonei[qp];
            // quantizer deadzone for I
            *qdat++ = g_deadzonei[qp];

            *qdat++ = g_thr_inter[qp]  - 0x7fff;
            *qdat++ = g_thr_inter2[qp] - 0x7fff;

            qdat[0] = qdat[2] = rc_rnd2thr(g_thr_inter[qp] - 0x7fff, qdat0[0]);
            qdat[1] = qdat[3] =
            qdat[4] = qdat[6] = rc_rnd2thr(g_thr_inter[qp] - 0x7fff, qdat0[2]);
            qdat[5] = qdat[7] = rc_rnd2thr(g_thr_inter[qp] - 0x7fff, qdat0[4]);
            qdat += 8;
            qdat[0] = qdat[2] = rc_rnd2thr(g_thr_inter2[qp] - 0x7fff, qdat0[0]);
            qdat[1] = qdat[3] =
            qdat[4] = qdat[6] = rc_rnd2thr(g_thr_inter2[qp] - 0x7fff, qdat0[2]);
            qdat[5] = qdat[7] = rc_rnd2thr(g_thr_inter2[qp] - 0x7fff, qdat0[4]);
            qdat += 8;
            qdat[0] = qdat[2] = qdat0[0];
            qdat[1] = qdat[3] =
            qdat[4] = qdat[6] = qdat0[2];
            qdat[5] = qdat[7] = qdat0[4];
            qdat += 8;
            qdat[0] = qdat[2] = qdat0[1];
            qdat[1] = qdat[3] =
            qdat[4] = qdat[6] = qdat0[3];
            qdat[5] = qdat[7] = qdat0[5];

            // Copy to luma QP to chroma QP according to LUT.
            qp = qpy2qpc[qp + DQP_CHROMA];
        } while (--cloop);
    }
}

/**
*   Estimate frame bit budget and QP
*
*   How bit budget allocated?
*   ~~~~~~~~~~~~~~~~~~~~~~~~~
*   1. Estimate target size of I and P macroblock, assuming same quality
*   2. Estimate I peak size
*   3. Estimate desired stationary VBV level
*
*/
static int rc_frame_start(h264e_enc_t *enc, int is_intra, int is_refers_to_long_term)
{
    unsigned np = MIN(enc->param.gop - 1u, 63u);
    int nmb = enc->frame.nmb;

    int qp = -1, add_bits, bit_budget = enc->run_param.desired_frame_bytes*8;
    int nominal_p, gop_bits, stationary_vbv_level;
    uint32_t peak_factor_q16;

    // Estimate QP
    do
    {
        qp++;
        gop_bits = bits_per_mb[0][qp]*np + bits_per_mb[1][qp];
    } while (gop_bits*nmb > (int)(np + 1)*enc->run_param.desired_frame_bytes*8 && qp < 40);

    /*
    *   desired*gop = i + p*(gop-1);   i/p = alpha;
    *   p = desired * gop / (gop-1+alpha) and i = p*alpha or i = (desired-p)*gop + p;
    */
    peak_factor_q16 = div_q16(bits_per_mb[1][qp] << 16, bits_per_mb[0][qp] << 16);
    if (np)
    {
        uint32_t ratio_q16 = div_q16((np + 1) << 16, (np << 16) + peak_factor_q16);
        nominal_p = mul32x32shr16(enc->run_param.desired_frame_bytes*8, ratio_q16);
    } else
    {
        nominal_p = 0;
    }

    stationary_vbv_level = MIN(enc->param.vbv_size_bytes*8 >> 4, enc->run_param.desired_frame_bytes*8);

    if (is_intra)
    {
        int nominal_i = mul32x32shr16(nominal_p, peak_factor_q16);
        add_bits = nominal_i - bit_budget;
    }
#if H264E_RATE_CONTROL_GOLDEN_FRAMES
    else if (is_refers_to_long_term)
    {
        int d_qp = enc->rc.max_dqp - enc->rc.dqp_smooth;
        unsigned peak_factor_golden_q16;
        int nominal_golden;
        d_qp = MAX(d_qp, 2);
        d_qp = MIN(d_qp, 12);
        d_qp = d_qp * 4 * 85 >> 8;//* 16 / 12;

        peak_factor_golden_q16 = (peak_factor_q16 - (1 << 16)) * d_qp >> 4;
        nominal_golden = nominal_p + mul32x32shr16(nominal_p, peak_factor_golden_q16);
        add_bits = nominal_golden - bit_budget;
    }
#endif
    else
    {
        add_bits = nominal_p - bit_budget;

        // drift to stationary level
        if (enc->param.vbv_size_bytes)
        {
            add_bits += (enc->rc.vbv_target_level - enc->rc.vbv_bits) >> 4;
        }
    }
    // Default enc->rc.vbv_target_level is 12500
    if (enc->param.vbv_size_bytes)
    {
        add_bits = MIN(add_bits, (enc->param.vbv_size_bytes*8*7 >> 3) - enc->rc.vbv_bits);
    }

    bit_budget += add_bits;
    bit_budget = MIN(bit_budget, enc->run_param.desired_frame_bytes*8*16);
    bit_budget = MAX(bit_budget, enc->run_param.desired_frame_bytes*8 >> 2);

#if H264E_RATE_CONTROL_GOLDEN_FRAMES
    if (is_intra || is_refers_to_long_term)
#else
    if (is_intra)
#endif
    {
        // Increase VBV target level due to to I-frame load: this avoids QP adaptation after I-frame
        // Restore VBV target level after I-frame
        // bit_budget - enc->run_param.desired_frame_bytes*8 is number of bits to be added to VBV buffer
        // enc->rc.vbv_bits is number of bits in VBV buffer
        enc->rc.vbv_target_level = enc->rc.vbv_bits + bit_budget - enc->run_param.desired_frame_bytes*8;
    }

    // Slow drift of VBV target to stationary level...
    enc->rc.vbv_target_level -= enc->run_param.desired_frame_bytes*8 - nominal_p;

    // ...until stationary level reached
    enc->rc.vbv_target_level = MAX(enc->rc.vbv_target_level, stationary_vbv_level);

    enc->rc.bit_budget = bit_budget;

    if (enc->param.fine_rate_control_flag && enc->frame.num)
    {
        qp = enc->rc.qp_smooth >> 8;
    } else
    {

#if H264E_RATE_CONTROL_GOLDEN_FRAMES
        if (is_refers_to_long_term)
        {
            for (qp = 0; qp < 42 - 1; qp++)
            {
                //if (((bits_per_mb[0][qp] + bits_per_mb[1][qp]) >> 1)*nmb < bit_budget)
                if (((bits_per_mb[0][qp] + bits_per_mb[1][qp]) >> 1)*nmb < bit_budget)
                    break;
            }
        } else
#endif
        {
            const uint16_t *bits = bits_per_mb[!!is_intra];
            for (qp = 0; qp < 42 - 1; qp++)
            {
                if (bits[qp]*nmb < bit_budget)
                {
                    break;
                }
            }
        }
        qp += MIN_QP;

#if H264E_RATE_CONTROL_GOLDEN_FRAMES
        if (is_refers_to_long_term)
        {
            int dqp = MAX(enc->rc.max_dqp, enc->rc.dqp_smooth);
            dqp  = MIN(dqp, enc->rc.dqp_smooth + 6);
            qp += dqp;
            qp = MAX(enc->rc.prev_qp, qp);
        } else
#endif
        {
            qp += enc->rc.dqp_smooth;
        }

        // If reference frame has high qp, motion compensation is less effective, so qp should be increased
        if (enc->rc.prev_qp > qp + 1)
        {
            qp = (enc->rc.prev_qp + qp + 1)/2;
        }
    }

    enc->rc.qp = 0; // force
    rc_set_qp(enc, qp);
    qp = enc->rc.qp;

    enc->rc.qp_smooth = qp << 8;
    enc->rc.prev_qp = qp;

    return (enc->rc.vbv_bits > enc->param.vbv_size_bytes*8);
}

/**
*   Update rate-control state after frame encode
*/
static void rc_frame_end(h264e_enc_t *enc, int intra_flag, int skip_flag, int is_refers_to_long_term)
{
    // 1. Update QP offset adaptive adjustment
    if (!skip_flag /*&& !is_refers_to_long_term*/)
    {
        int qp, nmb = enc->frame.nmb;
        // a posterior qp estimation
        for (qp = 0; qp != 41 && bits_per_mb[intra_flag][qp]*nmb > (int)enc->out_pos*8 - 32; qp++) {/*no action*/}

        qp += MIN_QP;

        if (!is_refers_to_long_term)
        {
            if ((enc->rc.qp_smooth >> 8) - enc->rc.dqp_smooth < qp - 1)
            {
                enc->rc.dqp_smooth--;
            } else if ((enc->rc.qp_smooth >> 8) - enc->rc.dqp_smooth > qp + 1)
            {
                enc->rc.dqp_smooth++;
            }
        }
        if (intra_flag || is_refers_to_long_term)
        {
            enc->rc.max_dqp = enc->rc.dqp_smooth;
        } else
        {
            enc->rc.max_dqp = MAX(enc->rc.max_dqp, (enc->rc.qp_smooth >> 8) - qp);
        }
    }

    // 2. Update VBV model state
    enc->rc.vbv_bits += enc->out_pos*8 - enc->run_param.desired_frame_bytes*8;

    // 3. If VBV model used, handle overflow/underflow
    if (enc->param.vbv_size_bytes)
    {
        if (enc->rc.vbv_bits < 0)       // VBV underflow
        {
            if (enc->param.vbv_underflow_stuffing_flag)
            {
                // put stuffing ('filler data')
                nal_start(enc, 12); // filler_data_rbsp
                do
                {
                    U(8, 0xFF);
                    enc->rc.vbv_bits += 8;
                } while (enc->rc.vbv_bits < 0);
                nal_end(enc);
            } else
            {
                // ignore underflow
                enc->rc.vbv_bits = 0;
            }
        }
        if (enc->rc.vbv_bits > enc->param.vbv_size_bytes*8) // VBV overflow
        {
            if (!enc->param.vbv_overflow_empty_frame_flag)
            {
                // ignore overflow
                enc->rc.vbv_bits = enc->param.vbv_size_bytes*8;
            }
        }
    } else
    {
        enc->rc.vbv_bits = 0;
    }
}

/**
*   Update rate-control state after macroblock encode, set QP for next MB
*/
static void rc_mb_end(h264e_enc_t *enc)
{
    // used / ncoded = budget/total
    int bits_coded = h264e_bs_get_pos_bits(enc->bs) +  enc->out_pos*8 + 1;
    int mb_coded = enc->mb.num; // after increment: 1, 2....
    int err = bits_coded *enc->frame.nmb - enc->rc.bit_budget*mb_coded;
    int d_err = err - enc->rc.prev_err;
    int qp = enc->rc.qp;
    assert(enc->mb.num);
    enc->rc.prev_err = err;

    if (err > 0 && d_err > 0)
    {   // Increasing risk of overflow
        if (enc->rc.stable_count < 3)
        {
            qp++;                       // State not stable: increase QP
        }
        enc->rc.stable_count = 0;       // Set state to "not stable"
    } else if (err < 0 && d_err < 0)
    {   // Increasing risk of underlow
        if (enc->rc.stable_count < 3)
        {
            qp--;
        }
        enc->rc.stable_count = 0;
    } else
    {   // Stable state
        enc->rc.stable_count++;
    }
    enc->rc.qp_smooth += qp - (enc->rc.qp_smooth >> 8);
    qp = MIN(qp, enc->rc.prev_qp + 3);
    qp = MAX(qp, enc->rc.prev_qp - 3);
    rc_set_qp(enc, qp);
}

/************************************************************************/
/*      Top-level API                                                   */
/************************************************************************/

#define ALIGN_128BIT(p) (void *)((uintptr_t)(((char*)(p)) + 15) & ~(uintptr_t)15)
#define ALLOC(ptr, size) p = ALIGN_128BIT(p); if (enc) ptr = (void *)p; p += size;

/**
*   Internal allocator for persistent RAM
*/
static int enc_alloc(h264e_enc_t *enc, const H264E_create_param_t *par, unsigned char *p, int inp_buf_flag)
{
    unsigned char *p0 = p;
    int nmbx = (par->width  + 15) >> 4;
    int nmby = (par->height + 15) >> 4;
    int nref_frames = 1 + par->max_long_term_reference_frames + par->const_input_flag;
#if H264E_ENABLE_DENOISE
    nref_frames += !!par->temporal_denoise_flag;
#endif
    ALLOC(enc->ref.yuv[0], ((nmbx + 2) * (nmby + 2) * 384) * nref_frames);
    (void)inp_buf_flag;
#if H264E_SVC_API
    if (inp_buf_flag)
    {
        ALLOC(enc->inp.yuv[0], ((nmbx)*(nmby)*384)); /* input buffer for base laeyr */
    }
#endif
    return (int)((p - p0) + 15) & ~15u;
}

/**
*   Internal allocator for scratch RAM
*/
static int enc_alloc_scratch(h264e_enc_t *enc, const H264E_create_param_t *par, unsigned char *p)
{
    unsigned char *p0 = p;
    int nmbx = (par->width  + 15) >> 4;
    int nmby = (par->height + 15) >> 4;
    ALLOC(enc->scratch, sizeof(scratch_t));
    ALLOC(enc->out, nmbx * nmby * (384 + 2 + 10) * 3/2);

    ALLOC(enc->nnz, nmbx*8 + 8);
    ALLOC(enc->mv_pred, (nmbx*4 + 8)*sizeof(point_t));
    ALLOC(enc->i4x4mode, nmbx*4 + 4);
    ALLOC(enc->df.df_qp, nmbx);
    ALLOC(enc->df.mb_type, nmbx);
    ALLOC(enc->df.df_nzflag, nmbx);
    ALLOC(enc->top_line, nmbx*32 + 32 + 16);
    return (int)(p - p0);
}

/**
*   Setup H264E_io_yuv_t structures
*/
static pix_t *io_yuv_set_pointers(pix_t *base, H264E_io_yuv_t *frm, int w, int h)
{
    int s = w + (16 + 16);    // guards
    int i, guard = 16;
    for (i = 0; i < 3; i++)
    {
        frm->stride[i] = s;
        frm->yuv[i] = base + (s + 1)*guard;
        base += s*(h + 2*guard);
        if (!i) guard >>= 1, s >>= 1, h >>= 1;
    }
    return base;
}

/**
*   Verify encoder creation parameters. Return error code, or 0 if prameters
*/
static int enc_check_create_params(const H264E_create_param_t *par)
{
    if (!par)
    {
        return H264E_STATUS_BAD_ARGUMENT;   // NULL argument
    }
    if ((int)(par->vbv_size_bytes | par->gop) < 0)
    {
        return H264E_STATUS_BAD_PARAMETER;  // negative GOP or VBV size
    }
    if (par->width <= 0 || par->height <= 0)
    {
        return H264E_STATUS_BAD_PARAMETER;  // non-positive frame size
    }
    if ((unsigned)(par->const_input_flag | par->fine_rate_control_flag |
        par->vbv_overflow_empty_frame_flag | par->vbv_underflow_stuffing_flag) > 1)
    {
        return H264E_STATUS_BAD_PARAMETER;  // Any flag is not 0 or 1
    }
    if ((unsigned)par->max_long_term_reference_frames > MAX_LONG_TERM_FRAMES)
    {
        return H264E_STATUS_BAD_PARAMETER;  // Too many long-term reference frames requested
    }
    if ((par->width | par->height) & 1)
    {
        return H264E_STATUS_SIZE_NOT_MULTIPLE_2; // frame size must be multiple of 2
    }
    if (((par->width | par->height) & 15) && !par->const_input_flag)
    {
        // if input buffer reused as scratch (par->const_input_flag == 0)
        // frame size must be multiple of 16
        return H264E_STATUS_SIZE_NOT_MULTIPLE_16;
    }
    return H264E_STATUS_SUCCESS;
};

static int H264E_sizeof_one(const H264E_create_param_t *par, int *sizeof_persist, int *sizeof_scratch, int inp_buf_flag)
{
    int error = enc_check_create_params(par);
    if (!sizeof_persist || !sizeof_scratch)
    {
        error = H264E_STATUS_BAD_ARGUMENT;
    }
    if (error)
    {
        return error;
    }

    *sizeof_persist = enc_alloc(NULL, par, (void*)(uintptr_t)1, inp_buf_flag) + sizeof(h264e_enc_t);
#if H264E_MAX_THREADS > 1
    *sizeof_scratch = enc_alloc_scratch(NULL, par, (void*)(uintptr_t)1) * (par->max_threads + 1);
#else
    *sizeof_scratch = enc_alloc_scratch(NULL, par, (void*)(uintptr_t)1);
#endif
    return error;
}

static int H264E_init_one(h264e_enc_t *enc, const H264E_create_param_t *opt, int inp_buf_flag)
{
    pix_t *base;
#if H264E_CONFIGS_COUNT > 1
    init_vft(opt->enableNEON);
#endif
    memset(enc, 0, sizeof(*enc));

    enc->frame.nmbx = (opt->width  + 15) >> 4;
    enc->frame.nmby = (opt->height + 15) >> 4;
    enc->frame.nmb = enc->frame.nmbx*enc->frame.nmby;
    enc->frame.w = enc->frame.nmbx*16;
    enc->frame.h = enc->frame.nmby*16;
    enc->frame.mv_limit.tl = point(-MV_GUARD*4, -MV_GUARD*4);
    enc->frame.mv_qpel_limit.tl = mv_add(enc->frame.mv_limit.tl, point(4*4, 4*4));
    enc->frame.mv_limit.br = point((enc->frame.nmbx*16 - (16 - MV_GUARD))*4, (enc->frame.nmby*16 - (16 - MV_GUARD))*4);
    enc->frame.mv_qpel_limit.br = mv_add(enc->frame.mv_limit.br, point(-4*4, -4*4));
    enc->frame.cropping_flag = !!((opt->width | opt->height) & 15);
    enc->param = *opt;

    enc_alloc(enc, opt, (void*)(enc + 1), inp_buf_flag); // Init enc->ref.yuv[0], The reference frame's base address.

#if H264E_SVC_API
    if (inp_buf_flag)
    {
        enc->inp.yuv[1] = enc->inp.yuv[0] + enc->frame.w*enc->frame.h;
        enc->inp.yuv[2] = enc->inp.yuv[1] + enc->frame.w*enc->frame.h/4;
        enc->inp.stride[0] = enc->frame.w;
        enc->inp.stride[1] = enc->frame.w/2;
        enc->inp.stride[2] = enc->frame.w/2;
        enc->dec = enc->inp;
    }
#endif

    base = io_yuv_set_pointers(enc->ref.yuv[0], &enc->ref, enc->frame.nmbx*16, enc->frame.nmby*16);
#if H264E_ENABLE_DENOISE
    if (enc->param.temporal_denoise_flag)
    {
        pix_t *p = base;
        base = io_yuv_set_pointers(base, &enc->denoise, enc->frame.nmbx*16, enc->frame.nmby*16);
        while (p < base) *p++ = 0;
    }
#endif
    if (enc->param.const_input_flag)
    {
        base = io_yuv_set_pointers(base, &enc->dec, enc->frame.nmbx*16, enc->frame.nmby*16);
    }
    if (enc->param.max_long_term_reference_frames)
    {
        H264E_io_yuv_t t;
        int i;
        for (i = 0; i < enc->param.max_long_term_reference_frames; i++)
        {
            base = io_yuv_set_pointers(base, &t, enc->frame.nmbx*16, enc->frame.nmby*16);
            enc->lt_yuv[i][0] = t.yuv[0];
            enc->lt_yuv[i][1] = t.yuv[1];
            enc->lt_yuv[i][2] = t.yuv[2];
        }
    }
    return H264E_STATUS_SUCCESS;
}

/**
*   Encoder initialization
*   See header file for details.
*/
int H264E_init(h264e_enc_t *enc, const H264E_create_param_t *opt)
{
    h264e_enc_t *enc_curr = enc;
    int i, ret;
    (void)i;

    ret = H264E_init_one(enc_curr, opt, 0);

#if H264E_SVC_API
    for (i = opt->num_layers; i > 1; i--)
    {
        H264E_create_param_t opt_next = enc_curr->param;
        int sizeof_persist = 0, sizeof_scratch = 0;

        opt_next.const_input_flag = 0;
        opt_next.temporal_denoise_flag = 0;
        opt_next.width =  opt_next.width >> 1;
        opt_next.width += opt_next.width & 1;
        opt_next.height = opt_next.height >> 1;
        opt_next.height+= opt_next.height & 1;

        opt_next.vbv_size_bytes <<= 2;

        H264E_sizeof_one(&enc_curr->param, &sizeof_persist, &sizeof_scratch, 1);
        enc_curr = enc_curr->enc_next = (char *)enc_curr + sizeof_persist;

        ret = H264E_init_one(enc_curr, &opt_next, 1);
        if (ret)
            break;
    }
#endif
    return ret;
}

static void encode_slice(h264e_enc_t *enc, int frame_type, int long_term_idx_use, int long_term_idx_update, int pps_id, int enc_type)
{
    int i, k;
    encode_slice_header(enc, frame_type, long_term_idx_use, long_term_idx_update, pps_id,enc_type);
    // encode frame
    do
    {   // encode row
        do
        {   // encode macroblock
            if (enc->run_param.desired_nalu_bytes &&
                h264e_bs_get_pos_bits(enc->bs) > enc->run_param.desired_nalu_bytes*8u)
            {
                // start new slice
                nal_end(enc);
                encode_slice_header(enc, frame_type, long_term_idx_use, long_term_idx_update, pps_id, enc_type);
            }

            mb_encode(enc, enc_type);

            enc->dec.yuv[0] += 16;
            enc->dec.yuv[1] += 8;
            enc->dec.yuv[2] += 8;

            enc->mb.num++;  // before rc_mb_end
            if (enc->param.fine_rate_control_flag)
            {
                rc_mb_end(enc);
            }
        } while (++enc->mb.x < enc->frame.nmbx);

        for (i = 0, k = 16; i < 3; i++, k = 8)
        {
            enc->dec.yuv[i] += k*(enc->dec.stride[i] - enc->frame.nmbx);
        }

        // start new row
        enc->mb.x = 0;
        *((uint32_t*)(enc->nnz)) = *((uint32_t*)(enc->nnz + 4)) = 0x01010101 * NNZ_NA; // left edge of NNZ predictor
        enc->i4x4mode[0] = -1;

    } while (++enc->mb.y < enc->frame.nmby);

    if (enc->mb.skip_run)
    {
        UE(enc->mb.skip_run);
    }

    nal_end(enc);
    for (i = 0, k = 16; i < 3; i++, k = 8)
    {
        enc->dec.yuv[i] -= k*enc->dec.stride[i]*enc->frame.nmby;
    }
}

#if H264E_MAX_THREADS
typedef struct
{
    H264E_persist_t *enc;
    int frame_type, long_term_idx_use, long_term_idx_update, pps_id, enc_type;
} h264_enc_slice_thread_params_t;

static void encode_slice_thread_simple(void *arg)
{
    h264_enc_slice_thread_params_t *h = (h264_enc_slice_thread_params_t*)arg;
    encode_slice(h->enc, h->frame_type, h->long_term_idx_use, h->long_term_idx_update, h->pps_id, h->enc_type);
}
#endif

static int H264E_encode_one(H264E_persist_t *enc, const H264E_run_param_t *opt,
    int long_term_idx_use, int is_refers_to_long_term, int long_term_idx_update,
    int frame_type, int pps_id, int enc_type)
{
    int i, k;
    // slice reset
    // added by gxw, 2023-05-23
    // long_term_idx_use = -1; is_refers_to_long_term = 0; long_term_idx_update = 0;
    enc->slice.type = (long_term_idx_use < 0 ? SLICE_TYPE_I : SLICE_TYPE_P);
    // rc: Rate-control
    rc_frame_start(enc, (long_term_idx_use < 0) ? 1 : 0 /* is_intra */, is_refers_to_long_term);

    enc->mb.x = enc->mb.y = enc->mb.num = 0;

    if (long_term_idx_use > 0)
    {
        // Activate long-term reference buffer
        for (i = 0; i < 3; i++)
        {
            SWAP(pix_t*, enc->ref.yuv[i], enc->lt_yuv[long_term_idx_use - 1][i]);
        }
    }

    if (enc->param.vbv_size_bytes && !long_term_idx_use && long_term_idx_update <= 0 &&
        enc->rc.vbv_bits - enc->run_param.desired_frame_bytes*8 > enc->param.vbv_size_bytes*8)
    {
        // encode transparent frame on VBV overflow
        encode_slice_header(enc, frame_type, long_term_idx_use, long_term_idx_update, pps_id,enc_type);
        enc->mb.skip_run = enc->frame.nmb;
        UE(enc->mb.skip_run);
        nal_end(enc);
        for (i = 0, k = 16; i < 3; i++, k = 8)
        {
            pix_copy_pic(enc->dec.yuv[i], enc->dec.stride[i], enc->ref.yuv[i], enc->ref.stride[i], enc->frame.nmbx*k, enc->frame.nmby*k);
        }
    } else
    {
#if H264E_MAX_THREADS
        if (enc->param.max_threads > 1)
        {
            H264E_persist_t enc_thr[H264E_MAX_THREADS];
            int sizeof_scratch = enc_alloc_scratch(NULL, &enc->param, (void*)(uintptr_t)1);
            unsigned char *scratch_base = ((unsigned char*)enc->scratch) + sizeof_scratch;
            int mby = 0;
            int ithr;
            int nmby = enc->frame.nmby;
            void *savep[3];
            for (i = 0; i < 3; i++)
            {
                savep[i] = enc->dec.yuv[i];
            }

            for (ithr = 0; ithr < enc->param.max_threads; ithr++)
            {
                enc_thr[ithr] = *enc;
                enc_thr[ithr].mb.y = mby;
                enc_thr[ithr].mb.num = mby*enc->frame.nmbx;
                mby += (enc->frame.nmby - mby) / (enc->param.max_threads - ithr);
                enc_thr[ithr].frame.nmby = mby;
                enc_thr[ithr].rc.bit_budget /= enc->param.max_threads;
                enc_thr[ithr].frame.nmb = enc_thr[ithr].frame.nmbx * enc_thr[ithr].frame.nmby;

                for (i = 0, k = 16; i < 3; i++, k = 8)
                {
                    enc_thr[ithr].dec.yuv[i] += k*enc->dec.stride[i]*enc_thr[ithr].mb.y;
                }

                //enc_alloc_scratch(enc_thr + ithr, &enc->param, (unsigned char*)(scratch_thr[ithr]));
                scratch_base += enc_alloc_scratch(enc_thr + ithr, &enc->param, scratch_base);
                enc_thr[ithr].out_pos = 0;
                h264e_bs_init_bits(enc_thr[ithr].bs, enc_thr[ithr].out);
            }

            {
                h264_enc_slice_thread_params_t thread_par[H264E_MAX_THREADS];
                void *args[H264E_MAX_THREADS];
                for (i = 0; i < enc->param.max_threads; i++)
                {
                    thread_par[i].enc = enc_thr + i;
                    thread_par[i].frame_type = frame_type;
                    thread_par[i].long_term_idx_use = long_term_idx_use;
                    thread_par[i].long_term_idx_update = long_term_idx_update;
                    thread_par[i].pps_id = pps_id;
                    thread_par[i].enc_type = enc_type;
                    args[i] = thread_par + i;
                }
                enc->param.run_func_in_thread(enc->param.token, encode_slice_thread_simple, args, enc->param.max_threads);
            }

            for (i = 0; i < enc->param.max_threads; i++)
            {
                memcpy(enc->out + enc->out_pos, enc_thr[i].out, enc_thr[i].out_pos);
                enc->out_pos += enc_thr[i].out_pos;
            }
            enc->frame.nmby = nmby;
            for (i = 0; i < 3; i++)
            {
                enc->dec.yuv[i] = savep[i];
            }
        } else
#endif
        {
            // If undef H264E_SVC_API, enc_type is always 0
            // pps_id = enc->param.sps_id*4
            encode_slice(enc, frame_type, long_term_idx_use, long_term_idx_update, pps_id, enc_type);
        }
    }

    // Set flags for AMM state machine for standard compliance
    if (frame_type == H264E_FRAME_TYPE_KEY)
    {
        // Reset long-term reference frames
        memset(enc->lt_used, 0, sizeof(enc->lt_used));
        // Assume that this frame is not short-term (have effect only if AMM used)
        enc->short_term_used = 0;
    }
    if (long_term_idx_update > 0)
    {
        enc->lt_used[long_term_idx_update - 1] = 1;
    } else if (long_term_idx_update == 0)
    {
        enc->short_term_used = 1;
    }

    rc_frame_end(enc, long_term_idx_use == -1, enc->mb.skip_run == enc->frame.nmb, is_refers_to_long_term);

    if (long_term_idx_use > 0)
    {
        // deactivate long-term reference
        for (i = 0; i < 3; i++)
        {
            SWAP(pix_t*, enc->ref.yuv[i], enc->lt_yuv[long_term_idx_use - 1][i]);
        }
    }

    if (long_term_idx_update != -1)
    {
        pix_copy_recon_pic_to_ref(enc);

        if (++enc->frame.num >= enc->param.gop && enc->param.gop && (opt->frame_type == H264E_FRAME_TYPE_DEFAULT))
        {
            enc->frame.num = 0;     // trigger to encode IDR on next call
        }

        if (long_term_idx_update > 0)
        {
            for (i = 0; i < 3; i++)
            {
                SWAP(pix_t*, enc->ref.yuv[i], enc->lt_yuv[long_term_idx_update - 1][i]);
            }
        }
    }

    return H264E_STATUS_SUCCESS;
}

static int check_parameters_align(const H264E_create_param_t *opt, const H264E_io_yuv_t *in)
{
    int i;
    int min_align = 0;
#if H264E_ENABLE_NEON || H264E_ENABLE_SSE2
    min_align = 7;
#endif
    if (opt->const_input_flag && opt->temporal_denoise_flag)
    {
        min_align = 0;
    }
    for (i = 0; i < 3; i++)
    {
        if (((uintptr_t)in->yuv[i]) & min_align)
        {
            return i ? H264E_STATUS_BAD_CHROMA_ALIGN : H264E_STATUS_BAD_LUMA_ALIGN;
        }
        if (in->stride[i] & min_align)
        {
            return i ? H264E_STATUS_BAD_CHROMA_STRIDE : H264E_STATUS_BAD_LUMA_STRIDE;
        }
    }
    return H264E_STATUS_SUCCESS;
}

/**
*   Top-level encode function
*   See header file for details.
*/
int H264E_encode(H264E_persist_t *enc, H264E_scratch_t *scratch, const H264E_run_param_t *opt,
    H264E_io_yuv_t *in, unsigned char **coded_data, int *sizeof_coded_data)
{
    int i;
    int frame_type;
    int long_term_idx_use;
    int long_term_idx_update;
    int is_refers_to_long_term;
    int error;

    error = check_parameters_align(&enc->param, in);
    if (error)
    {
        return error;
    }
    (void)i;
    i = enc_alloc_scratch(enc, &enc->param, (unsigned char*)scratch);
#if H264E_SVC_API
    {
        H264E_persist_t *e = enc->enc_next;
        while (e)
        {
            i += enc_alloc_scratch(e, &enc->param, ((unsigned char*)scratch) + i);
            e = e->enc_next;
        }
    }
#endif

    enc->inp = *in;

#if H264E_ENABLE_DENOISE
    // 1. Run optional denoise filter
    if (enc->param.temporal_denoise_flag && opt->encode_speed < 2)
    {
        int sh = 0;
        for (i = 0; i < 3; i++)
        {
            h264e_denoise_run(in->yuv[i], enc->denoise.yuv[i],  enc->param.width >> sh, enc->param.height >> sh, in->stride[i], enc->denoise.stride[i]);
            enc->inp.yuv[i] = enc->denoise.yuv[i];
            enc->inp.stride[i] = enc->denoise.stride[i];
            sh = 1;
        }
    }
#endif

    enc->out_pos = 0;   // reset output bitbuffer position

    if (opt)
    {
        enc->run_param = *opt;  // local copy of run-time parameters
    }
    opt = &enc->run_param;      // refer to local copy

    // silently fix invalid QP without warning
    if (!enc->run_param.qp_max || enc->run_param.qp_max > 51)
    {
        enc->run_param.qp_max = 51;
    }
    if (!enc->run_param.qp_min || enc->run_param.qp_min < MIN_QP)
    {
        enc->run_param.qp_min = MIN_QP;
    }

    enc->speed.disable_deblock = (opt->encode_speed == 8 || opt->encode_speed == 10);

    if (!enc->param.const_input_flag)
    {
        // if input frame can be re-used as a scratch, set reconstructed frame to the input
        enc->dec = *in;
    }

    // Set default frame type
    frame_type = opt->frame_type;
    if (frame_type == H264E_FRAME_TYPE_DEFAULT)
    {
        frame_type = enc->frame.num ? H264E_FRAME_TYPE_P : H264E_FRAME_TYPE_KEY;
    }
    // Estimate long-term indexes from frame type
    // index 0 means "short-term" reference
    // index -1 means "not used"
    switch (frame_type)
    {
    default:
    case H264E_FRAME_TYPE_I:        long_term_idx_use = -1; long_term_idx_update = 0; break;
    case H264E_FRAME_TYPE_KEY:      long_term_idx_use = -1; long_term_idx_update = enc->param.max_long_term_reference_frames > 0; break;
    case H264E_FRAME_TYPE_GOLDEN:   long_term_idx_use =  1; long_term_idx_update = 1; break;
    case H264E_FRAME_TYPE_RECOVERY: long_term_idx_use =  1; long_term_idx_update = 0; break;
    case H264E_FRAME_TYPE_P:        long_term_idx_use =  enc->most_recent_ref_frame_idx; long_term_idx_update =  0; break;
    case H264E_FRAME_TYPE_DROPPABLE:long_term_idx_use =  enc->most_recent_ref_frame_idx; long_term_idx_update = -1; break;
    case H264E_FRAME_TYPE_CUSTOM:   long_term_idx_use =  opt->long_term_idx_use; long_term_idx_update = opt->long_term_idx_update;
        if (!long_term_idx_use)
        {
            long_term_idx_use = enc->most_recent_ref_frame_idx;
        }
        if (long_term_idx_use < 0)
        {
            // hack: redefine frame type, always encode IDR
            frame_type = H264E_FRAME_TYPE_KEY;
        }
        break;
    }

#if H264E_RATE_CONTROL_GOLDEN_FRAMES
    is_refers_to_long_term = (long_term_idx_use != enc->most_recent_ref_frame_idx && long_term_idx_use >= 0);
#else
    is_refers_to_long_term = 0;
#endif

    if (long_term_idx_update >= 0)
    {
        enc->most_recent_ref_frame_idx = long_term_idx_update;
    }
    if (frame_type == H264E_FRAME_TYPE_KEY)
    {
        int pic_init_qp = 30;
        pic_init_qp = MIN(pic_init_qp, enc->run_param.qp_max);
        pic_init_qp = MAX(pic_init_qp, enc->run_param.qp_min);

        //temp only two layers!
        enc->sps.pic_init_qp = pic_init_qp;
        enc->next_idr_pic_id ^= 1;
        enc->frame.num = 0;

#if H264E_SVC_API
        if (enc->param.num_layers > 1)
        {
            H264E_persist_t *enc_base = enc->enc_next;
            enc_base->sps.pic_init_qp = pic_init_qp;
            enc_base->next_idr_pic_id ^= 1;
            enc_base->frame.num = 0;

            enc_base->out = enc->out;
            enc_base->out_pos = 0;
            encode_sps(enc_base, 66);
            encode_pps(enc_base, 0);

            enc->out_pos += enc_base->out_pos;
            encode_sps(enc, 83);
            encode_pps(enc, 1);
        } else
#endif
        {
            encode_sps(enc, 66);
            encode_pps(enc, 0);
        }
    } else
    {
        if (!enc->sps.pic_init_qp)
        {
            return H264E_STATUS_BAD_FRAME_TYPE;
        }
        if (long_term_idx_use > enc->param.max_long_term_reference_frames ||
            long_term_idx_update > enc->param.max_long_term_reference_frames ||
            long_term_idx_use > MAX_LONG_TERM_FRAMES)
        {
            return H264E_STATUS_BAD_FRAME_TYPE;
        }
    }

#if H264E_SVC_API
    if (enc->param.num_layers > 1)
    {
        H264E_persist_t *enc_base = enc->enc_next;
        int sh = 0;

        enc_base->run_param = enc->run_param;
        enc_base->run_param.desired_frame_bytes = enc->run_param.desired_frame_bytes >> 2;

        for (i = 0; i < 3; i++)
        {
            h264e_frame_downsampling(enc_base->inp.yuv[i], enc_base->inp.stride[i], enc_base->frame.h >> sh,
                in->yuv[i], in->stride[i], enc->param.height >> sh, enc_base->param.width >> sh,
                enc_base->param.height >> sh, enc->param.width >> sh, enc->param.height >> sh);
            sh = 1;
        }

        enc_base->scratch = enc->scratch;
        enc_base->out = enc->out + enc->out_pos;
        enc_base->out_pos = 0;

        H264E_encode_one(enc_base, &enc_base->run_param, long_term_idx_use, is_refers_to_long_term, long_term_idx_update,
            frame_type, enc->param.sps_id*4 + 0, 0);

        enc->out_pos += enc_base->out_pos;

        if ((frame_type == H264E_FRAME_TYPE_I || frame_type == H264E_FRAME_TYPE_KEY) && enc->param.inter_layer_pred_flag)
        {
            for (i = 0, sh = 0; i < 3; i++, sh = 1)
            {
                h264e_intra_upsampling(enc_base->frame.w >> sh, enc_base->frame.h >> sh, enc->frame.w >> sh, enc->frame.h >> sh,
                    sh, enc_base->dec.yuv[i], enc_base->dec.stride[i], enc->ref.yuv[i], enc->ref.stride[i]);
            }
        }

        memset(enc->df.df_nzflag, 0, enc->frame.nmbx);
        H264E_encode_one(enc, opt, long_term_idx_use, is_refers_to_long_term, long_term_idx_update,
            frame_type, enc->param.sps_id*4 + 1, 20);
    } else
#endif // H264E_SVC_API
    {
        H264E_encode_one(enc, opt, long_term_idx_use, is_refers_to_long_term, long_term_idx_update,
            frame_type, enc->param.sps_id*4 + 0, 0);
    }

    *sizeof_coded_data = enc->out_pos;
    *coded_data = enc->out;
    return H264E_STATUS_SUCCESS;
}

/**
*   Return persistent and scratch memory requirements
*   for given encoding options.
*   See header file for details.
*/
int H264E_sizeof(const H264E_create_param_t *par, int *sizeof_persist, int *sizeof_scratch)
{
    int i;
    int error = H264E_sizeof_one(par, sizeof_persist, sizeof_scratch, 0);
    (void)i;
#if H264E_SVC_API
    for (i = par->num_layers; i > 1; i--)
    {
        H264E_create_param_t opt_next = *par;
        opt_next.const_input_flag = 1;
        opt_next.temporal_denoise_flag = 0;
        opt_next.width   = opt_next.width >> 1;
        opt_next.width  += opt_next.width & 1;
        opt_next.height  = opt_next.height >> 1;
        opt_next.height += opt_next.height & 1;
        *sizeof_persist += enc_alloc(NULL, par, (void*)(uintptr_t)1, 1) + sizeof(h264e_enc_t);
#if H264E_MAX_THREADS > 1
        *sizeof_scratch += enc_alloc_scratch(NULL, par, (void*)(uintptr_t)1) * (H264E_MAX_THREADS + 1);
#else
        *sizeof_scratch += enc_alloc_scratch(NULL, par, (void*)(uintptr_t)1);
#endif
    }
#endif
    return error;
}

/**
*   Set VBV size and fullness
*   See header file for details.
*/
void H264E_set_vbv_state(
    H264E_persist_t *enc,
    int vbv_size_bytes,     //< New VBV size
    int vbv_fullness_bytes  //< New VBV fulness, -1 = no change
)
{
    if (enc)
    {
        enc->param.vbv_size_bytes = vbv_size_bytes;
        if (vbv_fullness_bytes >= 0)
        {
            enc->rc.vbv_bits = vbv_fullness_bytes*8;
            enc->rc.vbv_target_level = enc->rc.vbv_bits;
        }
    }
}
#endif