path: root/src/kernels.h

#ifndef WSAP_TERM_KERNELS_H
#define WSAP_TERM_KERNELS_H

#include "cell.h"
#include "compat.h"
#include "simd.h"

#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>

#if defined(TERM_KERNEL_AVX512)
TERM_TARGET_AVX512
static inline size_t scan_printable_span_avx512(const char *buf, size_t len, unsigned char hi)
{
    size_t i = 0;
    __m512i v20_512 = _mm512_set1_epi8(0x20);
    __m512i vlim_512 = _mm512_set1_epi8((char)(hi - 0x20));
    for (; i + 64 <= len; i += 64) {
        __m512i v = _mm512_loadu_si512((const void *)(buf + i));
        __mmask64 mask = _mm512_cmpge_epu8_mask(_mm512_sub_epi8(v, v20_512), vlim_512);
        if (mask != 0)
            return i + term_ctz64(mask);
    }
    if (i < len) {
        __mmask64 load_mask = ((uint64_t)1 << (len - i)) - 1;
        __m512i v = _mm512_maskz_loadu_epi8(load_mask, buf + i);
        __mmask64 mask = _mm512_mask_cmpge_epu8_mask(load_mask, _mm512_sub_epi8(v, v20_512), vlim_512);
        if (mask != 0)
            return i + term_ctz64(mask);
    }
    return len;
}
#endif

#if defined(TERM_KERNEL_AVX2)
TERM_TARGET_AVX2
static inline size_t scan_printable_span_avx2(const char *buf, size_t len, unsigned char hi)
{
    size_t i = 0;
    const __m256i v_offset = _mm256_set1_epi8((char)(0x20 + 0x80));
    const __m256i v_limit = _mm256_set1_epi8((char)(hi - 0xa1));
    for (; i + 64 <= len; i += 64) {
        __m256i v0 = _mm256_loadu_si256((const __m256i *)(buf + i));
        __m256i v1 = _mm256_loadu_si256((const __m256i *)(buf + i + 32));
        __m256i sub0 = _mm256_sub_epi8(v0, v_offset);
        __m256i sub1 = _mm256_sub_epi8(v1, v_offset);
        int m0 = _mm256_movemask_epi8(_mm256_cmpgt_epi8(sub0, v_limit));
        int m1 = _mm256_movemask_epi8(_mm256_cmpgt_epi8(sub1, v_limit));
        if (TERM_EXPECT((m0 | m1) != 0, 0)) {
            if (m0 != 0)
                return i + term_ctz32(m0);
            return i + 32 + term_ctz32(m1);
        }
    }
    if (i + 32 <= len) {
        __m256i v = _mm256_loadu_si256((const __m256i *)(buf + i));
        __m256i sub = _mm256_sub_epi8(v, v_offset);
        int mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(sub, v_limit));
        if (mask != 0)
            return i + term_ctz32(mask);
        i += 32;
    }
    if (i + 16 <= len) {
        __m128i v = _mm_loadu_si128((const __m128i *)(buf + i));
        __m128i sub = _mm_sub_epi8(v, _mm_set1_epi8((char)(0x20 + 0x80)));
        int mask = _mm_movemask_epi8(_mm_cmpgt_epi8(sub, _mm_set1_epi8((char)(hi - 0xa1)))) & 0xFFFF;
        if (mask != 0)
            return i + term_ctz32(mask);
        i += 16;
    }
    for (; i < len; i++) {
        unsigned char c = (unsigned char)buf[i];
        if (c < 0x20 || c >= hi)
            return i;
    }
    return len;
}
#endif

static inline size_t scan_printable_span(const char *buf, size_t len,
                                         unsigned char hi)
{
    /* A short span costs less inline than as a call. */
#if defined(TERM_KERNEL_AVX512)
    if (len >= 64 && term_use_avx512())
        return scan_printable_span_avx512(buf, len, hi);
#endif
#if defined(TERM_KERNEL_AVX2)
    if (len >= 32 && term_use_avx2())
        return scan_printable_span_avx2(buf, len, hi);
#endif
    size_t i = 0;
#if defined(__aarch64__)
    uint8x16_t v20 = vdupq_n_u8(0x20);
    uint8x16_t vlim = vdupq_n_u8((uint8_t)(hi - 0x20));
    for (; i + 64 <= len; i += 64) {
        uint8x16_t v0 = vld1q_u8((const uint8_t *)buf + i);
        uint8x16_t v1 = vld1q_u8((const uint8_t *)buf + i + 16);
        uint8x16_t v2 = vld1q_u8((const uint8_t *)buf + i + 32);
        uint8x16_t v3 = vld1q_u8((const uint8_t *)buf + i + 48);

        uint8x16_t m0 = vcgeq_u8(vsubq_u8(v0, v20), vlim);
        uint8x16_t m1 = vcgeq_u8(vsubq_u8(v1, v20), vlim);
        uint8x16_t m2 = vcgeq_u8(vsubq_u8(v2, v20), vlim);
        uint8x16_t m3 = vcgeq_u8(vsubq_u8(v3, v20), vlim);

        uint8x16_t mor = vorrq_u8(vorrq_u8(m0, m1), vorrq_u8(m2, m3));
        uint64x2_t mor64 = vreinterpretq_u64_u8(mor);
        if (TERM_EXPECT((vgetq_lane_u64(mor64, 0) | vgetq_lane_u64(mor64, 1)) != 0, 0)) {
            uint64x2_t m0_64 = vreinterpretq_u64_u8(m0);
            uint64_t lo0 = vgetq_lane_u64(m0_64, 0), hi0 = vgetq_lane_u64(m0_64, 1);
            if (lo0) return i + (term_ctz64(lo0) >> 3);
            if (hi0) return i + 8 + (term_ctz64(hi0) >> 3);

            uint64x2_t m1_64 = vreinterpretq_u64_u8(m1);
            uint64_t lo1 = vgetq_lane_u64(m1_64, 0), hi1 = vgetq_lane_u64(m1_64, 1);
            if (lo1) return i + 16 + (term_ctz64(lo1) >> 3);
            if (hi1) return i + 24 + (term_ctz64(hi1) >> 3);

            uint64x2_t m2_64 = vreinterpretq_u64_u8(m2);
            uint64_t lo2 = vgetq_lane_u64(m2_64, 0), hi2 = vgetq_lane_u64(m2_64, 1);
            if (lo2) return i + 32 + (term_ctz64(lo2) >> 3);
            if (hi2) return i + 40 + (term_ctz64(hi2) >> 3);

            uint64x2_t m3_64 = vreinterpretq_u64_u8(m3);
            uint64_t lo3 = vgetq_lane_u64(m3_64, 0), hi3 = vgetq_lane_u64(m3_64, 1);
            if (lo3) return i + 48 + (term_ctz64(lo3) >> 3);
            if (hi3) return i + 56 + (term_ctz64(hi3) >> 3);
        }
    }
    for (; i + 16 <= len; i += 16) {
        uint8x16_t v = vld1q_u8((const uint8_t *)buf + i);
        uint8x16_t mask = vcgeq_u8(vsubq_u8(v, v20), vlim);
        uint64x2_t m64 = vreinterpretq_u64_u8(mask);
        uint64_t lo = vgetq_lane_u64(m64, 0);
        uint64_t hi64 = vgetq_lane_u64(m64, 1);
        if (lo) return i + (term_ctz64(lo) >> 3);
        if (hi64) return i + 8 + (term_ctz64(hi64) >> 3);
    }
    if (i + 8 <= len) {
        uint8x8_t v8 = vld1_u8((const uint8_t *)buf + i);
        uint8x8_t mask8 = vcge_u8(vsub_u8(v8, vget_low_u8(v20)), vget_low_u8(vlim));
        uint64_t m64 = vget_lane_u64(vreinterpret_u64_u8(mask8), 0);
        if (m64) return i + (term_ctz64(m64) >> 3);
        i += 8;
    }
#elif defined(__x86_64__)
    const __m128i v_offset = _mm_set1_epi8((char)(0x20 + 0x80));
    const __m128i v_limit = _mm_set1_epi8((char)(hi - 0xa1));
    for (; i + 64 <= len; i += 64) {
        __m128i m0 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i)), v_offset), v_limit);
        __m128i m1 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i + 16)), v_offset), v_limit);
        __m128i m2 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i + 32)), v_offset), v_limit);
        __m128i m3 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i + 48)), v_offset), v_limit);
        if (TERM_EXPECT(_mm_movemask_epi8(_mm_or_si128(_mm_or_si128(m0, m1), _mm_or_si128(m2, m3))) != 0, 0)) {
            uint64_t mask = (uint64_t)(unsigned)_mm_movemask_epi8(m0) |
                            (uint64_t)(unsigned)_mm_movemask_epi8(m1) << 16 |
                            (uint64_t)(unsigned)_mm_movemask_epi8(m2) << 32 |
                            (uint64_t)(unsigned)_mm_movemask_epi8(m3) << 48;
            return i + (size_t)term_ctz64(mask);
        }
    }
    for (; i + 16 <= len; i += 16) {
        int mask = _mm_movemask_epi8(_mm_cmpgt_epi8(
            _mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i)), v_offset), v_limit));
        if (mask != 0)
            return i + (size_t)term_ctz32((unsigned)mask);
    }
#endif
    for (; i < len; i++) {
        unsigned char c = (unsigned char)buf[i];
        if (c < 0x20 || c >= hi)
            return i;
    }
    return len;
}

/* Copy n bytes with loads that stay inside [src, src + n). Short text runs
 * are common; a libc call costs more than the copy. */
static inline TERM_ALWAYS_INLINE void bytes_copy_small(uint8_t *restrict dst,
                                                                   const char *restrict src, int n)
{
    if (TERM_EXPECT((unsigned)(n - 16) <= 16u, 1)) {
        /* 16 to 32 bytes: two overlapping 16-byte copies. Both loads stay
         * inside [src, src + n). This is the usual history line length, and
         * two copies cost less here than the loop below. */
        memcpy(dst, src, 16);
        memcpy(dst + n - 16, src + n - 16, 16);
    } else if (n > 256) {
        memcpy(dst, src, (size_t)n);
    } else if (n >= 16) {
        int k = 0;
        for (; k + 16 <= n; k += 16)
            memcpy(dst + k, src + k, 16);
        if (k < n)
            memcpy(dst + n - 16, src + n - 16, 16);
    } else if (n >= 8) {
        uint64_t a, b;
        memcpy(&a, src, 8);
        memcpy(&b, src + n - 8, 8);
        memcpy(dst, &a, 8);
        memcpy(dst + n - 8, &b, 8);
    } else if (n >= 4) {
        uint32_t a, b;
        memcpy(&a, src, 4);
        memcpy(&b, src + n - 4, 4);
        memcpy(dst, &a, 4);
        memcpy(dst + n - 4, &b, 4);
    } else if (n >= 2) {
        uint16_t a, b;
        memcpy(&a, src, 2);
        memcpy(&b, src + n - 2, 2);
        memcpy(dst, &a, 2);
        memcpy(dst + n - 2, &b, 2);
    } else if (n == 1) {
        dst[0] = (uint8_t)src[0];
    }
}

static inline size_t scan_crlf_tail(const char *buf, size_t i, size_t len)
{
    while (i + 1 < len && buf[i] == '\r' && buf[i + 1] == '\n')
        i += 2;
    return i;
}

#if defined(TERM_KERNEL_AVX512)
TERM_TARGET_AVX512
static inline size_t scan_crlf_pairs_avx512(const char *buf, size_t len)
{
    size_t i = 0;
    const __m512i pair = _mm512_set1_epi16(0x0A0D);
    for (; i + 64 <= len; i += 64) {
        uint64_t eq = _mm512_cmpeq_epi8_mask(_mm512_loadu_si512((const void *)(buf + i)), pair);
        if (eq != ~0ull)
            return i + ((size_t)term_ctz64(~eq) & ~(size_t)1);
    }
    return scan_crlf_tail(buf, i, len);
}
#endif

#if defined(TERM_KERNEL_AVX2)
TERM_TARGET_AVX2
static inline size_t scan_crlf_pairs_avx2(const char *buf, size_t len)
{
    size_t i = 0;
    const __m256i pair = _mm256_set1_epi16(0x0A0D);
    for (; i + 32 <= len; i += 32) {
        uint32_t eq = (uint32_t)_mm256_movemask_epi8(
            _mm256_cmpeq_epi8(_mm256_loadu_si256((const __m256i *)(const void *)(buf + i)), pair));
        if (eq != ~0u)
            return i + ((size_t)term_ctz32(~eq) & ~(size_t)1);
    }
    return scan_crlf_tail(buf, i, len);
}
#endif

/* The length of the prefix of buf made of "\r\n" pairs; always even. */
static inline size_t scan_crlf_pairs(const char *buf, size_t len)
{
#if defined(TERM_KERNEL_AVX512)
    if (term_use_avx512())
        return scan_crlf_pairs_avx512(buf, len);
#endif
#if defined(TERM_KERNEL_AVX2)
    if (term_use_avx2())
        return scan_crlf_pairs_avx2(buf, len);
#endif
    size_t i = 0;
#if defined(__aarch64__)
    const uint8x16_t pair = vreinterpretq_u8_u16(vdupq_n_u16(0x0A0D));
    for (; i + 16 <= len; i += 16) {
        uint8x16_t eq = vceqq_u8(vld1q_u8((const uint8_t *)buf + i), pair);
        uint16_t bits = term_lane_bits(eq);
        if (bits != 0xFFFF)
            return i + ((size_t)term_ctz32((unsigned)~bits & 0xFFFFu) & ~(size_t)1);
    }
#endif
    return scan_crlf_tail(buf, i, len);
}

#if defined(TERM_KERNEL_AVX512)
TERM_TARGET_AVX512
static inline size_t scan_byte_run_avx512(const char *buf, size_t len, char byte)
{
    size_t i = 0;
    const __m512i want = _mm512_set1_epi8(byte);
    for (; i + 64 <= len; i += 64) {
        uint64_t eq = _mm512_cmpeq_epi8_mask(_mm512_loadu_si512((const void *)(buf + i)), want);
        if (eq != ~0ull)
            return i + (size_t)term_ctz64(~eq);
    }
    while (i < len && buf[i] == byte)
        i++;
    return i;
}
#endif

#if defined(TERM_KERNEL_AVX2)
TERM_TARGET_AVX2
static inline size_t scan_byte_run_avx2(const char *buf, size_t len, char byte)
{
    size_t i = 0;
    const __m256i want = _mm256_set1_epi8(byte);
    for (; i + 32 <= len; i += 32) {
        uint32_t eq = (uint32_t)_mm256_movemask_epi8(
            _mm256_cmpeq_epi8(_mm256_loadu_si256((const __m256i *)(const void *)(buf + i)), want));
        if (eq != ~0u)
            return i + (size_t)term_ctz32(~eq);
    }
    while (i < len && buf[i] == byte)
        i++;
    return i;
}
#endif

/* The length of the prefix of buf made of byte. */
static inline size_t scan_byte_run(const char *buf, size_t len, char byte)
{
#if defined(TERM_KERNEL_AVX512)
    if (term_use_avx512())
        return scan_byte_run_avx512(buf, len, byte);
#endif
#if defined(TERM_KERNEL_AVX2)
    if (term_use_avx2())
        return scan_byte_run_avx2(buf, len, byte);
#endif
    size_t i = 0;
#if defined(__aarch64__)
    const uint8x16_t want = vdupq_n_u8((uint8_t)byte);
    for (; i + 16 <= len; i += 16) {
        uint16_t bits = term_lane_bits(vceqq_u8(vld1q_u8((const uint8_t *)buf + i), want));
        if (bits != 0xFFFF)
            return i + (size_t)term_ctz32((unsigned)~bits & 0xFFFFu);
    }
#elif defined(__x86_64__)
    const __m128i want = _mm_set1_epi8(byte);
    for (; i + 16 <= len; i += 16) {
        unsigned eq = (unsigned)_mm_movemask_epi8(
            _mm_cmpeq_epi8(_mm_loadu_si128((const __m128i *)(const void *)(buf + i)), want));
        if (eq != 0xFFFFu)
            return i + (size_t)term_ctz32(~eq & 0xFFFFu);
    }
#endif
    while (i < len && buf[i] == byte)
        i++;
    return i;
}

static inline int scan_ascii_run(const char *buf, int len)
{
    return (int)scan_printable_span(buf, (size_t)len, 0x7f);
}

static inline int scan_ascii_lf_tail(const char *buf, int i, int len, uint16_t *ends, int count)
{
    for (; i < len; i++) {
        unsigned char ch = (unsigned char)buf[i];
        if (ch == '\n') ends[++count] = (uint16_t)(i + 1);
        else if (ch - 0x20u >= 0x5fu) break;
    }
    return count;
}

#if defined(TERM_KERNEL_AVX512)
TERM_TARGET_AVX512 TERM_MAYBE_UNUSED
static int scan_ascii_lf_avx512(const char *buf, int len, uint16_t *ends)
{
    int i = 0, count = 0;
    ends[0] = 0;
    const __m512i lanes = _mm512_setr_epi32(
        0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c,
        0x13121110,0x17161514,0x1b1a1918,0x1f1e1d1c,
        0x23222120,0x27262524,0x2b2a2928,0x2f2e2d2c,
        0x33323130,0x37363534,0x3b3a3938,0x3f3e3d3c);
    __m128i block = _mm_set1_epi16(1);
    for (; i + 256 <= len; i += 256) {
        __m512i v[4];
        uint64_t nl[4];
        uint64_t bad = 0;
        for (int q = 0; q < 4; q++) {
            v[q] = _mm512_loadu_si512(buf + i + q * 64);
            nl[q] = _mm512_cmpeq_epi8_mask(v[q], _mm512_set1_epi8('\n'));
            bad |= _mm512_cmpge_epu8_mask(_mm512_sub_epi8(v[q], _mm512_set1_epi8(0x20)),
                                         _mm512_set1_epi8(0x5f)) & ~nl[q];
        }
        if (bad) break;
        for (int q = 0; q < 4; q++) {
            unsigned n = (unsigned)term_popcount64(nl[q]);
            __m512i packed = _mm512_maskz_compress_epi8(nl[q], lanes);
            __m128i offsets = _mm_add_epi16(_mm_cvtepu8_epi16(_mm512_castsi512_si128(packed)), block);
            _mm_storeu_si128((__m128i *)(ends + count + 1), offsets);
            if (TERM_EXPECT(n > 8, 0)) {
                __m512i base = _mm512_broadcast_i32x4(block);
                __m512i wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_castsi512_si256(packed)), base);
                _mm512_storeu_si512(ends + count + 1, wide);
                if (n > 32) {
                    wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(packed, 1)), base);
                    _mm512_storeu_si512(ends + count + 33, wide);
                }
            }
            count += (int)n;
            block = _mm_add_epi16(block, _mm_set1_epi16(64));
        }
    }
    for (; i + 64 <= len; i += 64) {
        __m512i v = _mm512_loadu_si512(buf + i);
        uint64_t nl = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\n'));
        uint64_t bad = _mm512_cmpge_epu8_mask(
            _mm512_sub_epi8(v, _mm512_set1_epi8(0x20)), _mm512_set1_epi8(0x5f)) & ~nl;
        if (bad) break;
        unsigned n = (unsigned)term_popcount64(nl);
        __m512i packed = _mm512_maskz_compress_epi8(nl, lanes);
        __m128i offsets = _mm_add_epi16(_mm_cvtepu8_epi16(_mm512_castsi512_si128(packed)), block);
        _mm_storeu_si128((__m128i *)(ends + count + 1), offsets);
        if (TERM_EXPECT(n > 8, 0)) {
            __m512i base = _mm512_broadcast_i32x4(block);
            __m512i wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_castsi512_si256(packed)), base);
            _mm512_storeu_si512(ends + count + 1, wide);
            if (n > 32) {
                wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(packed, 1)), base);
                _mm512_storeu_si512(ends + count + 33, wide);
            }
        }
        count += (int)n;
        block = _mm_add_epi16(block, _mm_set1_epi16(64));
    }
    return scan_ascii_lf_tail(buf, i, len, ends, count);
}
#endif

#if defined(TERM_KERNEL_AVX2)
TERM_TARGET_AVX2 TERM_MAYBE_UNUSED
static int scan_ascii_lf_avx2(const char *buf, int len, uint16_t *ends)
{
    int i = 0, count = 0;
    ends[0] = 0;
    for (; i + 32 <= len; i += 32) {
        __m256i v = _mm256_loadu_si256((const __m256i *)(buf + i));
        uint32_t nl = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\n')));
        __m256i d = _mm256_sub_epi8(v, _mm256_set1_epi8((char)0xa0));
        uint32_t bad = (uint32_t)_mm256_movemask_epi8(
            _mm256_cmpgt_epi8(d, _mm256_set1_epi8((char)0xde))) & ~nl;
        if (bad) break;
        while (nl) {
            ends[++count] = (uint16_t)(i + term_ctz32(nl) + 1);
            nl &= nl - 1;
        }
    }
    return scan_ascii_lf_tail(buf, i, len, ends, count);
}
#endif

/* Find complete ASCII lines. Stop at the first byte other than text or LF.
 * The caller supplies len + 65 entries. len must fit in uint16_t. */
static TERM_NOINLINE TERM_MAYBE_UNUSED int scan_ascii_lf(const char *buf, int len, uint16_t *ends)
{
#if defined(TERM_KERNEL_AVX512)
    if (term_use_avx512())
        return scan_ascii_lf_avx512(buf, len, ends);
#endif
#if defined(TERM_KERNEL_AVX2)
    if (term_use_avx2())
        return scan_ascii_lf_avx2(buf, len, ends);
#endif
    int i = 0, count = 0;
    ends[0] = 0;
#if defined(__aarch64__)
    for (; i + 32 <= len; i += 32) {
        uint32_t nl = 0, bad = 0;
        for (int q = 0; q < 2; q++) {
            uint8x16_t v = vld1q_u8((const uint8_t *)buf + i + q * 16);
            nl |= (uint32_t)term_lane_bits(vceqq_u8(v, vdupq_n_u8('\n'))) << (q * 16);
            bad |= (uint32_t)term_lane_bits(vcgeq_u8(vsubq_u8(v, vdupq_n_u8(0x20)),
                                                    vdupq_n_u8(0x5f))) << (q * 16);
        }
        bad &= ~nl;
        if (bad) break;
        while (nl) {
            ends[++count] = (uint16_t)(i + term_ctz32(nl) + 1);
            nl &= nl - 1;
        }
    }
#endif
    return scan_ascii_lf_tail(buf, i, len, ends, count);
}

#if defined(TERM_KERNEL_AVX512)
TERM_TARGET_AVX512
static inline void fill_cells_avx512(struct cell * restrict row, int cols, uint32_t fg, uint32_t bg)
{
    __m128i cell = _mm_setr_epi32(' ', (int)fg, (int)bg, 0);
    __m512i blank = _mm512_broadcast_i32x4(cell);
    int c = 0;
    for (; c + 8 <= cols; c += 8) {
        _mm512_storeu_si512((void *)(row + c), blank);
        _mm512_storeu_si512((void *)(row + c + 4), blank);
    }
    int rem = cols - c;
    if (rem >= 4) {
        _mm512_storeu_si512((void *)(row + c), blank);
        c += 4;
        rem -= 4;
    }
    if (rem > 0) {
        uint16_t store_mask = (uint16_t)((1u << (4 * rem)) - 1);
        _mm512_mask_storeu_epi32((void *)(row + c), store_mask, blank);
    }
}
#endif

#if defined(TERM_KERNEL_AVX2)
TERM_TARGET_AVX2
static inline void fill_cells_avx2(struct cell * restrict row, int cols, uint32_t fg, uint32_t bg)
{
    struct cell fill = { .c = ' ', .fg = fg, .bg = bg, .flags = 0, .pad = 0 };
    __m256i blank = _mm256_setr_epi32(' ', (int)fg, (int)bg, 0, ' ', (int)fg, (int)bg, 0);
    int c = 0;
    for (; c + 8 <= cols; c += 8) {
        char *d = (char *)(row + c);
        _mm256_storeu_si256((__m256i *)(d), blank);
        _mm256_storeu_si256((__m256i *)(d + 32), blank);
        _mm256_storeu_si256((__m256i *)(d + 64), blank);
        _mm256_storeu_si256((__m256i *)(d + 96), blank);
    }
    for (; c < cols; c++) {
        row[c] = fill;
    }
}
#endif

static inline void fill_cells(struct cell * restrict row, int cols, uint32_t fg, uint32_t bg)
{
    struct cell fill = { .c = ' ', .fg = fg, .bg = bg, .flags = 0, .pad = 0 };
#if defined(TERM_KERNEL_AVX512)
    if (term_use_avx512()) {
        if (TERM_EXPECT(cols > 0, 1))
            fill_cells_avx512(row, cols, fg, bg);
        return;
    }
#endif
#if defined(TERM_KERNEL_AVX2)
    if (term_use_avx2() && TERM_EXPECT(cols >= 8, 1)) {
        fill_cells_avx2(row, cols, fg, bg);
        return;
    }
#endif
#if defined(__aarch64__)
    if (TERM_EXPECT(cols > 0, 1)) {
        uint32x4_t blank = { ' ', fg, bg, 0 };
        uint32x4x4_t b4 = {{ blank, blank, blank, blank }};
        int c = 0;
        for (; c + 8 <= cols; c += 8) {
            vst1q_u32_x4((uint32_t *)(void *)(row + c), b4);
            vst1q_u32_x4((uint32_t *)(void *)(row + c + 4), b4);
        }
        for (; c + 4 <= cols; c += 4) {
            vst1q_u32_x4((uint32_t *)(void *)(row + c), b4);
        }
        for (; c < cols; c++) {
            vst1q_u32((uint32_t *)(void *)(row + c), blank);
        }
        return;
    }
#elif defined(__x86_64__)
    {
        __m128i blank = _mm_setr_epi32(' ', (int)fg, (int)bg, 0);
        int c = 0;
        for (; c + 4 <= cols; c += 4) {
            _mm_storeu_si128((__m128i *)(void *)(row + c), blank);
            _mm_storeu_si128((__m128i *)(void *)(row + c + 1), blank);
            _mm_storeu_si128((__m128i *)(void *)(row + c + 2), blank);
            _mm_storeu_si128((__m128i *)(void *)(row + c + 3), blank);
        }
        for (; c < cols; c++)
            _mm_storeu_si128((__m128i *)(void *)(row + c), blank);
        return;
    }
#endif
    for (int c = 0; c < cols; c++) {
        row[c] = fill;
    }
}

static inline void clear_cells_default(struct cell *row, int cols)
{
    fill_cells(row, cols, TERM_COLOR_FG, TERM_COLOR_BG);
}

#if defined(TERM_KERNEL_AVX512)
TERM_TARGET_AVX512
static inline void put_ascii_chunk_avx512(struct cell * restrict dst, const char * restrict src, int n,
                                         uint32_t fg, uint32_t bg, uint32_t flags)
{
    int k = 0;
    __m128i cell = _mm_setr_epi32(0, (int)fg, (int)bg, (int)flags);
    __m512i base = _mm512_broadcast_i32x4(cell);
    const __m512i idx0 = _mm512_setr_epi32(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0);
    const __m512i idx1 = _mm512_setr_epi32(4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0);
    const __m512i idx2 = _mm512_setr_epi32(8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0);
    const __m512i idx3 = _mm512_setr_epi32(12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0);

    for (; k + 16 <= n; k += 16) {
        __m512i cps = _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i *)(src + k)));
        _mm512_storeu_si512(dst + k + 0, _mm512_mask_permutexvar_epi32(base, 0x1111, idx0, cps));
        _mm512_storeu_si512(dst + k + 4, _mm512_mask_permutexvar_epi32(base, 0x1111, idx1, cps));
        _mm512_storeu_si512(dst + k + 8, _mm512_mask_permutexvar_epi32(base, 0x1111, idx2, cps));
        _mm512_storeu_si512(dst + k + 12, _mm512_mask_permutexvar_epi32(base, 0x1111, idx3, cps));
    }
    int rem = n - k;
    if (rem > 0) {
        __m128i v = _mm_maskz_loadu_epi8((1 << rem) - 1, src + k);
        __m512i cps = _mm512_cvtepu8_epi32(v);
        if (rem <= 4) {
            uint16_t m = (uint16_t)((1u << (4 * rem)) - 1);
            _mm512_mask_storeu_epi32(dst + k, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx0, cps));
            return;
        }
        _mm512_storeu_si512(dst + k, _mm512_mask_permutexvar_epi32(base, 0x1111, idx0, cps));
        if (rem <= 8) {
            uint16_t m = (uint16_t)((1u << (4 * (rem - 4))) - 1);
            _mm512_mask_storeu_epi32(dst + k + 4, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx1, cps));
            return;
        }
        _mm512_storeu_si512(dst + k + 4, _mm512_mask_permutexvar_epi32(base, 0x1111, idx1, cps));
        if (rem <= 12) {
            uint16_t m = (uint16_t)((1u << (4 * (rem - 8))) - 1);
            _mm512_mask_storeu_epi32(dst + k + 8, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx2, cps));
            return;
        }
        _mm512_storeu_si512(dst + k + 8, _mm512_mask_permutexvar_epi32(base, 0x1111, idx2, cps));
        uint16_t m = (uint16_t)((1u << (4 * (rem - 12))) - 1);
        _mm512_mask_storeu_epi32(dst + k + 12, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx3, cps));
    }
}
#endif

#if defined(TERM_KERNEL_AVX2)
TERM_TARGET_AVX2
static inline void put_ascii_chunk_avx2(struct cell * restrict dst, const char * restrict src, int n,
                                       uint32_t fg, uint32_t bg, uint32_t flags)
{
    int k = 0;
    __m128i cell = _mm_setr_epi32(0, (int)fg, (int)bg, (int)flags);
    __m256i base = _mm256_broadcastsi128_si256(cell);
#define ASCII_PAIR(o, q) _mm256_storeu_si256((__m256i *)(void *)(dst + (o) + 2 * (q)), \
        _mm256_blend_epi32(_mm256_permutevar8x32_epi32(cps, \
            _mm256_setr_epi32(2*(q), 0, 0, 0, 2*(q)+1, 0, 0, 0)), base, 0xee))
#define ASCII_8(o) do { \
        __m256i cps = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(void *)(src + (o)))); \
        ASCII_PAIR(o, 0); ASCII_PAIR(o, 1); ASCII_PAIR(o, 2); ASCII_PAIR(o, 3); \
    } while (0)
    for (; k + 16 <= n; k += 16) {
        ASCII_8(k);
        ASCII_8(k + 8);
    }
    if (k + 8 <= n) {
        ASCII_8(k);
        k += 8;
    }
    if (k < n && n >= 8) {
        ASCII_8(n - 8);
        k = n;
    }
#undef ASCII_8
#undef ASCII_PAIR
    for (; k < n; k++)
        _mm_storeu_si128((__m128i *)(void *)(dst + k),
            _mm_or_si128(cell, _mm_cvtsi32_si128((unsigned char)src[k])));
}
#endif

TERM_ALWAYS_INLINE
static inline void put_ascii_chunk(struct cell * restrict dst, const char * restrict src, int n,
                                  uint32_t fg, uint32_t bg, uint32_t flags)
{
    /* A short chunk costs less inline than as a call. */
#if defined(TERM_KERNEL_AVX512)
    if (n >= 16 && term_use_avx512()) {
        put_ascii_chunk_avx512(dst, src, n, fg, bg, flags);
        return;
    }
#endif
#if defined(TERM_KERNEL_AVX2)
    if (n >= 16 && term_use_avx2()) {
        put_ascii_chunk_avx2(dst, src, n, fg, bg, flags);
        return;
    }
#endif
    int k = 0;
#if defined(__aarch64__)
    uint32x4_t fg4 = vdupq_n_u32(fg);
    uint32x4_t bg_flags = vcombine_u32(vcreate_u32(((uint64_t)flags << 32) | bg),
                                       vcreate_u32(((uint64_t)flags << 32) | bg));
    for (; k + 8 <= n; k += 8) {
        uint16x8_t cp16 = vmovl_u8(vld1_u8((const uint8_t *)src + k));
        uint32x4_t cp_lo = vmovl_u16(vget_low_u16(cp16));
        uint32x4_t cp_hi = vmovl_u16(vget_high_u16(cp16));

        uint32x4_t c_fg_01 = vzip1q_u32(cp_lo, fg4);
        uint32x4_t c_fg_23 = vzip2q_u32(cp_lo, fg4);
        uint32x4_t c_fg_45 = vzip1q_u32(cp_hi, fg4);
        uint32x4_t c_fg_67 = vzip2q_u32(cp_hi, fg4);

        uint32x4_t cell0 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
        uint32x4_t cell1 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
        uint32x4_t cell2 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
        uint32x4_t cell3 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
        uint32x4_t cell4 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_45, (uint64x2_t)bg_flags);
        uint32x4_t cell5 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_45, (uint64x2_t)bg_flags);
        uint32x4_t cell6 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_67, (uint64x2_t)bg_flags);
        uint32x4_t cell7 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_67, (uint64x2_t)bg_flags);

        uint32x4x4_t q0 = {{ cell0, cell1, cell2, cell3 }};
        uint32x4x4_t q1 = {{ cell4, cell5, cell6, cell7 }};
        vst1q_u32_x4((uint32_t *)(void *)(dst + k + 0), q0);
        vst1q_u32_x4((uint32_t *)(void *)(dst + k + 4), q1);
    }
    for (; k + 4 <= n; k += 4) {
        uint32_t bytes;
        memcpy(&bytes, src + k, sizeof(bytes));
        uint16x4_t cp16 = vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(bytes))));
        uint32x4_t cp_lo = vmovl_u16(cp16);
        uint32x4_t c_fg_01 = vzip1q_u32(cp_lo, fg4);
        uint32x4_t c_fg_23 = vzip2q_u32(cp_lo, fg4);
        uint32x4_t cell0 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
        uint32x4_t cell1 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
        uint32x4_t cell2 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
        uint32x4_t cell3 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
        uint32x4x4_t q0 = {{ cell0, cell1, cell2, cell3 }};
        vst1q_u32_x4((uint32_t *)(void *)(dst + k + 0), q0);
    }
    if (k < n && n >= 4) {
        uint32_t bytes;
        memcpy(&bytes, src + n - 4, sizeof(bytes));
        uint16x4_t cp16 = vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(bytes))));
        uint32x4_t cp_lo = vmovl_u16(cp16);
        uint32x4_t c_fg_01 = vzip1q_u32(cp_lo, fg4);
        uint32x4_t c_fg_23 = vzip2q_u32(cp_lo, fg4);
        uint32x4_t cell0 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
        uint32x4_t cell1 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
        uint32x4_t cell2 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
        uint32x4_t cell3 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
        uint32x4x4_t q0 = {{ cell0, cell1, cell2, cell3 }};
        vst1q_u32_x4((uint32_t *)(void *)(dst + n - 4), q0);
        k = n;
    }
#elif defined(__x86_64__)
    const __m128i zero = _mm_setzero_si128();
    const __m128i fg4 = _mm_set1_epi32((int)fg);
    const __m128i bg_flags = _mm_setr_epi32((int)bg, (int)flags, (int)bg, (int)flags);
#define ASCII_4(o, cp32) do { \
        __m128i c_fg_01 = _mm_unpacklo_epi32((cp32), fg4); \
        __m128i c_fg_23 = _mm_unpackhi_epi32((cp32), fg4); \
        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 0), _mm_unpacklo_epi64(c_fg_01, bg_flags)); \
        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 1), _mm_unpackhi_epi64(c_fg_01, bg_flags)); \
        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 2), _mm_unpacklo_epi64(c_fg_23, bg_flags)); \
        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 3), _mm_unpackhi_epi64(c_fg_23, bg_flags)); \
    } while (0)
    for (; k + 8 <= n; k += 8) {
        __m128i cp16 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(const void *)(src + k)), zero);
        ASCII_4(k, _mm_unpacklo_epi16(cp16, zero));
        ASCII_4(k + 4, _mm_unpackhi_epi16(cp16, zero));
    }
#define ASCII_4_AT(o) do { \
        uint32_t bytes; \
        memcpy(&bytes, src + (o), sizeof(bytes)); \
        __m128i cp16 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)bytes), zero); \
        ASCII_4(o, _mm_unpacklo_epi16(cp16, zero)); \
    } while (0)
    if (k + 4 <= n) {
        ASCII_4_AT(k);
        k += 4;
    }
    if (k < n && n >= 4) {
        ASCII_4_AT(n - 4);
        k = n;
    }
#undef ASCII_4_AT
#undef ASCII_4
#endif
    for (; k < n; k++) {
        dst[k] = (struct cell){
            .c = (unsigned char)src[k],
            .fg = fg,
            .bg = bg,
            .flags = (uint16_t)flags,
            .pad = (uint16_t)(flags >> 16),
        };
    }
}

#endif