term: move the term core to subprojects/term

The core now builds, tests, and benchmarks alone, as a Meson subproject
with no dependency on the main tree.

- include/term/ holds term.h and the header-only unicode/. src/ holds
  the core and its private headers. The main tree gets only
  include/term through term_dep, so it cannot reach a private header.
- The term tests, the term benchmarks, the Kitty parser suite, and
  tools/amalgamate.py and tools/generate_unicode.py move with it. The
  tests run as term:<name>.
- The main build passes its -march (arch_flag) and b_ndebug to the
  subproject. A directory configured before this commit needs
  meson configure -Dterm:arch_flag=-march=... once.
- The term tests compile with -UNDEBUG. They do work inside assert.
- The subproject exports term_test_dep with term_api.h, for the main
  tree's render, thumbnail, color, and selection tests.
- The Kitty bridge used struct vector2i and has not compiled since the
  public API redesign. It builds again. 2 of its 14 tests fail: REP
  after a tab, and a cursor report after a resize.
- dirgraph.py drops the term and unicode layers. Its self-test uses pty.
- The symbols test ignores the __odr_asan symbols that ASan adds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

1 files changed, 790 insertions, 0 deletions

diff --git a/src/kernels.h b/src/kernels.h
new file mode 100644
index 0000000..48088b5
--- /dev/null
+++ b/src/kernels.h
@@ -0,0 +1,790 @@
+#ifndef WSAP_TERM_KERNELS_H
+#define WSAP_TERM_KERNELS_H
+
+#include "cell.h"
+#include "compat.h"
+#include "simd.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#if defined(TERM_KERNEL_AVX512)
+TERM_TARGET_AVX512
+static inline size_t scan_printable_span_avx512(const char *buf, size_t len, unsigned char hi)
+{
+    size_t i = 0;
+    __m512i v20_512 = _mm512_set1_epi8(0x20);
+    __m512i vlim_512 = _mm512_set1_epi8((char)(hi - 0x20));
+    for (; i + 64 <= len; i += 64) {
+        __m512i v = _mm512_loadu_si512((const void *)(buf + i));
+        __mmask64 mask = _mm512_cmpge_epu8_mask(_mm512_sub_epi8(v, v20_512), vlim_512);
+        if (mask != 0)
+            return i + term_ctz64(mask);
+    }
+    if (i < len) {
+        __mmask64 load_mask = ((uint64_t)1 << (len - i)) - 1;
+        __m512i v = _mm512_maskz_loadu_epi8(load_mask, buf + i);
+        __mmask64 mask = _mm512_mask_cmpge_epu8_mask(load_mask, _mm512_sub_epi8(v, v20_512), vlim_512);
+        if (mask != 0)
+            return i + term_ctz64(mask);
+    }
+    return len;
+}
+#endif
+
+#if defined(TERM_KERNEL_AVX2)
+TERM_TARGET_AVX2
+static inline size_t scan_printable_span_avx2(const char *buf, size_t len, unsigned char hi)
+{
+    size_t i = 0;
+    const __m256i v_offset = _mm256_set1_epi8((char)(0x20 + 0x80));
+    const __m256i v_limit = _mm256_set1_epi8((char)(hi - 0xa1));
+    for (; i + 64 <= len; i += 64) {
+        __m256i v0 = _mm256_loadu_si256((const __m256i *)(buf + i));
+        __m256i v1 = _mm256_loadu_si256((const __m256i *)(buf + i + 32));
+        __m256i sub0 = _mm256_sub_epi8(v0, v_offset);
+        __m256i sub1 = _mm256_sub_epi8(v1, v_offset);
+        int m0 = _mm256_movemask_epi8(_mm256_cmpgt_epi8(sub0, v_limit));
+        int m1 = _mm256_movemask_epi8(_mm256_cmpgt_epi8(sub1, v_limit));
+        if (TERM_EXPECT((m0 | m1) != 0, 0)) {
+            if (m0 != 0)
+                return i + term_ctz32(m0);
+            return i + 32 + term_ctz32(m1);
+        }
+    }
+    if (i + 32 <= len) {
+        __m256i v = _mm256_loadu_si256((const __m256i *)(buf + i));
+        __m256i sub = _mm256_sub_epi8(v, v_offset);
+        int mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(sub, v_limit));
+        if (mask != 0)
+            return i + term_ctz32(mask);
+        i += 32;
+    }
+    if (i + 16 <= len) {
+        __m128i v = _mm_loadu_si128((const __m128i *)(buf + i));
+        __m128i sub = _mm_sub_epi8(v, _mm_set1_epi8((char)(0x20 + 0x80)));
+        int mask = _mm_movemask_epi8(_mm_cmpgt_epi8(sub, _mm_set1_epi8((char)(hi - 0xa1)))) & 0xFFFF;
+        if (mask != 0)
+            return i + term_ctz32(mask);
+        i += 16;
+    }
+    for (; i < len; i++) {
+        unsigned char c = (unsigned char)buf[i];
+        if (c < 0x20 || c >= hi)
+            return i;
+    }
+    return len;
+}
+#endif
+
+static inline size_t scan_printable_span(const char *buf, size_t len,
+                                         unsigned char hi)
+{
+    /* A short span costs less inline than as a call. */
+#if defined(TERM_KERNEL_AVX512)
+    if (len >= 64 && term_use_avx512())
+        return scan_printable_span_avx512(buf, len, hi);
+#endif
+#if defined(TERM_KERNEL_AVX2)
+    if (len >= 64 && term_use_avx2())
+        return scan_printable_span_avx2(buf, len, hi);
+#endif
+    size_t i = 0;
+#if defined(__aarch64__)
+    uint8x16_t v20 = vdupq_n_u8(0x20);
+    uint8x16_t vlim = vdupq_n_u8((uint8_t)(hi - 0x20));
+    for (; i + 64 <= len; i += 64) {
+        uint8x16_t v0 = vld1q_u8((const uint8_t *)buf + i);
+        uint8x16_t v1 = vld1q_u8((const uint8_t *)buf + i + 16);
+        uint8x16_t v2 = vld1q_u8((const uint8_t *)buf + i + 32);
+        uint8x16_t v3 = vld1q_u8((const uint8_t *)buf + i + 48);
+
+        uint8x16_t m0 = vcgeq_u8(vsubq_u8(v0, v20), vlim);
+        uint8x16_t m1 = vcgeq_u8(vsubq_u8(v1, v20), vlim);
+        uint8x16_t m2 = vcgeq_u8(vsubq_u8(v2, v20), vlim);
+        uint8x16_t m3 = vcgeq_u8(vsubq_u8(v3, v20), vlim);
+
+        uint8x16_t mor = vorrq_u8(vorrq_u8(m0, m1), vorrq_u8(m2, m3));
+        uint64x2_t mor64 = vreinterpretq_u64_u8(mor);
+        if (TERM_EXPECT((vgetq_lane_u64(mor64, 0) | vgetq_lane_u64(mor64, 1)) != 0, 0)) {
+            uint64x2_t m0_64 = vreinterpretq_u64_u8(m0);
+            uint64_t lo0 = vgetq_lane_u64(m0_64, 0), hi0 = vgetq_lane_u64(m0_64, 1);
+            if (lo0) return i + (term_ctz64(lo0) >> 3);
+            if (hi0) return i + 8 + (term_ctz64(hi0) >> 3);
+
+            uint64x2_t m1_64 = vreinterpretq_u64_u8(m1);
+            uint64_t lo1 = vgetq_lane_u64(m1_64, 0), hi1 = vgetq_lane_u64(m1_64, 1);
+            if (lo1) return i + 16 + (term_ctz64(lo1) >> 3);
+            if (hi1) return i + 24 + (term_ctz64(hi1) >> 3);
+
+            uint64x2_t m2_64 = vreinterpretq_u64_u8(m2);
+            uint64_t lo2 = vgetq_lane_u64(m2_64, 0), hi2 = vgetq_lane_u64(m2_64, 1);
+            if (lo2) return i + 32 + (term_ctz64(lo2) >> 3);
+            if (hi2) return i + 40 + (term_ctz64(hi2) >> 3);
+
+            uint64x2_t m3_64 = vreinterpretq_u64_u8(m3);
+            uint64_t lo3 = vgetq_lane_u64(m3_64, 0), hi3 = vgetq_lane_u64(m3_64, 1);
+            if (lo3) return i + 48 + (term_ctz64(lo3) >> 3);
+            if (hi3) return i + 56 + (term_ctz64(hi3) >> 3);
+        }
+    }
+    for (; i + 16 <= len; i += 16) {
+        uint8x16_t v = vld1q_u8((const uint8_t *)buf + i);
+        uint8x16_t mask = vcgeq_u8(vsubq_u8(v, v20), vlim);
+        uint64x2_t m64 = vreinterpretq_u64_u8(mask);
+        uint64_t lo = vgetq_lane_u64(m64, 0);
+        uint64_t hi64 = vgetq_lane_u64(m64, 1);
+        if (lo) return i + (term_ctz64(lo) >> 3);
+        if (hi64) return i + 8 + (term_ctz64(hi64) >> 3);
+    }
+    if (i + 8 <= len) {
+        uint8x8_t v8 = vld1_u8((const uint8_t *)buf + i);
+        uint8x8_t mask8 = vcge_u8(vsub_u8(v8, vget_low_u8(v20)), vget_low_u8(vlim));
+        uint64_t m64 = vget_lane_u64(vreinterpret_u64_u8(mask8), 0);
+        if (m64) return i + (term_ctz64(m64) >> 3);
+        i += 8;
+    }
+#elif defined(__x86_64__)
+    const __m128i v_offset = _mm_set1_epi8((char)(0x20 + 0x80));
+    const __m128i v_limit = _mm_set1_epi8((char)(hi - 0xa1));
+    for (; i + 64 <= len; i += 64) {
+        __m128i m0 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i)), v_offset), v_limit);
+        __m128i m1 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i + 16)), v_offset), v_limit);
+        __m128i m2 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i + 32)), v_offset), v_limit);
+        __m128i m3 = _mm_cmpgt_epi8(_mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i + 48)), v_offset), v_limit);
+        if (TERM_EXPECT(_mm_movemask_epi8(_mm_or_si128(_mm_or_si128(m0, m1), _mm_or_si128(m2, m3))) != 0, 0)) {
+            uint64_t mask = (uint64_t)(unsigned)_mm_movemask_epi8(m0) |
+                            (uint64_t)(unsigned)_mm_movemask_epi8(m1) << 16 |
+                            (uint64_t)(unsigned)_mm_movemask_epi8(m2) << 32 |
+                            (uint64_t)(unsigned)_mm_movemask_epi8(m3) << 48;
+            return i + (size_t)term_ctz64(mask);
+        }
+    }
+    for (; i + 16 <= len; i += 16) {
+        int mask = _mm_movemask_epi8(_mm_cmpgt_epi8(
+            _mm_sub_epi8(_mm_loadu_si128((const __m128i *)(buf + i)), v_offset), v_limit));
+        if (mask != 0)
+            return i + (size_t)term_ctz32((unsigned)mask);
+    }
+#endif
+    for (; i < len; i++) {
+        unsigned char c = (unsigned char)buf[i];
+        if (c < 0x20 || c >= hi)
+            return i;
+    }
+    return len;
+}
+
+/* Copy n bytes with loads that stay inside [src, src + n). Short text runs
+ * are common; a libc call costs more than the copy. */
+static inline TERM_ALWAYS_INLINE void bytes_copy_small(uint8_t *restrict dst,
+                                                                   const char *restrict src, int n)
+{
+    if (TERM_EXPECT((unsigned)(n - 16) <= 16u, 1)) {
+        /* 16 to 32 bytes: two overlapping 16-byte copies. Both loads stay
+         * inside [src, src + n). This is the usual history line length, and
+         * two copies cost less here than the loop below. */
+        memcpy(dst, src, 16);
+        memcpy(dst + n - 16, src + n - 16, 16);
+    } else if (n > 256) {
+        memcpy(dst, src, (size_t)n);
+    } else if (n >= 16) {
+        int k = 0;
+        for (; k + 16 <= n; k += 16)
+            memcpy(dst + k, src + k, 16);
+        if (k < n)
+            memcpy(dst + n - 16, src + n - 16, 16);
+    } else if (n >= 8) {
+        uint64_t a, b;
+        memcpy(&a, src, 8);
+        memcpy(&b, src + n - 8, 8);
+        memcpy(dst, &a, 8);
+        memcpy(dst + n - 8, &b, 8);
+    } else if (n >= 4) {
+        uint32_t a, b;
+        memcpy(&a, src, 4);
+        memcpy(&b, src + n - 4, 4);
+        memcpy(dst, &a, 4);
+        memcpy(dst + n - 4, &b, 4);
+    } else {
+        for (int k = 0; k < n; k++)
+            dst[k] = (uint8_t)src[k];
+    }
+}
+
+static inline size_t scan_crlf_tail(const char *buf, size_t i, size_t len)
+{
+    while (i + 1 < len && buf[i] == '\r' && buf[i + 1] == '\n')
+        i += 2;
+    return i;
+}
+
+#if defined(TERM_KERNEL_AVX512)
+TERM_TARGET_AVX512
+static inline size_t scan_crlf_pairs_avx512(const char *buf, size_t len)
+{
+    size_t i = 0;
+    const __m512i pair = _mm512_set1_epi16(0x0A0D);
+    for (; i + 64 <= len; i += 64) {
+        uint64_t eq = _mm512_cmpeq_epi8_mask(_mm512_loadu_si512((const void *)(buf + i)), pair);
+        if (eq != ~0ull)
+            return i + ((size_t)term_ctz64(~eq) & ~(size_t)1);
+    }
+    return scan_crlf_tail(buf, i, len);
+}
+#endif
+
+#if defined(TERM_KERNEL_AVX2)
+TERM_TARGET_AVX2
+static inline size_t scan_crlf_pairs_avx2(const char *buf, size_t len)
+{
+    size_t i = 0;
+    const __m256i pair = _mm256_set1_epi16(0x0A0D);
+    for (; i + 32 <= len; i += 32) {
+        uint32_t eq = (uint32_t)_mm256_movemask_epi8(
+            _mm256_cmpeq_epi8(_mm256_loadu_si256((const __m256i *)(const void *)(buf + i)), pair));
+        if (eq != ~0u)
+            return i + ((size_t)term_ctz32(~eq) & ~(size_t)1);
+    }
+    return scan_crlf_tail(buf, i, len);
+}
+#endif
+
+/* The length of the prefix of buf made of "\r\n" pairs; always even. */
+static inline size_t scan_crlf_pairs(const char *buf, size_t len)
+{
+#if defined(TERM_KERNEL_AVX512)
+    if (term_use_avx512())
+        return scan_crlf_pairs_avx512(buf, len);
+#endif
+#if defined(TERM_KERNEL_AVX2)
+    if (term_use_avx2())
+        return scan_crlf_pairs_avx2(buf, len);
+#endif
+    size_t i = 0;
+#if defined(__aarch64__)
+    const uint8x16_t pair = vreinterpretq_u8_u16(vdupq_n_u16(0x0A0D));
+    for (; i + 16 <= len; i += 16) {
+        uint8x16_t eq = vceqq_u8(vld1q_u8((const uint8_t *)buf + i), pair);
+        uint16_t bits = term_lane_bits(eq);
+        if (bits != 0xFFFF)
+            return i + ((size_t)term_ctz32((unsigned)~bits & 0xFFFFu) & ~(size_t)1);
+    }
+#endif
+    return scan_crlf_tail(buf, i, len);
+}
+
+#if defined(TERM_KERNEL_AVX512)
+TERM_TARGET_AVX512
+static inline size_t scan_byte_run_avx512(const char *buf, size_t len, char byte)
+{
+    size_t i = 0;
+    const __m512i want = _mm512_set1_epi8(byte);
+    for (; i + 64 <= len; i += 64) {
+        uint64_t eq = _mm512_cmpeq_epi8_mask(_mm512_loadu_si512((const void *)(buf + i)), want);
+        if (eq != ~0ull)
+            return i + (size_t)term_ctz64(~eq);
+    }
+    while (i < len && buf[i] == byte)
+        i++;
+    return i;
+}
+#endif
+
+#if defined(TERM_KERNEL_AVX2)
+TERM_TARGET_AVX2
+static inline size_t scan_byte_run_avx2(const char *buf, size_t len, char byte)
+{
+    size_t i = 0;
+    const __m256i want = _mm256_set1_epi8(byte);
+    for (; i + 32 <= len; i += 32) {
+        uint32_t eq = (uint32_t)_mm256_movemask_epi8(
+            _mm256_cmpeq_epi8(_mm256_loadu_si256((const __m256i *)(const void *)(buf + i)), want));
+        if (eq != ~0u)
+            return i + (size_t)term_ctz32(~eq);
+    }
+    while (i < len && buf[i] == byte)
+        i++;
+    return i;
+}
+#endif
+
+/* The length of the prefix of buf made of byte. */
+static inline size_t scan_byte_run(const char *buf, size_t len, char byte)
+{
+#if defined(TERM_KERNEL_AVX512)
+    if (term_use_avx512())
+        return scan_byte_run_avx512(buf, len, byte);
+#endif
+#if defined(TERM_KERNEL_AVX2)
+    if (term_use_avx2())
+        return scan_byte_run_avx2(buf, len, byte);
+#endif
+    size_t i = 0;
+#if defined(__aarch64__)
+    const uint8x16_t want = vdupq_n_u8((uint8_t)byte);
+    for (; i + 16 <= len; i += 16) {
+        uint16_t bits = term_lane_bits(vceqq_u8(vld1q_u8((const uint8_t *)buf + i), want));
+        if (bits != 0xFFFF)
+            return i + (size_t)term_ctz32((unsigned)~bits & 0xFFFFu);
+    }
+#elif defined(__x86_64__)
+    const __m128i want = _mm_set1_epi8(byte);
+    for (; i + 16 <= len; i += 16) {
+        unsigned eq = (unsigned)_mm_movemask_epi8(
+            _mm_cmpeq_epi8(_mm_loadu_si128((const __m128i *)(const void *)(buf + i)), want));
+        if (eq != 0xFFFFu)
+            return i + (size_t)term_ctz32(~eq & 0xFFFFu);
+    }
+#endif
+    while (i < len && buf[i] == byte)
+        i++;
+    return i;
+}
+
+static inline int scan_ascii_run(const char *buf, int len)
+{
+    return (int)scan_printable_span(buf, (size_t)len, 0x7f);
+}
+
+static inline int scan_ascii_lf_tail(const char *buf, int i, int len, uint16_t *ends, int count)
+{
+    for (; i < len; i++) {
+        unsigned char ch = (unsigned char)buf[i];
+        if (ch == '\n') ends[++count] = (uint16_t)(i + 1);
+        else if (ch - 0x20u >= 0x5fu) break;
+    }
+    return count;
+}
+
+#if defined(TERM_KERNEL_AVX512)
+TERM_TARGET_AVX512 TERM_MAYBE_UNUSED
+static int scan_ascii_lf_avx512(const char *buf, int len, uint16_t *ends)
+{
+    int i = 0, count = 0;
+    ends[0] = 0;
+    const __m512i lanes = _mm512_setr_epi32(
+        0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c,
+        0x13121110,0x17161514,0x1b1a1918,0x1f1e1d1c,
+        0x23222120,0x27262524,0x2b2a2928,0x2f2e2d2c,
+        0x33323130,0x37363534,0x3b3a3938,0x3f3e3d3c);
+    __m128i block = _mm_set1_epi16(1);
+    for (; i + 256 <= len; i += 256) {
+        __m512i v[4];
+        uint64_t nl[4];
+        uint64_t bad = 0;
+        for (int q = 0; q < 4; q++) {
+            v[q] = _mm512_loadu_si512(buf + i + q * 64);
+            nl[q] = _mm512_cmpeq_epi8_mask(v[q], _mm512_set1_epi8('\n'));
+            bad |= _mm512_cmpge_epu8_mask(_mm512_sub_epi8(v[q], _mm512_set1_epi8(0x20)),
+                                         _mm512_set1_epi8(0x5f)) & ~nl[q];
+        }
+        if (bad) break;
+        for (int q = 0; q < 4; q++) {
+            unsigned n = (unsigned)term_popcount64(nl[q]);
+            __m512i packed = _mm512_maskz_compress_epi8(nl[q], lanes);
+            __m128i offsets = _mm_add_epi16(_mm_cvtepu8_epi16(_mm512_castsi512_si128(packed)), block);
+            _mm_storeu_si128((__m128i *)(ends + count + 1), offsets);
+            if (TERM_EXPECT(n > 8, 0)) {
+                __m512i base = _mm512_broadcast_i32x4(block);
+                __m512i wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_castsi512_si256(packed)), base);
+                _mm512_storeu_si512(ends + count + 1, wide);
+                if (n > 32) {
+                    wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(packed, 1)), base);
+                    _mm512_storeu_si512(ends + count + 33, wide);
+                }
+            }
+            count += (int)n;
+            block = _mm_add_epi16(block, _mm_set1_epi16(64));
+        }
+    }
+    for (; i + 64 <= len; i += 64) {
+        __m512i v = _mm512_loadu_si512(buf + i);
+        uint64_t nl = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\n'));
+        uint64_t bad = _mm512_cmpge_epu8_mask(
+            _mm512_sub_epi8(v, _mm512_set1_epi8(0x20)), _mm512_set1_epi8(0x5f)) & ~nl;
+        if (bad) break;
+        unsigned n = (unsigned)term_popcount64(nl);
+        __m512i packed = _mm512_maskz_compress_epi8(nl, lanes);
+        __m128i offsets = _mm_add_epi16(_mm_cvtepu8_epi16(_mm512_castsi512_si128(packed)), block);
+        _mm_storeu_si128((__m128i *)(ends + count + 1), offsets);
+        if (TERM_EXPECT(n > 8, 0)) {
+            __m512i base = _mm512_broadcast_i32x4(block);
+            __m512i wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_castsi512_si256(packed)), base);
+            _mm512_storeu_si512(ends + count + 1, wide);
+            if (n > 32) {
+                wide = _mm512_add_epi16(_mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(packed, 1)), base);
+                _mm512_storeu_si512(ends + count + 33, wide);
+            }
+        }
+        count += (int)n;
+        block = _mm_add_epi16(block, _mm_set1_epi16(64));
+    }
+    return scan_ascii_lf_tail(buf, i, len, ends, count);
+}
+#endif
+
+#if defined(TERM_KERNEL_AVX2)
+TERM_TARGET_AVX2 TERM_MAYBE_UNUSED
+static int scan_ascii_lf_avx2(const char *buf, int len, uint16_t *ends)
+{
+    int i = 0, count = 0;
+    ends[0] = 0;
+    for (; i + 32 <= len; i += 32) {
+        __m256i v = _mm256_loadu_si256((const __m256i *)(buf + i));
+        uint32_t nl = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\n')));
+        __m256i d = _mm256_sub_epi8(v, _mm256_set1_epi8((char)0xa0));
+        uint32_t bad = (uint32_t)_mm256_movemask_epi8(
+            _mm256_cmpgt_epi8(d, _mm256_set1_epi8((char)0xde))) & ~nl;
+        if (bad) break;
+        while (nl) {
+            ends[++count] = (uint16_t)(i + term_ctz32(nl) + 1);
+            nl &= nl - 1;
+        }
+    }
+    return scan_ascii_lf_tail(buf, i, len, ends, count);
+}
+#endif
+
+/* Find complete ASCII lines. Stop at the first byte other than text or LF.
+ * The caller supplies len + 65 entries. len must fit in uint16_t. */
+static TERM_NOINLINE TERM_MAYBE_UNUSED int scan_ascii_lf(const char *buf, int len, uint16_t *ends)
+{
+#if defined(TERM_KERNEL_AVX512)
+    if (term_use_avx512())
+        return scan_ascii_lf_avx512(buf, len, ends);
+#endif
+#if defined(TERM_KERNEL_AVX2)
+    if (term_use_avx2())
+        return scan_ascii_lf_avx2(buf, len, ends);
+#endif
+    int i = 0, count = 0;
+    ends[0] = 0;
+#if defined(__aarch64__)
+    for (; i + 32 <= len; i += 32) {
+        uint32_t nl = 0, bad = 0;
+        for (int q = 0; q < 2; q++) {
+            uint8x16_t v = vld1q_u8((const uint8_t *)buf + i + q * 16);
+            nl |= (uint32_t)term_lane_bits(vceqq_u8(v, vdupq_n_u8('\n'))) << (q * 16);
+            bad |= (uint32_t)term_lane_bits(vcgeq_u8(vsubq_u8(v, vdupq_n_u8(0x20)),
+                                                    vdupq_n_u8(0x5f))) << (q * 16);
+        }
+        bad &= ~nl;
+        if (bad) break;
+        while (nl) {
+            ends[++count] = (uint16_t)(i + term_ctz32(nl) + 1);
+            nl &= nl - 1;
+        }
+    }
+#endif
+    return scan_ascii_lf_tail(buf, i, len, ends, count);
+}
+
+#if defined(TERM_KERNEL_AVX512)
+TERM_TARGET_AVX512
+static inline void fill_cells_avx512(struct cell * restrict row, int cols, uint32_t fg, uint32_t bg)
+{
+    __m128i cell = _mm_setr_epi32(' ', (int)fg, (int)bg, 0);
+    __m512i blank = _mm512_broadcast_i32x4(cell);
+    int c = 0;
+    for (; c + 8 <= cols; c += 8) {
+        _mm512_storeu_si512((void *)(row + c), blank);
+        _mm512_storeu_si512((void *)(row + c + 4), blank);
+    }
+    int rem = cols - c;
+    if (rem >= 4) {
+        _mm512_storeu_si512((void *)(row + c), blank);
+        c += 4;
+        rem -= 4;
+    }
+    if (rem > 0) {
+        uint16_t store_mask = (uint16_t)((1u << (4 * rem)) - 1);
+        _mm512_mask_storeu_epi32((void *)(row + c), store_mask, blank);
+    }
+}
+#endif
+
+#if defined(TERM_KERNEL_AVX2)
+TERM_TARGET_AVX2
+static inline void fill_cells_avx2(struct cell * restrict row, int cols, uint32_t fg, uint32_t bg)
+{
+    struct cell fill = { .c = ' ', .fg = fg, .bg = bg, .flags = 0, .pad = 0 };
+    __m256i blank = _mm256_setr_epi32(' ', (int)fg, (int)bg, 0, ' ', (int)fg, (int)bg, 0);
+    int c = 0;
+    for (; c + 8 <= cols; c += 8) {
+        char *d = (char *)(row + c);
+        _mm256_storeu_si256((__m256i *)(d), blank);
+        _mm256_storeu_si256((__m256i *)(d + 32), blank);
+        _mm256_storeu_si256((__m256i *)(d + 64), blank);
+        _mm256_storeu_si256((__m256i *)(d + 96), blank);
+    }
+    for (; c < cols; c++) {
+        row[c] = fill;
+    }
+}
+#endif
+
+static inline void fill_cells(struct cell * restrict row, int cols, uint32_t fg, uint32_t bg)
+{
+    struct cell fill = { .c = ' ', .fg = fg, .bg = bg, .flags = 0, .pad = 0 };
+#if defined(TERM_KERNEL_AVX512)
+    if (term_use_avx512()) {
+        if (TERM_EXPECT(cols > 0, 1))
+            fill_cells_avx512(row, cols, fg, bg);
+        return;
+    }
+#endif
+#if defined(TERM_KERNEL_AVX2)
+    if (term_use_avx2() && TERM_EXPECT(cols >= 8, 1)) {
+        fill_cells_avx2(row, cols, fg, bg);
+        return;
+    }
+#endif
+#if defined(__aarch64__)
+    if (TERM_EXPECT(cols > 0, 1)) {
+        uint32x4_t blank = { ' ', fg, bg, 0 };
+        uint32x4x4_t b4 = {{ blank, blank, blank, blank }};
+        int c = 0;
+        for (; c + 8 <= cols; c += 8) {
+            vst1q_u32_x4((uint32_t *)(void *)(row + c), b4);
+            vst1q_u32_x4((uint32_t *)(void *)(row + c + 4), b4);
+        }
+        for (; c + 4 <= cols; c += 4) {
+            vst1q_u32_x4((uint32_t *)(void *)(row + c), b4);
+        }
+        for (; c < cols; c++) {
+            vst1q_u32((uint32_t *)(void *)(row + c), blank);
+        }
+        return;
+    }
+#elif defined(__x86_64__)
+    {
+        __m128i blank = _mm_setr_epi32(' ', (int)fg, (int)bg, 0);
+        int c = 0;
+        for (; c + 4 <= cols; c += 4) {
+            _mm_storeu_si128((__m128i *)(void *)(row + c), blank);
+            _mm_storeu_si128((__m128i *)(void *)(row + c + 1), blank);
+            _mm_storeu_si128((__m128i *)(void *)(row + c + 2), blank);
+            _mm_storeu_si128((__m128i *)(void *)(row + c + 3), blank);
+        }
+        for (; c < cols; c++)
+            _mm_storeu_si128((__m128i *)(void *)(row + c), blank);
+        return;
+    }
+#endif
+    for (int c = 0; c < cols; c++) {
+        row[c] = fill;
+    }
+}
+
+static inline void clear_cells_default(struct cell *row, int cols)
+{
+    fill_cells(row, cols, TERM_COLOR_FG, TERM_COLOR_BG);
+}
+
+#if defined(TERM_KERNEL_AVX512)
+TERM_TARGET_AVX512
+static inline void put_ascii_chunk_avx512(struct cell * restrict dst, const char * restrict src, int n,
+                                         uint32_t fg, uint32_t bg, uint32_t flags)
+{
+    int k = 0;
+    __m128i cell = _mm_setr_epi32(0, (int)fg, (int)bg, (int)flags);
+    __m512i base = _mm512_broadcast_i32x4(cell);
+    const __m512i idx0 = _mm512_setr_epi32(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0);
+    const __m512i idx1 = _mm512_setr_epi32(4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0);
+    const __m512i idx2 = _mm512_setr_epi32(8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0);
+    const __m512i idx3 = _mm512_setr_epi32(12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0);
+
+    for (; k + 16 <= n; k += 16) {
+        __m512i cps = _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i *)(src + k)));
+        _mm512_storeu_si512(dst + k + 0, _mm512_mask_permutexvar_epi32(base, 0x1111, idx0, cps));
+        _mm512_storeu_si512(dst + k + 4, _mm512_mask_permutexvar_epi32(base, 0x1111, idx1, cps));
+        _mm512_storeu_si512(dst + k + 8, _mm512_mask_permutexvar_epi32(base, 0x1111, idx2, cps));
+        _mm512_storeu_si512(dst + k + 12, _mm512_mask_permutexvar_epi32(base, 0x1111, idx3, cps));
+    }
+    int rem = n - k;
+    if (rem > 0) {
+        __m128i v = _mm_maskz_loadu_epi8((1 << rem) - 1, src + k);
+        __m512i cps = _mm512_cvtepu8_epi32(v);
+        if (rem <= 4) {
+            uint16_t m = (uint16_t)((1u << (4 * rem)) - 1);
+            _mm512_mask_storeu_epi32(dst + k, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx0, cps));
+            return;
+        }
+        _mm512_storeu_si512(dst + k, _mm512_mask_permutexvar_epi32(base, 0x1111, idx0, cps));
+        if (rem <= 8) {
+            uint16_t m = (uint16_t)((1u << (4 * (rem - 4))) - 1);
+            _mm512_mask_storeu_epi32(dst + k + 4, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx1, cps));
+            return;
+        }
+        _mm512_storeu_si512(dst + k + 4, _mm512_mask_permutexvar_epi32(base, 0x1111, idx1, cps));
+        if (rem <= 12) {
+            uint16_t m = (uint16_t)((1u << (4 * (rem - 8))) - 1);
+            _mm512_mask_storeu_epi32(dst + k + 8, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx2, cps));
+            return;
+        }
+        _mm512_storeu_si512(dst + k + 8, _mm512_mask_permutexvar_epi32(base, 0x1111, idx2, cps));
+        uint16_t m = (uint16_t)((1u << (4 * (rem - 12))) - 1);
+        _mm512_mask_storeu_epi32(dst + k + 12, m, _mm512_mask_permutexvar_epi32(base, 0x1111, idx3, cps));
+    }
+}
+#endif
+
+#if defined(TERM_KERNEL_AVX2)
+TERM_TARGET_AVX2
+static inline void put_ascii_chunk_avx2(struct cell * restrict dst, const char * restrict src, int n,
+                                       uint32_t fg, uint32_t bg, uint32_t flags)
+{
+    int k = 0;
+    __m128i cell = _mm_setr_epi32(0, (int)fg, (int)bg, (int)flags);
+    __m256i base = _mm256_broadcastsi128_si256(cell);
+#define ASCII_PAIR(o, q) _mm256_storeu_si256((__m256i *)(void *)(dst + (o) + 2 * (q)), \
+        _mm256_blend_epi32(_mm256_permutevar8x32_epi32(cps, \
+            _mm256_setr_epi32(2*(q), 0, 0, 0, 2*(q)+1, 0, 0, 0)), base, 0xee))
+#define ASCII_8(o) do { \
+        __m256i cps = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(void *)(src + (o)))); \
+        ASCII_PAIR(o, 0); ASCII_PAIR(o, 1); ASCII_PAIR(o, 2); ASCII_PAIR(o, 3); \
+    } while (0)
+    for (; k + 16 <= n; k += 16) {
+        ASCII_8(k);
+        ASCII_8(k + 8);
+    }
+    if (k + 8 <= n) {
+        ASCII_8(k);
+        k += 8;
+    }
+    if (k < n && n >= 8) {
+        ASCII_8(n - 8);
+        k = n;
+    }
+#undef ASCII_8
+#undef ASCII_PAIR
+    for (; k < n; k++)
+        _mm_storeu_si128((__m128i *)(void *)(dst + k),
+            _mm_or_si128(cell, _mm_cvtsi32_si128((unsigned char)src[k])));
+}
+#endif
+
+TERM_ALWAYS_INLINE
+static inline void put_ascii_chunk(struct cell * restrict dst, const char * restrict src, int n,
+                                  uint32_t fg, uint32_t bg, uint32_t flags)
+{
+    /* A short chunk costs less inline than as a call. */
+#if defined(TERM_KERNEL_AVX512)
+    if (n >= 16 && term_use_avx512()) {
+        put_ascii_chunk_avx512(dst, src, n, fg, bg, flags);
+        return;
+    }
+#endif
+#if defined(TERM_KERNEL_AVX2)
+    if (n >= 16 && term_use_avx2()) {
+        put_ascii_chunk_avx2(dst, src, n, fg, bg, flags);
+        return;
+    }
+#endif
+    int k = 0;
+#if defined(__aarch64__)
+    uint32x4_t fg4 = vdupq_n_u32(fg);
+    uint32x4_t bg_flags = vcombine_u32(vcreate_u32(((uint64_t)flags << 32) | bg),
+                                       vcreate_u32(((uint64_t)flags << 32) | bg));
+    for (; k + 8 <= n; k += 8) {
+        uint16x8_t cp16 = vmovl_u8(vld1_u8((const uint8_t *)src + k));
+        uint32x4_t cp_lo = vmovl_u16(vget_low_u16(cp16));
+        uint32x4_t cp_hi = vmovl_u16(vget_high_u16(cp16));
+
+        uint32x4_t c_fg_01 = vzip1q_u32(cp_lo, fg4);
+        uint32x4_t c_fg_23 = vzip2q_u32(cp_lo, fg4);
+        uint32x4_t c_fg_45 = vzip1q_u32(cp_hi, fg4);
+        uint32x4_t c_fg_67 = vzip2q_u32(cp_hi, fg4);
+
+        uint32x4_t cell0 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
+        uint32x4_t cell1 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
+        uint32x4_t cell2 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
+        uint32x4_t cell3 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
+        uint32x4_t cell4 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_45, (uint64x2_t)bg_flags);
+        uint32x4_t cell5 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_45, (uint64x2_t)bg_flags);
+        uint32x4_t cell6 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_67, (uint64x2_t)bg_flags);
+        uint32x4_t cell7 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_67, (uint64x2_t)bg_flags);
+
+        uint32x4x4_t q0 = {{ cell0, cell1, cell2, cell3 }};
+        uint32x4x4_t q1 = {{ cell4, cell5, cell6, cell7 }};
+        vst1q_u32_x4((uint32_t *)(void *)(dst + k + 0), q0);
+        vst1q_u32_x4((uint32_t *)(void *)(dst + k + 4), q1);
+    }
+    for (; k + 4 <= n; k += 4) {
+        uint32_t bytes;
+        memcpy(&bytes, src + k, sizeof(bytes));
+        uint16x4_t cp16 = vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(bytes))));
+        uint32x4_t cp_lo = vmovl_u16(cp16);
+        uint32x4_t c_fg_01 = vzip1q_u32(cp_lo, fg4);
+        uint32x4_t c_fg_23 = vzip2q_u32(cp_lo, fg4);
+        uint32x4_t cell0 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
+        uint32x4_t cell1 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
+        uint32x4_t cell2 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
+        uint32x4_t cell3 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
+        uint32x4x4_t q0 = {{ cell0, cell1, cell2, cell3 }};
+        vst1q_u32_x4((uint32_t *)(void *)(dst + k + 0), q0);
+    }
+    if (k < n && n >= 4) {
+        uint32_t bytes;
+        memcpy(&bytes, src + n - 4, sizeof(bytes));
+        uint16x4_t cp16 = vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(bytes))));
+        uint32x4_t cp_lo = vmovl_u16(cp16);
+        uint32x4_t c_fg_01 = vzip1q_u32(cp_lo, fg4);
+        uint32x4_t c_fg_23 = vzip2q_u32(cp_lo, fg4);
+        uint32x4_t cell0 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
+        uint32x4_t cell1 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_01, (uint64x2_t)bg_flags);
+        uint32x4_t cell2 = (uint32x4_t)vzip1q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
+        uint32x4_t cell3 = (uint32x4_t)vzip2q_u64((uint64x2_t)c_fg_23, (uint64x2_t)bg_flags);
+        uint32x4x4_t q0 = {{ cell0, cell1, cell2, cell3 }};
+        vst1q_u32_x4((uint32_t *)(void *)(dst + n - 4), q0);
+        k = n;
+    }
+#elif defined(__x86_64__)
+    const __m128i zero = _mm_setzero_si128();
+    const __m128i fg4 = _mm_set1_epi32((int)fg);
+    const __m128i bg_flags = _mm_setr_epi32((int)bg, (int)flags, (int)bg, (int)flags);
+#define ASCII_4(o, cp32) do { \
+        __m128i c_fg_01 = _mm_unpacklo_epi32((cp32), fg4); \
+        __m128i c_fg_23 = _mm_unpackhi_epi32((cp32), fg4); \
+        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 0), _mm_unpacklo_epi64(c_fg_01, bg_flags)); \
+        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 1), _mm_unpackhi_epi64(c_fg_01, bg_flags)); \
+        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 2), _mm_unpacklo_epi64(c_fg_23, bg_flags)); \
+        _mm_storeu_si128((__m128i *)(void *)(dst + (o) + 3), _mm_unpackhi_epi64(c_fg_23, bg_flags)); \
+    } while (0)
+    for (; k + 8 <= n; k += 8) {
+        __m128i cp16 = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *)(const void *)(src + k)), zero);
+        ASCII_4(k, _mm_unpacklo_epi16(cp16, zero));
+        ASCII_4(k + 4, _mm_unpackhi_epi16(cp16, zero));
+    }
+#define ASCII_4_AT(o) do { \
+        uint32_t bytes; \
+        memcpy(&bytes, src + (o), sizeof(bytes)); \
+        __m128i cp16 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)bytes), zero); \
+        ASCII_4(o, _mm_unpacklo_epi16(cp16, zero)); \
+    } while (0)
+    if (k + 4 <= n) {
+        ASCII_4_AT(k);
+        k += 4;
+    }
+    if (k < n && n >= 4) {
+        ASCII_4_AT(n - 4);
+        k = n;
+    }
+#undef ASCII_4_AT
+#undef ASCII_4
+#endif
+    for (; k < n; k++) {
+        dst[k] = (struct cell){
+            .c = (unsigned char)src[k],
+            .fg = fg,
+            .bg = bg,
+            .flags = (uint16_t)flags,
+            .pad = (uint16_t)(flags >> 16),
+        };
+    }
+}
+
+#endif