From 4384435941b8c7658856b381d071aee2e0035e89 Mon Sep 17 00:00:00 2001 From: Berke Güzel Date: Mon, 12 Oct 2026 17:49:34 +0300 Subject: term: move the term core to subprojects/term The core now builds, tests, and benchmarks alone, as a Meson subproject with no dependency on the main tree. - include/term/ holds term.h and the header-only unicode/. src/ holds the core and its private headers. The main tree gets only include/term through term_dep, so it cannot reach a private header. - The term tests, the term benchmarks, the Kitty parser suite, and tools/amalgamate.py and tools/generate_unicode.py move with it. The tests run as term:. - The main build passes its -march (arch_flag) and b_ndebug to the subproject. A directory configured before this commit needs meson configure -Dterm:arch_flag=-march=... once. - The term tests compile with -UNDEBUG. They do work inside assert. - The subproject exports term_test_dep with term_api.h, for the main tree's render, thumbnail, color, and selection tests. - The Kitty bridge used struct vector2i and has not compiled since the public API redesign. It builds again. 2 of its 14 tests fail: REP after a tab, and a cursor report after a resize. - dirgraph.py drops the term and unicode layers. Its self-test uses pty. - The symbols test ignores the __odr_asan symbols that ASan adds. Co-Authored-By: Claude Opus 5.5 --- tests/unicode/unicode_test.c | 138 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 tests/unicode/unicode_test.c (limited to 'tests/unicode') diff --git a/tests/unicode/unicode_test.c b/tests/unicode/unicode_test.c new file mode 100644 index 0000000..7fc6e3e --- /dev/null +++ b/tests/unicode/unicode_test.c @@ -0,0 +1,138 @@ +#include "term/unicode/utf8.h" +#include +#include +#include + +static void test_utf8_decode_one(void) +{ + static const struct { const char *bytes; int len; uint32_t cp; } valid[] = { + {"\0", 1, 0}, {"\x7f", 1, 0x7f}, + {"\xc2\x80", 2, 0x80}, {"\xdf\xbf", 2, 0x7ff}, + {"\xe0\xa0\x80", 3, 0x800}, {"\xed\x9f\xbf", 3, 0xd7ff}, + {"\xee\x80\x80", 3, 0xe000}, {"\xef\xbf\xbf", 3, 0xffff}, + {"\xf0\x90\x80\x80", 4, 0x10000}, + {"\xf4\x8f\xbf\xbf", 4, 0x10ffff}, + }; + for (size_t i = 0; i < sizeof(valid) / sizeof(*valid); i++) { + uint32_t cp = 0xdeadbeef; + for (int n = 0; n < valid[i].len; n++) { + int r = utf8_decode_one(valid[i].bytes, n, &cp); + assert(r == 0); + assert(cp == 0xdeadbeef); + } + int r = utf8_decode_one(valid[i].bytes, valid[i].len, &cp); + assert(r == valid[i].len); + assert(cp == valid[i].cp); + char followed[8]; + memcpy(followed, valid[i].bytes, valid[i].len); + memcpy(followed + valid[i].len, "next", 4); + r = utf8_decode_one(followed, valid[i].len + 4, &cp); + assert(r == valid[i].len); + assert(cp == valid[i].cp); + } + static const char *const invalid[] = { + "\x80", "\xbf", "\xc0\x80", "\xc1\xbf", "\xc2X", + "\xe0\x9f\xbf", "\xed\xa0\x80", "\xed\xbf\xbf", "\xe1\x80X", + "\xf0\x8f\xbf\xbf", "\xf4\x90\x80\x80", "\xf5\x80\x80\x80", + "\xf8\x88\x80\x80\x80", "\xff", "\xf1\x80\x80X", + }; + for (size_t i = 0; i < sizeof(invalid) / sizeof(*invalid); i++) { + uint32_t cp = 0xdeadbeef; + int r = utf8_decode_one(invalid[i], strlen(invalid[i]), &cp); + assert(r == 0); + assert(cp == 0xdeadbeef); + } +} + +static void test_scalar_roundtrips(void) +{ + for (uint32_t cp = 0; cp <= 0x10ffff; cp++) { + if (cp >= 0xd800 && cp <= 0xdfff) continue; + char bytes[UTF8_MAX_BYTES]; + int len = utf8_encode(cp, bytes); + uint32_t scalar = 0, streamed = 0, state = UTF8_ACCEPT; + int r = utf8_decode_one(bytes, (size_t)len, &scalar); + assert(r == len); + assert(scalar == cp); + for (int i = 0; i < len; i++) { + utf8_decode(&state, &streamed, (unsigned char)bytes[i]); + if (i + 1 < len) assert(state != UTF8_ACCEPT && state != UTF8_REJECT); + } + assert(state == UTF8_ACCEPT && streamed == cp); + } + static const uint32_t invalid[] = {0xd800, 0xdfff, 0x110000, UINT32_MAX}; + for (size_t i = 0; i < sizeof(invalid) / sizeof(*invalid); i++) { + char bytes[UTF8_MAX_BYTES]; + int len = utf8_encode(invalid[i], bytes); + uint32_t cp = 0; + int r = utf8_decode_one(bytes, len, &cp); + assert(len == 3 && r == len && cp == 0xfffd); + } +} + +static void test_utf8_sanitize(void) +{ + char dst[64]; + + utf8_sanitize(dst, sizeof(dst), "hello \xe2\x82\xac\xf0\x9f\x98\x80"); + assert(!strcmp(dst, "hello \xe2\x82\xac\xf0\x9f\x98\x80")); + + utf8_sanitize(dst, sizeof(dst), "a\xff\x80" "b\xed\xa0\x80" "c\xe2\x82"); + assert(!strcmp(dst, "abc")); + + utf8_sanitize(dst, sizeof(dst), ""); + assert(!strcmp(dst, "")); + utf8_sanitize(dst, sizeof(dst), "\xff\xfe"); + assert(!strcmp(dst, "")); + + utf8_sanitize(dst, 6, "ab\xe2\x82\xac" "cd"); + assert(!strcmp(dst, "ab\xe2\x82\xac")); + utf8_sanitize(dst, 5, "ab\xe2\x82\xac" "cd"); + assert(!strcmp(dst, "abcd")); + memset(dst, 'x', sizeof(dst)); + utf8_sanitize(dst, 4, "abcdef"); + assert(!strcmp(dst, "abc")); + memset(dst, 'x', sizeof(dst)); + utf8_sanitize(dst, 0, "abc"); + assert(dst[0] == 'x'); +} + +/* Each maximal invalid subpart gives U+FFFD. The byte that ends a subpart + * stays unused. */ +static void test_ground_invalid_subparts(void) +{ + uint32_t state = UTF8_ACCEPT, codep = 0, cp = 0xdead; + int n = utf8_decode_ground(&state, &codep, "\xe2\x80\x1b\xac", 4, &cp); + assert(n == 2); + assert(cp == 0xFFFD); + assert(state == UTF8_ACCEPT); + n = utf8_decode_ground(&state, &codep, "\xac", 1, &cp); + assert(n == 1); + assert(cp == 0xFFFD); + assert(state == UTF8_ACCEPT); + n = utf8_decode_ground(&state, &codep, "\xe2\x82z", 3, &cp); + assert(n == 2); + assert(cp == 0xFFFD); + assert(state == UTF8_ACCEPT); + n = utf8_decode_ground(&state, &codep, "\xe2\x82", 2, &cp); + assert(n == 2); + assert(cp == 0); + assert(state != UTF8_ACCEPT); + + state = UTF8_ACCEPT; + codep = 0; + n = utf8_decode_ground(&state, &codep, "\xe2\x82\xac", 3, &cp); + assert(n == 3); + assert(cp == 0x20ac); + assert(state == UTF8_ACCEPT); +} + +int main(void) +{ + test_utf8_decode_one(); + test_scalar_roundtrips(); + test_utf8_sanitize(); + test_ground_invalid_subparts(); + puts("PASS: Unicode codec tests"); + return 0; +} -- cgit v1.2.3