1 files changed, 82 insertions, 0 deletions
diff --git a/src/common/utf8.c b/src/common/utf8.c
new file mode 100644
index 0000000..089fd89
--- /dev/null
+++ b/src/common/utf8.c
@@ -0,0 +1,82 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+bool yasl_utf8_validate(const char *str, size_t len)
+{
+    const unsigned char *p = (const unsigned char *)str;
+    const unsigned char *end = p + len;
+
+    while (p < end) {
+        if (*p < 0x80) {
+            p++;
+        } else if ((*p & 0xe0) == 0xc0) {
+            if (p + 1 >= end || (p[1] & 0xc0) != 0x80 || (*p & 0x1e) == 0)
+                return false;
+            p += 2;
+        } else if ((*p & 0xf0) == 0xe0) {
+            if (p + 2 >= end || (p[1] & 0xc0) != 0x80 || (p[2] & 0xc0) != 0x80)
+                return false;
+            uint32_t cp = ((p[0] & 0x0f) << 12) | ((p[1] & 0x3f) << 6) | (p[2] & 0x3f);
+            if (cp < 0x800 || (cp >= 0xd800 && cp <= 0xdfff))
+                return false;
+            p += 3;
+        } else if ((*p & 0xf8) == 0xf0) {
+            if (p + 3 >= end || (p[1] & 0xc0) != 0x80 || (p[2] & 0xc0) != 0x80 || (p[3] & 0xc0) != 0x80)
+                return false;
+            uint32_t cp = ((p[0] & 0x07) << 18) | ((p[1] & 0x3f) << 12) | ((p[2] & 0x3f) << 6) | (p[3] & 0x3f);
+            if (cp < 0x10000 || cp > 0x10ffff)
+                return false;
+            p += 4;
+        } else {
+            return false;
+        }
+    }
+    return true;
+}
+
+char *yasl_utf8_sanitize_dup(const char *str)
+{
+    if (!str) return NULL;
+    size_t len = strlen(str);
+    char *out = malloc(len + 1);
+    if (!out) return NULL;
+
+    const unsigned char *p = (const unsigned char *)str;
+    size_t o = 0;
+    while (*p) {
+        if (*p < 0x80) {
+            out[o++] = (char)*p++;
+        } else if ((*p & 0xe0) == 0xc0 && (p[1] & 0xc0) == 0x80 && (*p & 0x1e)) {
+            out[o++] = (char)*p++;
+            out[o++] = (char)*p++;
+        } else if ((*p & 0xf0) == 0xe0 && (p[1] & 0xc0) == 0x80 && (p[2] & 0xc0) == 0x80) {
+            uint32_t cp = ((p[0] & 0x0f) << 12) | ((p[1] & 0x3f) << 6) | (p[2] & 0x3f);
+            if (cp >= 0x800 && !(cp >= 0xd800 && cp <= 0xdfff)) {
+                out[o++] = (char)*p++;
+                out[o++] = (char)*p++;
+                out[o++] = (char)*p++;
+            } else {
+                out[o++] = '?';
+                p++;
+            }
+        } else if ((*p & 0xf8) == 0xf0 && (p[1] & 0xc0) == 0x80 && (p[2] & 0xc0) == 0x80 && (p[3] & 0xc0) == 0x80) {
+            uint32_t cp = ((p[0] & 0x07) << 18) | ((p[1] & 0x3f) << 12) | ((p[2] & 0x3f) << 6) | (p[3] & 0x3f);
+            if (cp >= 0x10000 && cp <= 0x10ffff) {
+                out[o++] = (char)*p++;
+                out[o++] = (char)*p++;
+                out[o++] = (char)*p++;
+                out[o++] = (char)*p++;
+            } else {
+                out[o++] = '?';
+                p++;
+            }
+        } else {
+            out[o++] = '?';
+            p++;
+        }
+    }
+    out[o] = '\0';
+    return out;
+}