src/common/utf8.c


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <stdint.h>

bool yasl_utf8_validate(const char *str, size_t len)
{
    const unsigned char *p = (const unsigned char *)str;
    const unsigned char *end = p + len;

    while (p < end) {
        if (*p < 0x80) {
            p++;
        } else if ((*p & 0xe0) == 0xc0) {
            if (p + 1 >= end || (p[1] & 0xc0) != 0x80 || (*p & 0x1e) == 0)
                return false;
            p += 2;
        } else if ((*p & 0xf0) == 0xe0) {
            if (p + 2 >= end || (p[1] & 0xc0) != 0x80 || (p[2] & 0xc0) != 0x80)
                return false;
            uint32_t cp = ((p[0] & 0x0f) << 12) | ((p[1] & 0x3f) << 6) | (p[2] & 0x3f);
            if (cp < 0x800 || (cp >= 0xd800 && cp <= 0xdfff))
                return false;
            p += 3;
        } else if ((*p & 0xf8) == 0xf0) {
            if (p + 3 >= end || (p[1] & 0xc0) != 0x80 || (p[2] & 0xc0) != 0x80 || (p[3] & 0xc0) != 0x80)
                return false;
            uint32_t cp = ((p[0] & 0x07) << 18) | ((p[1] & 0x3f) << 12) | ((p[2] & 0x3f) << 6) | (p[3] & 0x3f);
            if (cp < 0x10000 || cp > 0x10ffff)
                return false;
            p += 4;
        } else {
            return false;
        }
    }
    return true;
}

char *yasl_utf8_sanitize_dup(const char *str)
{
    if (!str) return NULL;
    size_t len = strlen(str);
    char *out = malloc(len + 1);
    if (!out) return NULL;

    const unsigned char *p = (const unsigned char *)str;
    size_t o = 0;
    while (*p) {
        if (*p < 0x80) {
            out[o++] = (char)*p++;
        } else if ((*p & 0xe0) == 0xc0 && (p[1] & 0xc0) == 0x80 && (*p & 0x1e)) {
            out[o++] = (char)*p++;
            out[o++] = (char)*p++;
        } else if ((*p & 0xf0) == 0xe0 && (p[1] & 0xc0) == 0x80 && (p[2] & 0xc0) == 0x80) {
            uint32_t cp = ((p[0] & 0x0f) << 12) | ((p[1] & 0x3f) << 6) | (p[2] & 0x3f);
            if (cp >= 0x800 && !(cp >= 0xd800 && cp <= 0xdfff)) {
                out[o++] = (char)*p++;
                out[o++] = (char)*p++;
                out[o++] = (char)*p++;
            } else {
                out[o++] = '?';
                p++;
            }
        } else if ((*p & 0xf8) == 0xf0 && (p[1] & 0xc0) == 0x80 && (p[2] & 0xc0) == 0x80 && (p[3] & 0xc0) == 0x80) {
            uint32_t cp = ((p[0] & 0x07) << 18) | ((p[1] & 0x3f) << 12) | ((p[2] & 0x3f) << 6) | (p[3] & 0x3f);
            if (cp >= 0x10000 && cp <= 0x10ffff) {
                out[o++] = (char)*p++;
                out[o++] = (char)*p++;
                out[o++] = (char)*p++;
                out[o++] = (char)*p++;
            } else {
                out[o++] = '?';
                p++;
            }
        } else {
            out[o++] = '?';
            p++;
        }
    }
    out[o] = '\0';
    return out;
}