1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
#ifndef TERM_COMPAT_H
#define TERM_COMPAT_H
/*
* Compiler and libc differences, kept in one place. GCC and Clang use their
* builtins. MSVC uses its intrinsics. Any other compiler, and any build with
* TERM_NO_COMPILER_BUILTINS, uses plain C. Plain C gives the same results,
* only slower.
*/
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#if defined(__GNUC__) || defined(__clang__)
#define TERM_COMPILER_GNU 1
#elif defined(_MSC_VER)
#define TERM_COMPILER_MSVC 1
#include <intrin.h>
#endif
#if !defined(TERM_NO_COMPILER_BUILTINS) && defined(TERM_COMPILER_GNU)
#define TERM_HAVE_BUILTINS 1
#elif !defined(TERM_NO_COMPILER_BUILTINS) && defined(TERM_COMPILER_MSVC)
#define TERM_HAVE_MSVC_BITOPS 1
#endif
#if defined(TERM_COMPILER_GNU)
#define TERM_HOT __attribute__((hot))
#define TERM_NOINLINE __attribute__((noinline))
#define TERM_ALWAYS_INLINE __attribute__((always_inline))
#define TERM_MAYBE_UNUSED __attribute__((unused))
#elif defined(TERM_COMPILER_MSVC)
#define TERM_HOT
#define TERM_NOINLINE __declspec(noinline)
#define TERM_ALWAYS_INLINE __forceinline
#define TERM_MAYBE_UNUSED
#else
#define TERM_HOT
#define TERM_NOINLINE
#define TERM_ALWAYS_INLINE
#define TERM_MAYBE_UNUSED
#endif
#if defined(TERM_HAVE_BUILTINS)
#define TERM_EXPECT(x, v) __builtin_expect((x), (v))
#else
#define TERM_EXPECT(x, v) (x)
#endif
/* term_ctz32(0) is 32 and term_ctz64(0) is 64 on every path. */
#if defined(TERM_HAVE_BUILTINS)
static inline int term_ctz32(uint32_t x) { return x ? __builtin_ctz(x) : 32; }
static inline int term_popcount32(uint32_t x) { return __builtin_popcount(x); }
static inline int term_ctz64(uint64_t x) { return x ? __builtin_ctzll(x) : 64; }
static inline int term_popcount64(uint64_t x) { return __builtin_popcountll(x); }
#else
#if defined(TERM_HAVE_MSVC_BITOPS)
static inline int term_ctz32(uint32_t x)
{
unsigned long i;
return _BitScanForward(&i, x) ? (int)i : 32;
}
static inline int term_popcount32(uint32_t x) { return (int)__popcnt(x); }
#else
static inline int term_ctz32(uint32_t x)
{
if (!x)
return 32;
int n = 0;
while (!(x & 1u)) {
x >>= 1;
n++;
}
return n;
}
static inline int term_popcount32(uint32_t x)
{
int n = 0;
for (; x; x &= x - 1)
n++;
return n;
}
#endif
/* Without builtins, the 64-bit forms are built from the 32-bit ones. */
static inline int term_ctz64(uint64_t x)
{
uint32_t lo = (uint32_t)x;
return lo ? term_ctz32(lo) : 32 + term_ctz32((uint32_t)(x >> 32));
}
static inline int term_popcount64(uint64_t x)
{
return term_popcount32((uint32_t)x) + term_popcount32((uint32_t)(x >> 32));
}
#endif
/* A spin-wait hint. Compilers without one get a no-op. */
static inline void term_cpu_relax(void)
{
#if defined(TERM_COMPILER_GNU)
#if defined(TERM_HAVE_BUILTINS) && defined(__x86_64__)
__builtin_ia32_pause();
#elif defined(__aarch64__)
__asm__ __volatile__("yield");
#endif
#elif defined(TERM_COMPILER_MSVC)
#if defined(_M_X64) || defined(_M_IX86)
_mm_pause();
#elif defined(_M_ARM64)
__yield();
#endif
#endif
}
/* strdup is POSIX, so the core copies strings itself. */
static inline char *term_strdup(const char *s)
{
size_t n = strlen(s) + 1;
char *copy = malloc(n);
if (copy)
memcpy(copy, s, n);
return copy;
}
/* ASCII-only prefix test, as strncasecmp() does in the C locale. */
static inline bool term_has_prefix_ci(const char *s, const char *prefix)
{
for (; *prefix; prefix++, s++) {
unsigned char a = (unsigned char)*s;
unsigned char b = (unsigned char)*prefix;
if (a >= 'A' && a <= 'Z')
a += 'a' - 'A';
if (b >= 'A' && b <= 'Z')
b += 'a' - 'A';
if (a != b)
return false;
}
return true;
}
/* Allocates size bytes that start on an align-byte boundary. align must be a
* power of two. The raw pointer is stored just before the block, so
* term_aligned_free() can release it. C11 aligned_alloc() and POSIX
* posix_memalign() are not on every toolchain. */
static inline void *term_aligned_alloc(size_t align, size_t size)
{
if (size > SIZE_MAX - align - sizeof(void *))
return NULL;
void *raw = malloc(size + align + sizeof(void *));
if (!raw)
return NULL;
uintptr_t p = ((uintptr_t)raw + sizeof(void *) + align - 1) & ~((uintptr_t)align - 1);
((void **)p)[-1] = raw;
return (void *)p;
}
static inline void term_aligned_free(void *p)
{
if (p)
free(((void **)p)[-1]);
}
#endif
|
