Skip to content

Commit 8bfcc4a

Browse files
daanxdaanx
committed
simplify TLS defines
1 parent b8b66c0 commit 8bfcc4a

1 file changed

Lines changed: 103 additions & 165 deletions

File tree

‎include/mimalloc/prim-tls.h‎

Lines changed: 103 additions & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ terms of the MIT license. A copy of the license can be found in the file
2525
// Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better?
2626
// --------------------------------------------------------------------------
2727

28-
// static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept; // directly read an entry from the thread local storage (or thread control block)
29-
// static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept;
28+
// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage
29+
// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block
30+
// static inline void mi_prim_tls_slot_set(size_t slot, void* value);
3031

31-
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; // get a unique id for a thread
32+
static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread
3233
static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap
3334
static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api
3435
static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap
@@ -50,123 +51,7 @@ static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); //
5051

5152

5253
//-------------------------------------------------------------------
53-
// Access to TLS (thread local storage) slots.
54-
//-------------------------------------------------------------------
55-
56-
// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
57-
// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.
58-
// If you test on another platform and it works please send a PR :-)
59-
// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
60-
//
61-
// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,
62-
// but unfortunately we can not detect support reliably (see issue #883)
63-
#if (defined(_WIN32)) || \
64-
(defined(__GNUC__) && ( \
65-
(defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__) || defined(__riscv))) \
66-
|| (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
67-
|| (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
68-
|| (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
69-
|| (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
70-
))
71-
72-
static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
73-
void* res;
74-
const size_t ofs = (slot*sizeof(void*));
75-
#if defined(_WIN32)
76-
#if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
77-
res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs
78-
#elif _M_IX86 && !defined(_M_ARM64EC)
79-
res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs
80-
#else
81-
res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);
82-
#endif
83-
#elif defined(__i386__)
84-
__asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
85-
#elif defined(__APPLE__) && defined(__x86_64__)
86-
__asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
87-
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
88-
__asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
89-
#elif defined(__x86_64__)
90-
__asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
91-
#elif defined(__arm__)
92-
void** tcb; MI_UNUSED(ofs);
93-
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
94-
res = tcb[slot];
95-
#elif defined(__aarch64__)
96-
void** tcb; MI_UNUSED(ofs);
97-
#if defined(__APPLE__) // M1, issue #343
98-
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
99-
#else
100-
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
101-
#endif
102-
res = tcb[slot];
103-
#elif defined(__riscv)
104-
void** tcb; MI_UNUSED(ofs);
105-
__asm__ volatile ("mv %0, tp" : "=r" (tcb));
106-
res = tcb[slot];
107-
#elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
108-
MI_UNUSED(ofs);
109-
res = pthread_getspecific(slot);
110-
#else
111-
#define MI_HAS_TLS_SLOT 0
112-
MI_UNUSED(ofs);
113-
res = NULL;
114-
#endif
115-
return res;
116-
}
117-
118-
#ifndef MI_HAS_TLS_SLOT
119-
#define MI_HAS_TLS_SLOT 1
120-
#endif
121-
122-
// setting a tls slot is only used with TLS_MODEL_FIXED (which is not used by default on any platform)
123-
static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
124-
const size_t ofs = (slot*sizeof(void*));
125-
#if defined(_WIN32)
126-
((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);
127-
#elif defined(__i386__)
128-
__asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
129-
#elif defined(__APPLE__) && defined(__x86_64__)
130-
__asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
131-
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
132-
__asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
133-
#elif defined(__x86_64__)
134-
__asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
135-
#elif defined(__arm__)
136-
void** tcb; MI_UNUSED(ofs);
137-
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
138-
tcb[slot] = value;
139-
#elif defined(__aarch64__)
140-
void** tcb; MI_UNUSED(ofs);
141-
#if defined(__APPLE__) // M1, issue #343
142-
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
143-
#else
144-
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
145-
#endif
146-
tcb[slot] = value;
147-
#elif defined(__riscv)
148-
void** tcb; MI_UNUSED(ofs);
149-
__asm__ volatile ("mv %0, tp" : "=r" (tcb));
150-
tcb[slot] = value;
151-
#elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
152-
MI_UNUSED(ofs);
153-
pthread_setspecific(slot, value);
154-
#else
155-
MI_UNUSED(ofs); MI_UNUSED(value);
156-
#endif
157-
}
158-
159-
#endif
160-
161-
162-
//-------------------------------------------------------------------
163-
// Get a fast unique thread id.
164-
//
165-
// Getting the thread id should be performant as it is called in the
166-
// fast path of `_mi_free` and we specialize for various platforms as
167-
// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
168-
// We only require _mi_prim_thread_id() to return a unique id
169-
// for each thread (unequal to zero) with the bottom 2 bits clear.
54+
// Access to the thread pointer
17055
//-------------------------------------------------------------------
17156

17257
// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
@@ -187,69 +72,122 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce
18772
#endif
18873
#endif
18974

190-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
191-
192-
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
193-
const mi_threadid_t tid = __mi_prim_thread_id();
194-
mi_assert_internal(tid > 1);
195-
mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?
196-
return tid;
197-
}
198-
199-
// Get a unique id for the current thread.
200-
#if defined(MI_PRIM_THREAD_ID)
201-
202-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
203-
return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
75+
#if defined(MI_PRIM_THREAD_POINTER) // potential user override
76+
static inline void** mi_prim_thread_pointer(void) {
77+
return MI_PRIM_THREAD_POINTER();
20478
}
205-
20679
#elif defined(_WIN32)
207-
208-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
209-
// Windows: works on Intel and ARM in both 32- and 64-bit
210-
return (uintptr_t)NtCurrentTeb();
80+
static inline void** mi_prim_thread_pointer(void) {
81+
return (void**)NtCurrentTeb();
21182
}
212-
21383
#elif MI_USE_BUILTIN_THREAD_POINTER
214-
215-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
216-
// Works on most Unix based platforms with recent compilers
217-
return (uintptr_t)__builtin_thread_pointer();
84+
static inline void** mi_prim_thread_pointer(void) {
85+
return __builtin_thread_pointer();
21886
}
87+
#elif defined(__GNUC__)
88+
#if defined(__aarch64__)
89+
static inline void** mi_prim_thread_pointer(void) {
90+
void** tcb;
91+
#if defined(__APPLE__) // M1, issue rgb(62, 76, 62)
92+
__asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
93+
#else
94+
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
95+
#endif
96+
return tcb;
97+
}
98+
#elif defined(__riscv)
99+
static inline void** mi_prim_thread_pointer(void) {
100+
void** tcb;
101+
__asm__ volatile ("mv %0, tp" : "=r" (tcb));
102+
return tcb;
103+
}
104+
#elif defined(__arm__)
105+
static inline void** mi_prim_thread_pointer(void) {
106+
void** tcb;
107+
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
108+
return tcb;
109+
}
110+
#elif defined(__i386__)
111+
static inline void** mi_prim_thread_pointer(void) {
112+
void** tcb;
113+
__asm__("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS
114+
return tcb;
115+
}
116+
#elif defined(__x86_64__)
117+
static inline void** mi_prim_thread_pointer(void) {
118+
void** tcb;
119+
#if defined(__APPLE__) && defined(__x86_64__)
120+
__asm__("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS
121+
#elif (MI_INTPTR_SIZE==4)
122+
__asm__("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI
123+
#else
124+
__asm__("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS
125+
#endif
126+
return tcb;
127+
}
128+
#else
129+
#define MI_NO_THREAD_POINTER (1)
130+
#endif
131+
#elif MI_USE_PTHREADS && defined(__APPLE__)
132+
static inline void** mi_prim_thread_pointer(void) {
133+
return (void**)pthread_self();
134+
}
135+
#else
136+
#define MI_NO_THREAD_POINTER (1)
137+
#endif
219138

220-
#elif MI_HAS_TLS_SLOT
221-
222-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
223-
#if defined(__BIONIC__)
224-
// issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
225-
// see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
226-
return (uintptr_t)mi_prim_tls_slot(1);
139+
#if !MI_NO_THREAD_POINTER
140+
#define MI_HAS_TLS_SLOT (1)
141+
static inline void* mi_prim_tls_slot(size_t slot) {
142+
#if defined(_WIN32)
143+
#if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
144+
return (void*)__readgsqword((unsigned long)(slot*sizeof(void*))); // direct load at offset from gs
145+
#elif _M_IX86 && !defined(_M_ARM64EC)
146+
return (void*)__readfsdword((unsigned long)(slot*sizeof(void*))); // direct load at offset from fs
147+
#else
148+
return mi_prim_thread_pointer()[slot];
149+
#endif
227150
#else
228-
// in all our other targets, slot 0 is the thread id
229-
// glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
230-
// apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
231-
return (uintptr_t)mi_prim_tls_slot(0);
151+
return mi_prim_thread_pointer()[slot];
232152
#endif
233153
}
234154

235-
#elif defined(MI_USE_PTHREADS) && defined(__APPLE__)
236-
237-
// on macOS, pthread_t is pointer
238-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
239-
return (uintptr_t)((void*)pthread_self());
155+
static inline void mi_prim_tls_slot_set(size_t slot, void* value) {
156+
mi_prim_thread_pointer()[slot] = value;
240157
}
158+
#endif
241159

242-
#else
243-
244-
extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper;
160+
/* ----------------------------------------------------------------------------------------
161+
Get a unique thread-id for a thread
162+
---------------------------------------------------------------------------------------- */
245163

164+
// Get a unique id for the current thread.
165+
#if defined(MI_PRIM_THREAD_ID)
166+
static inline mi_threadid_t __mi_prim_thread_id(void) {
167+
return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
168+
}
169+
#elif !MI_NO_THREAD_POINTER
170+
static inline mi_threadid_t __mi_prim_thread_id(void) {
171+
#if defined(__BIONIC__)
172+
return (mi_threadid_t)mi_prim_tls_slot(1);
173+
#else
174+
return (mi_threadid_t)mi_prim_thread_pointer();
175+
#endif
176+
}
177+
#else
246178
// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
247-
static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
179+
extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper;
180+
static inline mi_threadid_t __mi_prim_thread_id(void) {
248181
return (uintptr_t)&__mi_thread_id_helper;
249182
}
250-
251183
#endif
252184

185+
static inline mi_threadid_t _mi_prim_thread_id(void) {
186+
const mi_threadid_t tid = __mi_prim_thread_id();
187+
mi_assert_internal(tid > MI_THREADID_DETACHED);
188+
mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?
189+
return tid;
190+
}
253191

254192

255193
/* ----------------------------------------------------------------------------------------

0 commit comments

Comments
 (0)