-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
/
locks.h
184 lines (164 loc) · 4.97 KB
/
locks.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
// This file is a part of Julia. License is MIT: https://julialang.org/license
#ifndef JL_LOCKS_H
#define JL_LOCKS_H
#include "julia_assert.h"
#ifdef __cplusplus
extern "C" {
#endif
// Lock acquire and release primitives
// JL_LOCK and jl_mutex_lock are GC safe points while JL_LOCK_NOGC
// and jl_mutex_lock_nogc are not.
// Always use JL_LOCK unless no one holding the lock can trigger a GC or GC
// safepoint. JL_LOCK_NOGC should only be needed for GC internal locks.
// The JL_LOCK* and JL_UNLOCK* macros are no-op for non-threading build
// while the jl_mutex_* functions are always locking and unlocking the locks.
static inline void jl_mutex_wait(jl_mutex_t *lock, int safepoint)
{
unsigned long self = jl_thread_self();
unsigned long owner = jl_atomic_load_acquire(&lock->owner);
if (owner == self) {
lock->count++;
return;
}
while (1) {
if (owner == 0 &&
jl_atomic_compare_exchange(&lock->owner, 0, self) == 0) {
lock->count = 1;
return;
}
if (safepoint) {
jl_ptls_t ptls = jl_get_ptls_states();
jl_gc_safepoint_(ptls);
}
jl_cpu_pause();
owner = lock->owner;
}
}
static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT
{
#ifndef __clang_analyzer__
// Hide this body from the analyzer, otherwise it complains that we're calling
// a non-safepoint from this function. The 0 arguments guarantees that we do
// not reach the safepoint, but the analyzer can't figure that out
jl_mutex_wait(lock, 0);
#endif
}
static inline void jl_lock_frame_push(jl_mutex_t *lock)
{
jl_ptls_t ptls = jl_get_ptls_states();
// For early bootstrap
if (__unlikely(!ptls->current_task))
return;
arraylist_t *locks = &ptls->current_task->locks;
size_t len = locks->len;
if (__unlikely(len >= locks->max)) {
arraylist_grow(locks, 1);
}
else {
locks->len = len + 1;
}
locks->items[len] = (void*)lock;
}
static inline void jl_lock_frame_pop(void)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (__likely(ptls->current_task)) {
ptls->current_task->locks.len--;
}
}
#define JL_SIGATOMIC_BEGIN() do { \
jl_get_ptls_states()->defer_signal++; \
jl_signal_fence(); \
} while (0)
#define JL_SIGATOMIC_END() do { \
jl_signal_fence(); \
if (--jl_get_ptls_states()->defer_signal == 0) { \
jl_sigint_safepoint(jl_get_ptls_states()); \
} \
} while (0)
static inline void jl_mutex_lock(jl_mutex_t *lock)
{
jl_ptls_t ptls = jl_get_ptls_states();
JL_SIGATOMIC_BEGIN();
jl_mutex_wait(lock, 1);
jl_lock_frame_push(lock);
jl_gc_enable_finalizers(ptls, 0);
}
static inline int jl_mutex_trylock_nogc(jl_mutex_t *lock)
{
unsigned long self = jl_thread_self();
unsigned long owner = jl_atomic_load_acquire(&lock->owner);
if (owner == self) {
lock->count++;
return 1;
}
if (owner == 0 &&
jl_atomic_compare_exchange(&lock->owner, 0, self) == 0) {
lock->count = 1;
return 1;
}
return 0;
}
static inline int jl_mutex_trylock(jl_mutex_t *lock)
{
int got = jl_mutex_trylock_nogc(lock);
if (got) {
jl_ptls_t ptls = jl_get_ptls_states();
JL_SIGATOMIC_BEGIN();
jl_lock_frame_push(lock);
jl_gc_enable_finalizers(ptls, 0);
}
return got;
}
/* Call this function for code that could be called from either a managed
or an unmanaged thread */
static inline void jl_mutex_lock_maybe_nogc(jl_mutex_t *lock)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (ptls->safepoint) {
jl_mutex_lock(lock);
} else {
jl_mutex_lock_nogc(lock);
}
}
static inline void jl_mutex_unlock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT
{
#ifndef __clang_analyzer__
assert(lock->owner == jl_thread_self() &&
"Unlocking a lock in a different thread.");
if (--lock->count == 0) {
jl_atomic_store_release(&lock->owner, 0);
jl_cpu_wake();
}
#endif
}
static inline void jl_mutex_unlock(jl_mutex_t *lock)
{
jl_ptls_t ptls = jl_get_ptls_states();
jl_mutex_unlock_nogc(lock);
jl_gc_enable_finalizers(ptls, 1);
jl_lock_frame_pop();
JL_SIGATOMIC_END();
}
static inline void jl_mutex_unlock_maybe_nogc(jl_mutex_t *lock) {
jl_ptls_t ptls = jl_get_ptls_states();
if (ptls->safepoint) {
jl_mutex_unlock(lock);
} else {
jl_mutex_unlock_nogc(lock);
}
}
static inline void jl_mutex_init(jl_mutex_t *lock) JL_NOTSAFEPOINT
{
lock->owner = 0;
lock->count = 0;
}
#define JL_MUTEX_INIT(m) jl_mutex_init(m)
#define JL_LOCK(m) jl_mutex_lock(m)
#define JL_UNLOCK(m) jl_mutex_unlock(m)
#define JL_LOCK_NOGC(m) jl_mutex_lock_nogc(m)
#define JL_UNLOCK_NOGC(m) jl_mutex_unlock_nogc(m)
#ifdef __cplusplus
}
#endif
#endif