Open Menu / osfmk / machine / atomic_impl.h
Loading...
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
/*
 * Copyright (c) 2018 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/*
 * This header provides some gory details to implement the <machine/atomic.h>
 * interfaces. Nothing in this header should be called directly, no promise is
 * made to keep this interface stable.
 */

#ifndef _MACHINE_ATOMIC_H
#error "Do not include <machine/atomic_impl.h> directly, use <machine/atomic.h>"
#endif

#ifndef _MACHINE_ATOMIC_IMPL_H
#define _MACHINE_ATOMIC_IMPL_H

#include <stdatomic.h>
#include <machine/smp.h>

static inline int
memory_order_has_acquire(enum memory_order ord)
{
	switch (ord) {
	case memory_order_consume:
	case memory_order_acquire:
	case memory_order_acq_rel:
	case memory_order_seq_cst:
		return 1;
	default:
		return 0;
	}
}

static inline int
memory_order_has_release(enum memory_order ord)
{
	switch (ord) {
	case memory_order_release:
	case memory_order_acq_rel:
	case memory_order_seq_cst:
		return 1;
	default:
		return 0;
	}
}

#if __SMP__

#define memory_order_relaxed_smp            memory_order_relaxed
#define memory_order_compiler_acquire_smp   memory_order_relaxed
#define memory_order_compiler_release_smp   memory_order_relaxed
#define memory_order_compiler_acq_rel_smp   memory_order_relaxed
#define memory_order_consume_smp            memory_order_consume
#define memory_order_dependency_smp         memory_order_acquire
#define memory_order_acquire_smp            memory_order_acquire
#define memory_order_release_smp            memory_order_release
#define memory_order_acq_rel_smp            memory_order_acq_rel
#define memory_order_seq_cst_smp            memory_order_seq_cst

#else

#define memory_order_relaxed_smp            memory_order_relaxed
#define memory_order_compiler_acquire_smp   memory_order_relaxed
#define memory_order_compiler_release_smp   memory_order_relaxed
#define memory_order_compiler_acq_rel_smp   memory_order_relaxed
#define memory_order_consume_smp            memory_order_relaxed
#define memory_order_dependency_smp         memory_order_relaxed
#define memory_order_acquire_smp            memory_order_relaxed
#define memory_order_release_smp            memory_order_relaxed
#define memory_order_acq_rel_smp            memory_order_relaxed
#define memory_order_seq_cst_smp            memory_order_relaxed

#endif

/*
 * Hack needed for os_compiler_barrier() to work (including with empty argument)
 */
#define _os_compiler_barrier_relaxed        memory_order_relaxed
#define _os_compiler_barrier_acquire        memory_order_acquire
#define _os_compiler_barrier_release        memory_order_release
#define _os_compiler_barrier_acq_rel        memory_order_acq_rel
#define _os_compiler_barrier_               memory_order_acq_rel

/*
 * Mapping between compiler barrier/memory orders and:
 * - compiler barriers before atomics ("rel_barrier")
 * - compiler barriers after atomics ("acq_barrier")
 */
#define _os_rel_barrier_relaxed             memory_order_relaxed
#define _os_rel_barrier_compiler_acquire    memory_order_relaxed
#define _os_rel_barrier_compiler_release    memory_order_release
#define _os_rel_barrier_compiler_acq_rel    memory_order_release
#define _os_rel_barrier_consume             memory_order_relaxed
#define _os_rel_barrier_dependency          memory_order_relaxed
#define _os_rel_barrier_acquire             memory_order_relaxed
#define _os_rel_barrier_release             memory_order_release
#define _os_rel_barrier_acq_rel             memory_order_release
#define _os_rel_barrier_seq_cst             memory_order_release

#define _os_acq_barrier_relaxed             memory_order_relaxed
#define _os_acq_barrier_compiler_acquire    memory_order_acquire
#define _os_acq_barrier_compiler_release    memory_order_relaxed
#define _os_acq_barrier_compiler_acq_rel    memory_order_acquire
#define _os_acq_barrier_consume             memory_order_acquire
#define _os_acq_barrier_dependency          memory_order_acquire
#define _os_acq_barrier_acquire             memory_order_acquire
#define _os_acq_barrier_release             memory_order_relaxed
#define _os_acq_barrier_acq_rel             memory_order_acquire
#define _os_acq_barrier_seq_cst             memory_order_acquire

#define _os_compiler_barrier_before_atomic(m) \
	        atomic_signal_fence(_os_rel_barrier_##m)
#define _os_compiler_barrier_after_atomic(m) \
	        atomic_signal_fence(_os_acq_barrier_##m)

/*
 * Mapping between compiler barrier/memmory orders and:
 * - memory fences before atomics ("rel_fence")
 * - memory fences after atomics ("acq_fence")
 */
#define _os_rel_fence_relaxed               memory_order_relaxed
#define _os_rel_fence_compiler_acquire      memory_order_relaxed
#define _os_rel_fence_compiler_release      memory_order_release
#define _os_rel_fence_compiler_acq_rel      memory_order_release
#define _os_rel_fence_consume               memory_order_relaxed_smp
#define _os_rel_fence_dependency            memory_order_relaxed_smp
#define _os_rel_fence_acquire               memory_order_relaxed_smp
#define _os_rel_fence_release               memory_order_release_smp
#define _os_rel_fence_acq_rel               memory_order_release_smp
#define _os_rel_fence_seq_cst               memory_order_release_smp

#define _os_acq_fence_relaxed               memory_order_relaxed
#define _os_acq_fence_compiler_acquire      memory_order_relaxed
#define _os_acq_fence_compiler_release      memory_order_relaxed
#define _os_acq_fence_compiler_acq_rel      memory_order_relaxed
#define _os_acq_fence_consume               memory_order_acquire_smp
#define _os_acq_fence_dependency            memory_order_dependency_smp
#define _os_acq_fence_acquire               memory_order_acquire_smp
#define _os_acq_fence_release               memory_order_relaxed_smp
#define _os_acq_fence_acq_rel               memory_order_acquire_smp
#define _os_acq_fence_seq_cst               memory_order_acquire_smp

#define _os_memory_fence_before_atomic(m) \
	        atomic_thread_fence(_os_rel_fence_##m)
#define _os_memory_fence_after_atomic(m) \
	        atomic_thread_fence(_os_acq_fence_##m)

/*
 * Misc. helpers
 */

/*
 * For this implementation, we make sure the compiler cannot coalesce any of the
 * os_atomic calls by casting all atomic variables to `volatile _Atomic`.
 *
 * At the time this decision was taken, clang has been treating all `_Atomic`
 * accesses as if qualified `volatile _Atomic`, so the cast below freezes that
 * aspect of the codegen in time.
 *
 * When/if clang starts coalescing non-volatile _Atomics, we may decide to add
 * coalescing orderings, e.g. {relaxed,acquire,release,acq_rel,seq_cst}_nv.
 */
#define _os_atomic_c11_atomic(p) \
	        ((typeof(*(p)) volatile _Atomic *)(p))

#define _os_atomic_basetypeof(p) \
	        typeof(atomic_load(_os_atomic_c11_atomic(p)))

#define _os_atomic_op_orig(p, v, m, o)  ({ \
	        _os_atomic_basetypeof(p) _r; \
	        _os_compiler_barrier_before_atomic(m); \
	        _r = o(_os_atomic_c11_atomic(p), v, memory_order_##m##_smp); \
	        _os_compiler_barrier_after_atomic(m); \
	        _r; \
})

#define _os_atomic_c11_op_orig(p, v, m, o) \
	        _os_atomic_op_orig(p, v, m, atomic_##o##_explicit)

#define _os_atomic_c11_op(p, v, m, o, op) \
	        ({ typeof(v) _v = (v); _os_atomic_c11_op_orig(p, _v, m, o) op _v; })

#define _os_atomic_clang_op_orig(p, v, m, o) \
	        _os_atomic_op_orig(p, v, m, __atomic_##o)

#define _os_atomic_clang_op(p, v, m, o, op) \
	        ({ typeof(v) _v = (v); _os_atomic_basetypeof(p) _r = \
	                _os_atomic_clang_op_orig(p, _v, m, o); op(_r, _v); })

#define _os_atomic_auto_dependency(e) \
	        _Generic(e, \
	                os_atomic_dependency_t: (e), \
	                default: os_atomic_make_dependency(e))

#endif /* _MACHINE_ATOMIC_IMPL_H */