Zephyr Project API 4.2.99
A Scalable Open Source RTOS
Loading...
Searching...
No Matches
spinlock.h
Go to the documentation of this file.
1/*
2 * Copyright (c) 2018 Intel Corporation.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
12#ifndef ZEPHYR_INCLUDE_SPINLOCK_H_
13#define ZEPHYR_INCLUDE_SPINLOCK_H_
14
15#include <errno.h>
16#include <stdbool.h>
17
18#include <zephyr/arch/cpu.h>
19#include <zephyr/sys/atomic.h>
20#include <zephyr/sys/__assert.h>
22
23#ifdef __cplusplus
24extern "C" {
25#endif
26
34struct z_spinlock_key {
35 int key;
36};
37
45struct k_spinlock {
49#ifdef CONFIG_SMP
50#ifdef CONFIG_TICKET_SPINLOCKS
51 /*
52 * Ticket spinlocks are conceptually two atomic variables,
53 * one indicating the current FIFO head (spinlock owner),
54 * and the other indicating the current FIFO tail.
55 * Spinlock is acquired in the following manner:
56 * - current FIFO tail value is atomically incremented while it's
57 * original value is saved as a "ticket"
58 * - we spin until the FIFO head becomes equal to the ticket value
59 *
60 * Spinlock is released by atomic increment of the FIFO head
61 */
62 atomic_t owner;
63 atomic_t tail;
64#else
65 atomic_t locked;
66#endif /* CONFIG_TICKET_SPINLOCKS */
67#endif /* CONFIG_SMP */
68
69#ifdef CONFIG_SPIN_VALIDATE
70 /* Stores the thread that holds the lock with the locking CPU
71 * ID in the bottom two bits.
72 */
73 uintptr_t thread_cpu;
74#ifdef CONFIG_SPIN_LOCK_TIME_LIMIT
75 /* Stores the time (in cycles) when a lock was taken
76 */
77 uint32_t lock_time;
78#endif /* CONFIG_SPIN_LOCK_TIME_LIMIT */
79#endif /* CONFIG_SPIN_VALIDATE */
80
81#if defined(CONFIG_CPP) && !defined(CONFIG_SMP) && \
82 !defined(CONFIG_SPIN_VALIDATE)
83 /* If CONFIG_SMP and CONFIG_SPIN_VALIDATE are both not defined
84 * the k_spinlock struct will have no members. The result
85 * is that in C sizeof(k_spinlock) is 0 and in C++ it is 1.
86 *
87 * This size difference causes problems when the k_spinlock
88 * is embedded into another struct like k_msgq, because C and
89 * C++ will have different ideas on the offsets of the members
90 * that come after the k_spinlock member.
91 *
92 * To prevent this we add a 1 byte dummy member to k_spinlock
93 * when the user selects C++ support and k_spinlock would
94 * otherwise be empty.
95 */
96 char dummy;
97#endif
101};
102
103/* There's a spinlock validation framework available when asserts are
104 * enabled. It adds a relatively hefty overhead (about 3k or so) to
105 * kernel code size, don't use on platforms known to be small.
106 */
107#ifdef CONFIG_SPIN_VALIDATE
108bool z_spin_lock_valid(struct k_spinlock *l);
109bool z_spin_unlock_valid(struct k_spinlock *l);
110void z_spin_lock_set_owner(struct k_spinlock *l);
111BUILD_ASSERT(CONFIG_MP_MAX_NUM_CPUS <= 4, "Too many CPUs for mask");
112
113# ifdef CONFIG_KERNEL_COHERENCE
114bool z_spin_lock_mem_coherent(struct k_spinlock *l);
115# endif /* CONFIG_KERNEL_COHERENCE */
116
117#endif /* CONFIG_SPIN_VALIDATE */
118
130typedef struct z_spinlock_key k_spinlock_key_t;
131
132static ALWAYS_INLINE void z_spinlock_validate_pre(struct k_spinlock *l)
133{
134 ARG_UNUSED(l);
135#ifdef CONFIG_SPIN_VALIDATE
136 __ASSERT(z_spin_lock_valid(l), "Invalid spinlock %p", l);
137#ifdef CONFIG_KERNEL_COHERENCE
138 __ASSERT_NO_MSG(z_spin_lock_mem_coherent(l));
139#endif
140#endif
141}
142
143static ALWAYS_INLINE void z_spinlock_validate_post(struct k_spinlock *l)
144{
145 ARG_UNUSED(l);
146#ifdef CONFIG_SPIN_VALIDATE
147 z_spin_lock_set_owner(l);
148#if defined(CONFIG_SPIN_LOCK_TIME_LIMIT) && (CONFIG_SPIN_LOCK_TIME_LIMIT != 0)
149 l->lock_time = sys_clock_cycle_get_32();
150#endif /* CONFIG_SPIN_LOCK_TIME_LIMIT */
151#endif /* CONFIG_SPIN_VALIDATE */
152}
153
186{
187 ARG_UNUSED(l);
189
190 /* Note that we need to use the underlying arch-specific lock
191 * implementation. The "irq_lock()" API in SMP context is
192 * actually a wrapper for a global spinlock!
193 */
194 k.key = arch_irq_lock();
195
196 z_spinlock_validate_pre(l);
197#ifdef CONFIG_SMP
198#ifdef CONFIG_TICKET_SPINLOCKS
199 /*
200 * Enqueue ourselves to the end of a spinlock waiters queue
201 * receiving a ticket
202 */
203 atomic_val_t ticket = atomic_inc(&l->tail);
204 /* Spin until our ticket is served */
205 while (atomic_get(&l->owner) != ticket) {
207 }
208#else
209 while (!atomic_cas(&l->locked, 0, 1)) {
211 }
212#endif /* CONFIG_TICKET_SPINLOCKS */
213#endif /* CONFIG_SMP */
214 z_spinlock_validate_post(l);
215
216 return k;
217}
218
234{
235 int key = arch_irq_lock();
236
237 z_spinlock_validate_pre(l);
238#ifdef CONFIG_SMP
239#ifdef CONFIG_TICKET_SPINLOCKS
240 /*
241 * atomic_get and atomic_cas operations below are not executed
242 * simultaneously.
243 * So in theory k_spin_trylock can lock an already locked spinlock.
244 * To reproduce this the following conditions should be met after we
245 * executed atomic_get and before we executed atomic_cas:
246 *
247 * - spinlock needs to be taken 0xffff_..._ffff + 1 times
248 * (which requires 0xffff_..._ffff number of CPUs, as k_spin_lock call
249 * is blocking) or
250 * - spinlock needs to be taken and released 0xffff_..._ffff times and
251 * then taken again
252 *
253 * In real-life systems this is considered non-reproducible given that
254 * required actions need to be done during this tiny window of several
255 * CPU instructions (which execute with interrupt locked,
256 * so no preemption can happen here)
257 */
258 atomic_val_t ticket_val = atomic_get(&l->owner);
259
260 if (!atomic_cas(&l->tail, ticket_val, ticket_val + 1)) {
261 goto busy;
262 }
263#else
264 if (!atomic_cas(&l->locked, 0, 1)) {
265 goto busy;
266 }
267#endif /* CONFIG_TICKET_SPINLOCKS */
268#endif /* CONFIG_SMP */
269 z_spinlock_validate_post(l);
270
271 k->key = key;
272
273 return 0;
274
275#ifdef CONFIG_SMP
276busy:
277 arch_irq_unlock(key);
278 return -EBUSY;
279#endif /* CONFIG_SMP */
280}
281
305{
306 ARG_UNUSED(l);
307#ifdef CONFIG_SPIN_VALIDATE
308 __ASSERT(z_spin_unlock_valid(l), "Not my spinlock %p", l);
309
310#if defined(CONFIG_SPIN_LOCK_TIME_LIMIT) && (CONFIG_SPIN_LOCK_TIME_LIMIT != 0)
311 uint32_t delta = sys_clock_cycle_get_32() - l->lock_time;
312
313 __ASSERT(delta < CONFIG_SPIN_LOCK_TIME_LIMIT,
314 "Spin lock %p held %u cycles, longer than limit of %u cycles",
315 l, delta, CONFIG_SPIN_LOCK_TIME_LIMIT);
316#endif /* CONFIG_SPIN_LOCK_TIME_LIMIT */
317#endif /* CONFIG_SPIN_VALIDATE */
318
319#ifdef CONFIG_SMP
320#ifdef CONFIG_TICKET_SPINLOCKS
321 /* Give the spinlock to the next CPU in a FIFO */
322 (void)atomic_inc(&l->owner);
323#else
324 /* Strictly we don't need atomic_clear() here (which is an
325 * exchange operation that returns the old value). We are always
326 * setting a zero and (because we hold the lock) know the existing
327 * state won't change due to a race. But some architectures need
328 * a memory barrier when used like this, and we don't have a
329 * Zephyr framework for that.
330 */
331 (void)atomic_clear(&l->locked);
332#endif /* CONFIG_TICKET_SPINLOCKS */
333#endif /* CONFIG_SMP */
334 arch_irq_unlock(key.key);
335}
336
341#if defined(CONFIG_SMP) && defined(CONFIG_TEST)
342/*
343 * @brief Checks if spinlock is held by some CPU, including the local CPU.
344 * This API shouldn't be used outside the tests for spinlock
345 *
346 * @param l A pointer to the spinlock
347 * @retval true - if spinlock is held by some CPU; false - otherwise
348 */
349static ALWAYS_INLINE bool z_spin_is_locked(struct k_spinlock *l)
350{
351#ifdef CONFIG_TICKET_SPINLOCKS
352 atomic_val_t ticket_val = atomic_get(&l->owner);
353
354 return !atomic_cas(&l->tail, ticket_val, ticket_val);
355#else
356 return l->locked;
357#endif /* CONFIG_TICKET_SPINLOCKS */
358}
359#endif /* defined(CONFIG_SMP) && defined(CONFIG_TEST) */
360
361/* Internal function: releases the lock, but leaves local interrupts disabled */
362static ALWAYS_INLINE void k_spin_release(struct k_spinlock *l)
363{
364 ARG_UNUSED(l);
365#ifdef CONFIG_SPIN_VALIDATE
366 __ASSERT(z_spin_unlock_valid(l), "Not my spinlock %p", l);
367#endif
368#ifdef CONFIG_SMP
369#ifdef CONFIG_TICKET_SPINLOCKS
370 (void)atomic_inc(&l->owner);
371#else
372 (void)atomic_clear(&l->locked);
373#endif /* CONFIG_TICKET_SPINLOCKS */
374#endif /* CONFIG_SMP */
375}
376
377#if defined(CONFIG_SPIN_VALIDATE) && defined(__GNUC__)
378static ALWAYS_INLINE void z_spin_onexit(__maybe_unused k_spinlock_key_t *k)
379{
380 __ASSERT(k->key, "K_SPINLOCK exited with goto, break or return, "
381 "use K_SPINLOCK_BREAK instead.");
382}
383#define K_SPINLOCK_ONEXIT __attribute__((__cleanup__(z_spin_onexit)))
384#else
385#define K_SPINLOCK_ONEXIT
386#endif
387
398#define K_SPINLOCK_BREAK continue
399
441#define K_SPINLOCK(lck) \
442 for (k_spinlock_key_t __i K_SPINLOCK_ONEXIT = {}, __key = k_spin_lock(lck); !__i.key; \
443 k_spin_unlock((lck), __key), __i.key = 1)
444
447#ifdef __cplusplus
448}
449#endif
450
451#endif /* ZEPHYR_INCLUDE_SPINLOCK_H_ */
uint32_t sys_clock_cycle_get_32(void)
static ALWAYS_INLINE unsigned int arch_irq_lock(void)
Disable all interrupts on the local CPU.
Definition irq.h:168
static ALWAYS_INLINE void arch_irq_unlock(unsigned int key)
Definition irq.h:176
void arch_spin_relax(void)
Perform architecture specific processing within spin loops.
long atomic_t
Definition atomic_types.h:15
atomic_t atomic_val_t
Definition atomic_types.h:16
System error numbers.
atomic_val_t atomic_get(const atomic_t *target)
Atomic get.
atomic_val_t atomic_clear(atomic_t *target)
Atomic clear.
atomic_val_t atomic_inc(atomic_t *target)
Atomic increment.
bool atomic_cas(atomic_t *target, atomic_val_t old_value, atomic_val_t new_value)
Atomic compare-and-set.
static ALWAYS_INLINE int k_spin_trylock(struct k_spinlock *l, k_spinlock_key_t *k)
Attempt to lock a spinlock.
Definition spinlock.h:233
static ALWAYS_INLINE void k_spin_unlock(struct k_spinlock *l, k_spinlock_key_t key)
Unlock a spin lock.
Definition spinlock.h:303
static ALWAYS_INLINE k_spinlock_key_t k_spin_lock(struct k_spinlock *l)
Lock a spinlock.
Definition spinlock.h:185
struct z_spinlock_key k_spinlock_key_t
Spinlock key type.
Definition spinlock.h:130
#define EBUSY
Mount device busy.
Definition errno.h:54
#define ALWAYS_INLINE
Definition common.h:160
__UINT32_TYPE__ uint32_t
Definition stdint.h:90
__UINTPTR_TYPE__ uintptr_t
Definition stdint.h:105
Kernel Spin Lock.
Definition spinlock.h:45