spinlock.h

Go to the documentation of this file.

/*
 * Copyright (c) 2018 Intel Corporation.
 *
 * SPDX-License-Identifier: Apache-2.0
 */

 
#ifndef ZEPHYR_INCLUDE_SPINLOCK_H_
#define ZEPHYR_INCLUDE_SPINLOCK_H_
 
#include <errno.h>
#include <stdbool.h>
 
#include <zephyr/arch/cpu.h>
#include <zephyr/sys/atomic.h>
#include <zephyr/sys/__assert.h>
#include <zephyr/sys/time_units.h>
 
#ifdef __cplusplus
extern "C" {
#endif

 
struct z_spinlock_key {
        int key;
};

struct k_spinlock {
#ifdef CONFIG_SMP
#ifdef CONFIG_TICKET_SPINLOCKS
        /*
         * Ticket spinlocks are conceptually two atomic variables,
         * one indicating the current FIFO head (spinlock owner),
         * and the other indicating the current FIFO tail.
         * Spinlock is acquired in the following manner:
         * - current FIFO tail value is atomically incremented while it's
         *   original value is saved as a "ticket"
         * - we spin until the FIFO head becomes equal to the ticket value
         *
         * Spinlock is released by atomic increment of the FIFO head
         */
        atomic_t owner;
        atomic_t tail;
#else
        atomic_t locked;
#endif /* CONFIG_TICKET_SPINLOCKS */
#endif /* CONFIG_SMP */
 
#ifdef CONFIG_SPIN_VALIDATE
        /* Stores the thread that holds the lock with the locking CPU
         * ID in the bottom bits (2 bits on 32-bit, 3 bits on 64-bit).
         */
        uintptr_t thread_cpu;
#ifdef CONFIG_SPIN_LOCK_TIME_LIMIT
        /* Stores the time (in cycles) when a lock was taken
         */
        uint32_t lock_time;
#endif /* CONFIG_SPIN_LOCK_TIME_LIMIT */
#endif /* CONFIG_SPIN_VALIDATE */
 
#if defined(CONFIG_NONZERO_SPINLOCK_SIZE) && !defined(CONFIG_SMP) && !defined(CONFIG_SPIN_VALIDATE)
        /* Add a dummy field to guarantee the spinlock has a non-zero
         * size. If neither CONFIG_SMP nor CONFIG_SPIN_VALIDATE are
         * defined then the k_spinlock struct would otherwise have no
         * members and sizeof(k_spinlock) would be 0 in C and 1 in C++.
         *
         * That size difference causes problems when the k_spinlock
         * is embedded into another struct like k_msgq, because C and
         * C++ will have different ideas on the offsets of the members
         * that come after the k_spinlock member.
         */
        char dummy;
#endif
};
 
/* There's a spinlock validation framework available when asserts are
 * enabled.  It adds a relatively hefty overhead (about 3k or so) to
 * kernel code size, don't use on platforms known to be small.
 */
#ifdef CONFIG_SPIN_VALIDATE
bool z_spin_lock_valid(struct k_spinlock *l);
bool z_spin_unlock_valid(struct k_spinlock *l);
void z_spin_lock_set_owner(struct k_spinlock *l);
void z_spin_lock_transfer_owner(struct k_spinlock *l);
void z_assert_can_swap(unsigned int key, struct k_spinlock *swap_lock);
void z_spin_validate_reset(bool lock_held);
extern const uint8_t z_spinlock_abort_sentinel;
#ifdef CONFIG_TEST
struct k_spinlock *z_spin_get_held_lock(void);
#endif /* CONFIG_TEST */
#if defined(CONFIG_64BIT)
BUILD_ASSERT(CONFIG_MP_MAX_NUM_CPUS <= 8, "Too many CPUs for mask");
#else
BUILD_ASSERT(CONFIG_MP_MAX_NUM_CPUS <= 4, "Too many CPUs for mask");
#endif
 
# ifdef CONFIG_KERNEL_COHERENCE
bool z_spin_lock_mem_coherent(struct k_spinlock *l);
# endif /* CONFIG_KERNEL_COHERENCE */
 
#endif /* CONFIG_SPIN_VALIDATE */

typedef struct z_spinlock_key k_spinlock_key_t;
 
static ALWAYS_INLINE void z_spinlock_validate_pre(struct k_spinlock *l)
{
        ARG_UNUSED(l);
#ifdef CONFIG_SPIN_VALIDATE
        __ASSERT(z_spin_lock_valid(l), "Invalid spinlock %p", l);
#ifdef CONFIG_KERNEL_COHERENCE
        __ASSERT_NO_MSG(z_spin_lock_mem_coherent(l));
#endif
#endif
}
 
static ALWAYS_INLINE void z_spinlock_validate_post(struct k_spinlock *l)
{
        ARG_UNUSED(l);
#ifdef CONFIG_SPIN_VALIDATE
        z_spin_lock_set_owner(l);
#if defined(CONFIG_SPIN_LOCK_TIME_LIMIT) && (CONFIG_SPIN_LOCK_TIME_LIMIT != 0)
        l->lock_time = sys_clock_cycle_get_32();
#endif /* CONFIG_SPIN_LOCK_TIME_LIMIT */
#endif /* CONFIG_SPIN_VALIDATE */
}

static ALWAYS_INLINE k_spinlock_key_t k_spin_lock(struct k_spinlock *l)
{
        ARG_UNUSED(l);
        k_spinlock_key_t k;
 
        /* Note that we need to use the underlying arch-specific lock
         * implementation.  The "irq_lock()" API in SMP context is
         * actually a wrapper for a global spinlock!
         */
        k.key = arch_irq_lock();
 
        z_spinlock_validate_pre(l);
#ifdef CONFIG_SMP
#ifdef CONFIG_TICKET_SPINLOCKS
        /*
         * Enqueue ourselves to the end of a spinlock waiters queue
         * receiving a ticket
         */
        atomic_val_t ticket = atomic_inc(&l->tail);
        /* Spin until our ticket is served */
        while (atomic_get(&l->owner) != ticket) {
                arch_spin_relax();
        }
#else
        while (!atomic_cas(&l->locked, 0, 1)) {
                do {
                        arch_spin_relax();
                } while (atomic_get(&l->locked) != 0);
        }
#endif /* CONFIG_TICKET_SPINLOCKS */
#endif /* CONFIG_SMP */
        z_spinlock_validate_post(l);
 
        return k;
}

static ALWAYS_INLINE int k_spin_trylock(struct k_spinlock *l, k_spinlock_key_t *k)
{
        int key = arch_irq_lock();
 
        z_spinlock_validate_pre(l);
#ifdef CONFIG_SMP
#ifdef CONFIG_TICKET_SPINLOCKS
        /*
         * atomic_get and atomic_cas operations below are not executed
         * simultaneously.
         * So in theory k_spin_trylock can lock an already locked spinlock.
         * To reproduce this the following conditions should be met after we
         * executed atomic_get and before we executed atomic_cas:
         *
         * - spinlock needs to be taken 0xffff_..._ffff + 1 times
         * (which requires 0xffff_..._ffff number of CPUs, as k_spin_lock call
         * is blocking) or
         * - spinlock needs to be taken and released 0xffff_..._ffff times and
         * then taken again
         *
         * In real-life systems this is considered non-reproducible given that
         * required actions need to be done during this tiny window of several
         * CPU instructions (which execute with interrupt locked,
         * so no preemption can happen here)
         */
        atomic_val_t ticket_val = atomic_get(&l->owner);
 
        if (!atomic_cas(&l->tail, ticket_val, ticket_val + 1)) {
                goto busy;
        }
#else
        if (!atomic_cas(&l->locked, 0, 1)) {
                goto busy;
        }
#endif /* CONFIG_TICKET_SPINLOCKS */
#endif /* CONFIG_SMP */
        z_spinlock_validate_post(l);
 
        k->key = key;
 
        return 0;
 
#ifdef CONFIG_SMP
busy:
        arch_irq_unlock(key);
        return -EBUSY;
#endif /* CONFIG_SMP */
}

static ALWAYS_INLINE void k_spin_unlock(struct k_spinlock *l,
                                        k_spinlock_key_t key)
{
        ARG_UNUSED(l);
#ifdef CONFIG_SPIN_VALIDATE
        __ASSERT(z_spin_unlock_valid(l), "Not my spinlock %p", l);
 
#if defined(CONFIG_SPIN_LOCK_TIME_LIMIT) && (CONFIG_SPIN_LOCK_TIME_LIMIT != 0)
        uint32_t delta = sys_clock_cycle_get_32() - l->lock_time;
 
        __ASSERT(delta < CONFIG_SPIN_LOCK_TIME_LIMIT,
                 "Spin lock %p held %u cycles, longer than limit of %u cycles",
                 l, delta, CONFIG_SPIN_LOCK_TIME_LIMIT);
#endif /* CONFIG_SPIN_LOCK_TIME_LIMIT */
#endif /* CONFIG_SPIN_VALIDATE */
 
#ifdef CONFIG_SMP
#ifdef CONFIG_TICKET_SPINLOCKS
        /* Give the spinlock to the next CPU in a FIFO */
        (void)atomic_inc(&l->owner);
#else
        /* Strictly we don't need atomic_clear() here (which is an
         * exchange operation that returns the old value).  We are always
         * setting a zero and (because we hold the lock) know the existing
         * state won't change due to a race.  But some architectures need
         * a memory barrier when used like this, and we don't have a
         * Zephyr framework for that.
         */
        (void)atomic_clear(&l->locked);
#endif /* CONFIG_TICKET_SPINLOCKS */
#endif /* CONFIG_SMP */
        arch_irq_unlock(key.key);
}

 
#if defined(CONFIG_TEST) || defined(CONFIG_ASSERT)
/*
 * @brief Checks if spinlock is held by some CPU, including the local CPU.
 *              This should only be used in tests or assertions, not to make
 *              runtime control flow decisions.
 *
 * @param l A pointer to the spinlock
 * @retval true - if spinlock is held by some CPU; false - otherwise
 */
static ALWAYS_INLINE bool z_spin_is_locked(struct k_spinlock *l)
{
#ifdef CONFIG_SMP
#ifdef CONFIG_TICKET_SPINLOCKS
        atomic_val_t ticket_val = atomic_get(&l->owner);
 
        return !atomic_cas(&l->tail, ticket_val, ticket_val);
#else
        return l->locked;
#endif /* CONFIG_TICKET_SPINLOCKS */
#else
        ARG_UNUSED(l);
        /* In UP builds a spinlock reduces to an IRQ lock. */
        return !arch_cpu_irqs_are_enabled();
#endif /* CONFIG_SMP */
}
#endif /* defined(CONFIG_TEST) || defined(CONFIG_ASSERT) */
 
/* Internal function: releases the lock, but leaves local interrupts disabled */
static ALWAYS_INLINE void k_spin_release(struct k_spinlock *l)
{
        ARG_UNUSED(l);
#ifdef CONFIG_SPIN_VALIDATE
        __ASSERT(z_spin_unlock_valid(l), "Not my spinlock %p", l);
#endif
#ifdef CONFIG_SMP
#ifdef CONFIG_TICKET_SPINLOCKS
        (void)atomic_inc(&l->owner);
#else
        (void)atomic_clear(&l->locked);
#endif /* CONFIG_TICKET_SPINLOCKS */
#endif /* CONFIG_SMP */
}
 
#if defined(CONFIG_SPIN_VALIDATE) && defined(__GNUC__)
static ALWAYS_INLINE void z_spin_onexit(__maybe_unused k_spinlock_key_t *k)
{
        __ASSERT(k->key, "K_SPINLOCK exited with goto, break or return, "
                         "use K_SPINLOCK_BREAK instead.");
}
#define K_SPINLOCK_ONEXIT __attribute__((__cleanup__(z_spin_onexit)))
#else
#define K_SPINLOCK_ONEXIT
#endif


#define K_SPINLOCK_BREAK continue

#define K_SPINLOCK(lck)                                                                            \
        for (k_spinlock_key_t __i K_SPINLOCK_ONEXIT = {}, __key = k_spin_lock(lck); !__i.key;      \
             k_spin_unlock((lck), __key), __i.key = 1)

 
#ifdef __cplusplus
}
#endif
 
#endif /* ZEPHYR_INCLUDE_SPINLOCK_H_ */