spin_lock.h

/*
 * Copyright (c) 2024 Raspberry Pi (Trading) Ltd.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */
 
#ifndef _HARDWARE_SYNC_SPIN_LOCK_H
#define _HARDWARE_SYNC_SPIN_LOCK_H
 
#include "pico.h"
#include "hardware/sync.h"
 
// PICO_CONFIG: PICO_USE_SW_SPIN_LOCKS, Use software implementation for spin locks, type=bool, default=1 on RP2350 due to errata, group=hardware_sync
#ifndef PICO_USE_SW_SPIN_LOCKS
#if PICO_RP2350
#define PICO_USE_SW_SPIN_LOCKS 1
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_IRQ, Spinlock ID for IRQ protection, min=0, max=31, default=9, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_IRQ
#define PICO_SPINLOCK_ID_IRQ 9
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_TIMER, Spinlock ID for Timer protection, min=0, max=31, default=10, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_TIMER
#define PICO_SPINLOCK_ID_TIMER 10
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_HARDWARE_CLAIM, Spinlock ID for Hardware claim protection, min=0, max=31, default=11, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_HARDWARE_CLAIM
#define PICO_SPINLOCK_ID_HARDWARE_CLAIM 11
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_RAND, Spinlock ID for Random Number Generator, min=0, max=31, default=12, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_RAND
#define PICO_SPINLOCK_ID_RAND 12
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_ATOMIC, Spinlock ID for atomics, min=0, max=31, default=13, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_ATOMIC
#define PICO_SPINLOCK_ID_ATOMIC 13
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_OS1, First Spinlock ID reserved for use by low level OS style software, min=0, max=31, default=14, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_OS1
#define PICO_SPINLOCK_ID_OS1 14
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_OS2, Second Spinlock ID reserved for use by low level OS style software, min=0, max=31, default=15, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_OS2
#define PICO_SPINLOCK_ID_OS2 15
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_STRIPED_FIRST, Lowest Spinlock ID in the 'striped' range, min=0, max=31, default=16, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_STRIPED_FIRST
#define PICO_SPINLOCK_ID_STRIPED_FIRST 16
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_STRIPED_LAST, Highest Spinlock ID in the 'striped' range, min=0, max=31, default=23, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_STRIPED_LAST
#define PICO_SPINLOCK_ID_STRIPED_LAST 23
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_CLAIM_FREE_FIRST, Lowest Spinlock ID in the 'claim free' range, min=0, max=31, default=24, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_CLAIM_FREE_FIRST
#define PICO_SPINLOCK_ID_CLAIM_FREE_FIRST 24
#endif
 
#ifdef PICO_SPINLOCK_ID_CLAIM_FREE_END
#warning PICO_SPINLOCK_ID_CLAIM_FREE_END has been renamed to PICO_SPINLOCK_ID_CLAIM_FREE_LAST
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_CLAIM_FREE_LAST, Highest Spinlock ID in the 'claim free' range, min=0, max=31, default=31, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_CLAIM_FREE_LAST
#define PICO_SPINLOCK_ID_CLAIM_FREE_LAST 31
#endif
 
#if !PICO_USE_SW_SPIN_LOCKS
// Hardware lock flag in SIO:
typedef io_rw_32 spin_lock_t;
#else
#ifndef SW_SPIN_LOCK_TYPE
// Byte flag in memory:
#define SW_SPIN_LOCK_TYPE volatile uint8_t
#endif
typedef SW_SPIN_LOCK_TYPE spin_lock_t;
#endif
 
#if PICO_USE_SW_SPIN_LOCKS
#ifndef SW_SPIN_LOCK_INSTANCE
#define SW_SPIN_LOCK_INSTANCE(lock_num) ({             \
    extern spin_lock_t _sw_spin_locks[NUM_SPIN_LOCKS]; \
    &_sw_spin_locks[lock_num];                         \
    })
#endif
 
#ifndef SW_SPIN_LOCK_NUM
#define SW_SPIN_LOCK_NUM(lock) ({                          \
        extern spin_lock_t _sw_spin_locks[NUM_SPIN_LOCKS]; \
        (lock) - _sw_spin_locks;                           \
        })
#endif
 
#ifndef SW_SPIN_LOCK_IS_LOCKED
#define SW_SPIN_LOCK_IS_LOCKED(lock) ((bool) *(lock))
#endif
 
#ifndef SW_SPIN_LOCK_LOCK
#if __ARM_ARCH_8M_MAIN__
#define SW_SPIN_LOCK_LOCK(lock) ({                             \
    uint32_t _tmp0, _tmp1;                                     \
    pico_default_asm_volatile (                                \
    "1:\n"                                                     \
    "ldaexb %1, [%2]\n"                                        \
    "movs %0, #1\n" /* fill dependency slot */                 \
    "cmp %1, #0\n"                                             \
    /* Immediately retry if lock is seen to be taken */        \
    "bne 1b\n"                                                 \
    /* Attempt to claim */                                     \
    "strexb %1, %0, [%2]\n"                                    \
    "cmp %1, #0\n"                                             \
    /* Claim failed due to intervening write, so retry */      \
    "bne 1b\n"                                                 \
    : "=&r" (_tmp0), "=&r" (_tmp1) : "r" (lock)                \
    );                                                         \
    __mem_fence_acquire();                                     \
    })
#elif __riscv && (defined(__riscv_a) || defined(__riscv_zaamo))
#define SW_SPIN_LOCK_LOCK(lock) ({                                              \
    uint32_t _tmp0, _tmp1;                                                      \
    pico_default_asm_volatile (                                                 \
        /* Get word address, and bit mask for LSB of the */                     \
        /* correct byte within that word -- note shamt is modulo xlen: */       \
        "slli %1, %0, 3\n"                                                      \
        "bset %1, zero, %1\n"                                                   \
        "andi %0, %0, -4\n"                                                     \
        /* Repeatedly set the bit until we see that it was clear at the */      \
        /* point we set it. A set from 0 -> 1 is a successful lock take. */     \
    "1:"                                                                        \
        "amoor.w.aq %2, %1, (%0)\n"                                             \
        "and %2, %2, %1\n"                                                      \
        "bnez %2, 1b\n"                                                         \
        : "+r" (lock), "=r" (_tmp0), "=r" (_tmp1)                               \
    );                                                                          \
    __mem_fence_acquire();                                                      \
    })
#else
#error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
#endif
#endif
 
#ifndef SW_SPIN_TRY_LOCK
#if __ARM_ARCH_8M_MAIN__
#define SW_SPIN_TRY_LOCK(lock) ({                              \
    uint32_t _tmp0, _tmp1;                                     \
    pico_default_asm_volatile (                                \
    "ldaexb %1, [%2]\n"                                        \
    "movs %0, #1\n" /* fill dependency slot */                 \
    "cmp %1, #0\n"                                             \
    /* Immediately give up if lock is seen to be taken */      \
    "bne 1f\n"                                                 \
    /* Otherwise attempt to claim, once. */                    \
    "strexb %1, %0, [%2]\n"                                    \
    "1:\n"                                                     \
    : "=&r" (_tmp0), "=&r" (_tmp1) : "r" (lock)                \
    );                                                         \
    __mem_fence_acquire();                                     \
    !_tmp1;                                                    \
    })
#elif __riscv && (defined(__riscv_a) || defined(__riscv_zaamo))
#define SW_SPIN_TRY_LOCK(lock) ({                                               \
    uint32_t _tmp0;                                                             \
    pico_default_asm_volatile (                                                 \
        /* Get word address, and bit mask for LSB of the */                     \
        /* correct byte within that word -- note shamt is modulo xlen: */       \
        "slli %1, %0, 3\n"                                                      \
        "bset %1, zero, %1\n"                                                   \
        "andi %0, %0, -4\n"                                                     \
        /* Set the bit. If it was clear at the point we set it, then we took */ \
        /* the lock. Otherwise the lock was already held, and we give up. */    \
        "amoor.w.aq %0, %1, (%0)\n"                                             \
        "and %1, %1, %0\n"                                                      \
        : "+r" (lock), "=r" (_tmp0)                                             \
    );                                                                          \
    __mem_fence_acquire();                                                      \
    !_tmp0;                                                                     \
    })
#else
#error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
#endif
#endif
 
#ifndef SW_SPIN_LOCK_UNLOCK
#if __ARM_ARCH_8M_MAIN__
#define SW_SPIN_LOCK_UNLOCK(lock) ({                                        \
    /* Release-ordered store is available: use instead of separate fence */ \
    uint32_t zero = 0;                                                      \
    pico_default_asm_volatile(                                              \
        "stlb %0, [%1]\n"                                                   \
        : : "r" (zero), "r" (lock)                                          \
    );                                                                      \
    })
#elif __riscv
#define SW_SPIN_LOCK_UNLOCK(lock) ({                              \
    __mem_fence_release();                                        \
    *(lock) = 0; /* write to spinlock register (release lock) */  \
    })
#else
#error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
#endif
#endif
 
#endif
 
__force_inline static spin_lock_t *spin_lock_instance(uint lock_num) {
    invalid_params_if(HARDWARE_SYNC, lock_num >= NUM_SPIN_LOCKS);
#if PICO_USE_SW_SPIN_LOCKS
    return SW_SPIN_LOCK_INSTANCE(lock_num);
#else
    return (spin_lock_t *) (SIO_BASE + SIO_SPINLOCK0_OFFSET + lock_num * 4);
#endif
}
 
__force_inline static uint spin_lock_get_num(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    uint lock_num = SW_SPIN_LOCK_NUM(lock);
    invalid_params_if(HARDWARE_SYNC, lock_num >= (uint)NUM_SPIN_LOCKS);
    return lock_num;
#else
    invalid_params_if(HARDWARE_SYNC, (uint) lock < SIO_BASE + SIO_SPINLOCK0_OFFSET ||
                            (uint) lock >= NUM_SPIN_LOCKS * sizeof(spin_lock_t) + SIO_BASE + SIO_SPINLOCK0_OFFSET ||
                            ((uint) lock - SIO_BASE + SIO_SPINLOCK0_OFFSET) % sizeof(spin_lock_t) != 0);
    return (uint) (lock - (spin_lock_t *) (SIO_BASE + SIO_SPINLOCK0_OFFSET));
#endif
}
 
__force_inline static void spin_lock_unsafe_blocking(spin_lock_t *lock) {
    // Note we don't do a wfe or anything, because by convention these spin_locks are VERY SHORT LIVED and NEVER BLOCK and run
    // with INTERRUPTS disabled (to ensure that)... therefore nothing on our core could be blocking us, so we just need to wait on another core
    // anyway which should be finished soon
#if PICO_USE_SW_SPIN_LOCKS
    SW_SPIN_LOCK_LOCK(lock);
#else
    while (__builtin_expect(!*lock, 0)) { // read from spinlock register (tries to acquire the lock)
        tight_loop_contents();
    }
    __mem_fence_acquire();
#endif
}
 
__force_inline static bool spin_try_lock_unsafe(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    return SW_SPIN_TRY_LOCK(lock);
#else
    return *lock;
#endif
}
__force_inline static void spin_unlock_unsafe(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    SW_SPIN_LOCK_UNLOCK(lock);
#else
    __mem_fence_release();
    *lock = 0; // write to spinlock register (release lock)
#endif
}
 
__force_inline static uint32_t spin_lock_blocking(spin_lock_t *lock) {
    uint32_t save = save_and_disable_interrupts();
    spin_lock_unsafe_blocking(lock);
    return save;
}
 
inline static bool is_spin_locked(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    return SW_SPIN_LOCK_IS_LOCKED(lock);
#else
    check_hw_size(spin_lock_t, 4);
    uint lock_num = spin_lock_get_num(lock);
    return 0 != (*(io_ro_32 *) (SIO_BASE + SIO_SPINLOCK_ST_OFFSET) & (1u << lock_num));
#endif
}
 
__force_inline static void spin_unlock(spin_lock_t *lock, uint32_t saved_irq) {
    spin_unlock_unsafe(lock);
    restore_interrupts_from_disabled(saved_irq);
}
 
spin_lock_t *spin_lock_init(uint lock_num);
 
void spin_locks_reset(void);
 
#endif