#ifndef __CR_COMMON_LOCK_H__ #define __CR_COMMON_LOCK_H__ #include #include #include #include #include #include #include "common/asm/atomic.h" #include "common/compiler.h" /* scan-build complains about dereferencing a NULL pointer here. */ #ifndef __clang_analyzer__ #define LOCK_BUG_ON(condition) \ if ((condition)) \ *(volatile unsigned long *)NULL = 0xdead0000 + __LINE__ #define LOCK_BUG() LOCK_BUG_ON(1) #endif /* __clang_analyzer__ */ #ifdef CR_NOGLIBC #include #else #include #include static inline long sys_futex(uint32_t *addr1, int op, uint32_t val1, struct timespec *timeout, uint32_t *addr2, uint32_t val3) { int rc = syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3); if (rc == -1) rc = -errno; return rc; } #endif typedef struct { atomic_t raw; } __aligned(sizeof(int)) futex_t; #define FUTEX_ABORT_FLAG (0x80000000) #define FUTEX_ABORT_RAW (-1U) /* Get current futex @f value */ static inline uint32_t futex_get(futex_t *f) { return atomic_read(&f->raw); } /* Set futex @f value to @v */ static inline void futex_set(futex_t *f, uint32_t v) { atomic_set(&f->raw, (int)v); } #define futex_init(f) futex_set(f, 0) /* Wait on futex @__f value @__v become in condition @__c */ #define futex_wait_if_cond(__f, __v, __cond) \ do { \ int ret; \ uint32_t tmp; \ \ while (1) { \ struct timespec to = { .tv_sec = 120 }; \ tmp = futex_get(__f); \ if ((tmp & FUTEX_ABORT_FLAG) || (tmp __cond(__v))) \ break; \ ret = sys_futex((uint32_t *)&(__f)->raw.counter, FUTEX_WAIT, tmp, &to, NULL, 0); \ if (ret == -ETIMEDOUT) \ continue; \ if (ret == -EINTR || ret == -EWOULDBLOCK) \ continue; \ if (ret < 0) \ LOCK_BUG(); \ } \ } while (0) /* Set futex @f to @v and wake up all waiters */ static inline void futex_set_and_wake(futex_t *f, uint32_t v) { atomic_set(&f->raw, (int)v); LOCK_BUG_ON(sys_futex((uint32_t *)&f->raw.counter, FUTEX_WAKE, INT_MAX, NULL, NULL, 0) < 0); } /* Wake up all futex @f waiters */ static inline void futex_wake(futex_t *f) { LOCK_BUG_ON(sys_futex((uint32_t *)&f->raw.counter, FUTEX_WAKE, INT_MAX, NULL, NULL, 0) < 0); } /* Mark futex @f as wait abort needed and wake up all waiters */ static inline void futex_abort_and_wake(futex_t *f) { BUILD_BUG_ON(!(FUTEX_ABORT_RAW & FUTEX_ABORT_FLAG)); futex_set_and_wake(f, FUTEX_ABORT_RAW); } /* Decrement futex @f value and wake up all waiters */ static inline void futex_dec_and_wake(futex_t *f) { atomic_dec(&f->raw); LOCK_BUG_ON(sys_futex((uint32_t *)&f->raw.counter, FUTEX_WAKE, INT_MAX, NULL, NULL, 0) < 0); } /* Increment futex @f value and wake up all waiters */ static inline void futex_inc_and_wake(futex_t *f) { atomic_inc(&f->raw); LOCK_BUG_ON(sys_futex((uint32_t *)&f->raw.counter, FUTEX_WAKE, INT_MAX, NULL, NULL, 0) < 0); } /* Plain increment futex @f value */ static inline void futex_inc(futex_t *f) { atomic_inc(&f->raw); } /* Plain decrement futex @f value */ static inline void futex_dec(futex_t *f) { atomic_dec(&f->raw); } /* Wait until futex @f value become @v */ #define futex_wait_until(f, v) futex_wait_if_cond(f, v, ==) /* Wait while futex @f value is greater than @v */ #define futex_wait_while_gt(f, v) futex_wait_if_cond(f, v, <=) /* Wait while futex @f value is less than @v */ #define futex_wait_while_lt(f, v) futex_wait_if_cond(f, v, >=) /* Wait while futex @f value is equal to @v */ #define futex_wait_while_eq(f, v) futex_wait_if_cond(f, v, !=) /* Wait while futex @f value is @v */ static inline void futex_wait_while(futex_t *f, uint32_t v) { while ((uint32_t)atomic_read(&f->raw) == v) { int ret = sys_futex((uint32_t *)&f->raw.counter, FUTEX_WAIT, v, NULL, NULL, 0); LOCK_BUG_ON(ret < 0 && ret != -EWOULDBLOCK); } } typedef struct { atomic_t raw; } mutex_t; static inline void mutex_init(mutex_t *m) { uint32_t c = 0; atomic_set(&m->raw, (int)c); } static inline void mutex_lock(mutex_t *m) { uint32_t c; int ret; while ((c = (uint32_t)atomic_inc_return(&m->raw)) != 1) { ret = sys_futex((uint32_t *)&m->raw.counter, FUTEX_WAIT, c, NULL, NULL, 0); LOCK_BUG_ON(ret < 0 && ret != -EWOULDBLOCK); } } static inline bool mutex_trylock(mutex_t *m) { return atomic_inc_return(&m->raw) == 1; } static inline void mutex_unlock(mutex_t *m) { uint32_t c = 0; atomic_set(&m->raw, (int)c); LOCK_BUG_ON(sys_futex((uint32_t *)&m->raw.counter, FUTEX_WAKE, 1, NULL, NULL, 0) < 0); } #endif /* __CR_COMMON_LOCK_H__ */