diff --git a/cr-restore.c b/cr-restore.c index 7e49bc388..8a611d37c 100644 --- a/cr-restore.c +++ b/cr-restore.c @@ -31,6 +31,7 @@ #include "syscall.h" #include "restorer.h" #include "sockets.h" +#include "lock.h" #include "crtools.h" @@ -1679,7 +1680,7 @@ static void sigreturn_restore(pid_t pstree_pid, pid_t pid) self_vmas_path, sizeof(task_args->self_vmas_path)); - rst_mutex_init(&task_args->rst_lock); + cr_mutex_init(&task_args->rst_lock); strncpy(task_args->ns_last_pid_path, LAST_PID_PATH, diff --git a/include/atomic.h b/include/atomic.h index 89323ce24..9934b1cd4 100644 --- a/include/atomic.h +++ b/include/atomic.h @@ -1,6 +1,8 @@ #ifndef ATOMIC_H__ #define ATOMIC_H__ +#include "types.h" + #define atomic_set(mem, v) \ ({ \ asm volatile ("lock xchg %0, %1\n" \ diff --git a/include/lock.h b/include/lock.h new file mode 100644 index 000000000..9becbe384 --- /dev/null +++ b/include/lock.h @@ -0,0 +1,107 @@ +#ifndef CR_LOCK_H_ +#define CR_LOCK_H_ + +#include +#include +#include +#include + +#include "types.h" +#include "atomic.h" +#include "syscall.h" +#include "util.h" + +/* + * Set futex @v value to @val and wake up all waiters + */ +static always_inline void cr_wait_set(u32 *v, u32 val) +{ + int ret; + + atomic_set(v, val); + + ret = sys_futex(v, FUTEX_WAKE, INT_MAX, NULL, NULL, 0); + BUG_ON(ret < 0); +} + +/* + * Decrement futex @v value to @val and wake up all waiters + */ +static always_inline void cr_wait_dec(u32 *v) +{ + int ret; + + atomic_dec(v); + + ret = sys_futex(v, FUTEX_WAKE, INT_MAX, NULL, NULL, 0); + BUG_ON(ret < 0); +} + +/* + * Wait until futex @v value become @val + */ +static always_inline void cr_wait_until(u32 *v, u32 val) +{ + int ret; + u32 tmp; + + while (1) { + tmp = *v; + if (tmp == val) + break; + ret = sys_futex(v, FUTEX_WAIT, tmp, NULL, NULL, 0); + BUG_ON(ret < 0 && ret != -EWOULDBLOCK); + } +} + +/* + * Wait while futex @v value is @val + */ +static always_inline void cr_wait_while(u32 *v, u32 val) +{ + int ret; + + while (*v == val) { + ret = sys_futex(v, FUTEX_WAIT, val, NULL, NULL, 0); + BUG_ON(ret < 0 && ret != -EWOULDBLOCK); + } +} + +/* + * Init @mutex value + */ +static void always_inline cr_mutex_init(u32 *mutex) +{ + u32 c = 0; + atomic_set(mutex, c); +} + +/* + * Lock @mutex + */ +static void always_inline cr_mutex_lock(u32 *mutex) +{ + u32 c; + int ret; + + while ((c = atomic_inc(mutex))) { + ret = sys_futex(mutex, FUTEX_WAIT, c + 1, NULL, NULL, 0); + BUG_ON(ret < 0 && ret != -EWOULDBLOCK); + } +} + +/* + * Unlock @mutex + */ +static void always_inline cr_mutex_unlock(u32 *mutex) +{ + u32 c = 0; + int ret; + + atomic_set(mutex, c); + + ret = sys_futex(mutex, FUTEX_WAKE, 1, NULL, NULL, 0); + BUG_ON(ret < 0); +} + +#endif /* CR_LOCK_H_ */ diff --git a/include/restorer.h b/include/restorer.h index 745741f08..36fdba6b9 100644 --- a/include/restorer.h +++ b/include/restorer.h @@ -7,6 +7,7 @@ #include "compiler.h" #include "types.h" #include "image.h" +#include "lock.h" #include "util.h" #ifndef CONFIG_X86_64 @@ -49,15 +50,13 @@ struct restore_mem_zone { #define first_on_heap(ptr, heap) ((typeof(ptr))heap) #define next_on_heap(ptr, prev) ((typeof(ptr))((long)(prev) + sizeof(*(prev)))) -typedef u32 rst_mutex_t; - /* Make sure it's pow2 in size */ struct thread_restore_args { struct restore_mem_zone mem_zone; int pid; int fd_core; - rst_mutex_t *rst_lock; + u32 *rst_lock; } __aligned(sizeof(long)); struct task_restore_core_args { @@ -72,7 +71,7 @@ struct task_restore_core_args { }; char ns_last_pid_path[PATH_MAX]; bool restore_threads; /* if to restore threads */ - rst_mutex_t rst_lock; + u32 rst_lock; /* threads restoration */ int nr_threads; /* number of threads */ @@ -288,29 +287,6 @@ static void always_inline write_hex_n(unsigned long num) sys_write(STDERR_FILENO, &c, 1); } -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -static void always_inline rst_mutex_init(rst_mutex_t *mutex) -{ - u32 c = 0; - atomic_set(mutex, c); -} - -static void always_inline rst_mutex_lock(rst_mutex_t *mutex) -{ - u32 c; - while ((c = atomic_inc(mutex))) - sys_futex(mutex, FUTEX_WAIT, c + 1, NULL, NULL, 0); -} - -static void always_inline rst_mutex_unlock(rst_mutex_t *mutex) -{ - u32 c = 0; - atomic_set(mutex, c); - sys_futex(mutex, FUTEX_WAKE, 1, NULL, NULL, 0); -} - /* We need own handler */ #ifdef BUG_ON_HANDLER diff --git a/restorer.c b/restorer.c index 2c376e08f..6e6485805 100644 --- a/restorer.c +++ b/restorer.c @@ -15,6 +15,7 @@ #include "image.h" #include "crtools.h" +#include "lock.h" #include "restorer.h" /* @@ -95,7 +96,7 @@ long restore_thread(long cmd, struct thread_restore_args *args) goto core_restore_end; } - rst_mutex_unlock(args->rst_lock); + cr_mutex_unlock(args->rst_lock); new_sp = (long)rt_sigframe + 8; asm volatile( @@ -454,7 +455,7 @@ self_len_end: if (thread_args[i].pid == args->pid) continue; - rst_mutex_lock(&args->rst_lock); + cr_mutex_lock(&args->rst_lock); new_sp = RESTORE_ALIGN_STACK((long)thread_args[i].mem_zone.stack,