restore: use the new kernel interface to restore timers

Thomas Gleixner introduced the new interface to create posix timers
with specifed timer IDs:
ec2d0c0462

Previously, CRIU recreated timers by repeatedly creating and deleting
them until the desired ID was reached. This approach isn't fast,
especially for timers with large IDs. For example, restoring two timers
with IDs 1000000 and 2000000 took approximately 1.5 seconds.

The new `prctl()` based interface allows direct creation of timers with
specified IDs, reducing the restoration time to around 3 microseconds
for the same example.

Signed-off-by: Andrei Vagin <avagin@gmail.com>

criu/cr-check.c

@@ -1392,6 +1392,14 @@ static int check_pagemap_scan(void)
 	return 0;
 }
 
+static int check_timer_cr_ids(void)
+{
+	if (!kdat.has_timer_cr_ids)
+		return -1;
+
+	return 0;
+}
+
 /* musl doesn't have a statx wrapper... */
 struct staty {
 	__u32 stx_dev_major;
@@ -1703,6 +1711,7 @@ int cr_check(void)
 		ret |= check_ipv6_freebind();
 		ret |= check_pagemap_scan();
 		ret |= check_overlayfs_maps();
+		ret |= check_timer_cr_ids();
 
 		if (kdat.lsm == LSMTYPE__APPARMOR)
 			ret |= check_apparmor_stacking();
@@ -1825,6 +1834,7 @@ static struct feature_list feature_list[] = {
 	{ "get_rseq_conf", check_ptrace_get_rseq_conf },
 	{ "ipv6_freebind", check_ipv6_freebind },
 	{ "pagemap_scan", check_pagemap_scan },
+	{ "timer_cr_ids", check_timer_cr_ids },
 	{ "overlayfs_maps", check_overlayfs_maps },
 	{ NULL, NULL },
 };

criu/include/kerndat.h

@@ -89,6 +89,7 @@ struct kerndat_s {
 	bool has_pagemap_scan;
 	bool has_shstk;
 	bool has_close_range;
+	bool has_timer_cr_ids;
 };
 
 extern struct kerndat_s kdat;

criu/include/prctl.h

@@ -97,4 +97,11 @@ struct prctl_mm_map {
 #define PR_GET_THP_DISABLE 42
 #endif
 
+#ifndef PR_TIMER_CREATE_RESTORE_IDS
+#define PR_TIMER_CREATE_RESTORE_IDS             77
+# define PR_TIMER_CREATE_RESTORE_IDS_OFF        0
+# define PR_TIMER_CREATE_RESTORE_IDS_ON         1
+# define PR_TIMER_CREATE_RESTORE_IDS_GET        2
+#endif
+
 #endif /* __CR_PRCTL_H__ */

criu/include/restorer.h

@@ -170,6 +170,7 @@ struct task_restore_args {
 
 	struct restore_posix_timer *posix_timers;
 	unsigned int posix_timers_n;
+	bool posix_timer_cr_ids;
 
 	struct restore_timerfd *timerfd;
 	unsigned int timerfd_n;

criu/kerndat.c

@@ -1720,6 +1720,22 @@ static int kerndat_has_close_range(void)
 	return 0;
 }
 
+static int kerndat_has_timer_cr_ids(void)
+{
+	if (prctl(PR_TIMER_CREATE_RESTORE_IDS,
+		  PR_TIMER_CREATE_RESTORE_IDS_GET, 0, 0, 0) == -1) {
+		if (errno == EINVAL) {
+			pr_debug("PR_TIMER_CREATE_RESTORE_IDS isn't supported\n");
+			return 0;
+		}
+		pr_perror("prctl returned unexpected error code");
+		return -1;
+	}
+
+	kdat.has_timer_cr_ids = true;
+	return 0;
+}
+
 /*
  * Some features depend on resource that can be dynamically changed
  * at the OS runtime. There are cases that we cannot determine the
@@ -1981,6 +1997,10 @@ int kerndat_init(void)
 		pr_err("kerndat_has_close_range has failed when initializing kerndat.\n");
 		ret = -1;
 	}
+	if (!ret && kerndat_has_timer_cr_ids()) {
+		pr_err("kerndat_has_timer_cr_ids has failed when initializing kerndat.\n");
+		ret = -1;
+	}
 
 	kerndat_lsm();
 	kerndat_mmap_min_addr();

criu/pie/restorer.c

@@ -1235,9 +1235,23 @@ static int timerfd_arm(struct task_restore_args *args)
 
 static int create_posix_timers(struct task_restore_args *args)
 {
-	int ret, i;
+	int ret, i, exit_code = -1;
 	kernel_timer_t next_id = 0, timer_id;
 	struct sigevent sev;
+	bool create_restore_ids = false;
+
+	if (!args->posix_timers_n)
+		return 0;
+
+	/* prctl returns EINVAL if PR_TIMER_CREATE_RESTORE_IDS isn't supported. */
+	ret = sys_prctl(PR_TIMER_CREATE_RESTORE_IDS,
+			PR_TIMER_CREATE_RESTORE_IDS_ON, 0, 0, 0);
+	if (ret == 0) {
+		create_restore_ids = true;
+	} else if (ret != -EINVAL) {
+		pr_err("Can't enabled PR_TIMER_CREATE_RESTORE_IDS: %d\n", ret);
+		return -1;
+	}
 
 	for (i = 0; i < args->posix_timers_n; i++) {
 		sev.sigev_notify = args->posix_timers[i].spt.it_sigev_notify;
@@ -1249,16 +1263,36 @@ static int create_posix_timers(struct task_restore_args *args)
 #endif
 		sev.sigev_value.sival_ptr = args->posix_timers[i].spt.sival_ptr;
 
+		if (create_restore_ids) {
+			/*
+			 * With enabled PR_TIMER_CREATE_RESTORE_IDS, the
+			 * timer_create syscall creates a new timer with the
+			 * specified ID.
+			 */
+			timer_id = args->posix_timers[i].spt.it_id;
+			ret = sys_timer_create(args->posix_timers[i].spt.clock_id, &sev, &timer_id);
+			if (ret < 0) {
+				pr_err("Can't create posix timer - %d: %d\n", i, ret);
+				goto out;
+			}
+			if (timer_id != args->posix_timers[i].spt.it_id) {
+				pr_err("Unexpected timer id %u (expected %lu)\n",
+				       timer_id, args->posix_timers[i].spt.it_id);
+				goto out;
+			}
+			continue;
+		}
+
 		while (1) {
 			ret = sys_timer_create(args->posix_timers[i].spt.clock_id, &sev, &timer_id);
 			if (ret < 0) {
 				pr_err("Can't create posix timer - %d\n", i);
-				return ret;
+				goto out;
 			}
 
 			if (timer_id != next_id) {
 				pr_err("Can't create timers, kernel don't give them consequently\n");
-				return -1;
+				goto out;
 			}
 			next_id++;
 
@@ -1268,12 +1302,22 @@ static int create_posix_timers(struct task_restore_args *args)
 			ret = sys_timer_delete(timer_id);
 			if (ret < 0) {
 				pr_err("Can't remove temporaty posix timer 0x%x\n", timer_id);
-				return ret;
+				goto out;
 			}
 		}
 	}
 
-	return 0;
+	exit_code = 0;
+out:
+	if (create_restore_ids) {
+		ret = sys_prctl(PR_TIMER_CREATE_RESTORE_IDS,
+				PR_TIMER_CREATE_RESTORE_IDS_OFF, 0, 0, 0);
+		if (ret != 0) {
+			pr_err("Can't disable PR_TIMER_CREATE_RESTORE_IDS: %d\n", ret);
+			exit_code = -1;
+		}
+	}
+	return exit_code;
 }
 
 static void restore_posix_timers(struct task_restore_args *args)

criu/timer.c

@@ -195,6 +195,7 @@ int prepare_posix_timers_from_fd(int pid, struct task_restore_args *ta)
 	if (!img)
 		return -1;
 
+	ta->posix_timer_cr_ids = kdat.has_timer_cr_ids;
 	ta->posix_timers_n = 0;
 	while (1) {
 		PosixTimerEntry *pte;
@@ -234,6 +235,7 @@ int prepare_posix_timers(int pid, struct task_restore_args *ta, CoreEntry *core)
 		return prepare_posix_timers_from_fd(pid, ta);
 
 	ta->posix_timers_n = tte->n_posix;
+	ta->posix_timer_cr_ids = kdat.has_timer_cr_ids;
 	for (i = 0; i < ta->posix_timers_n; i++) {
 		t = rst_mem_alloc(sizeof(struct restore_posix_timer), RM_PRIVATE);
 		if (!t)