cgroups: Introduce cgroup management modes

When been playing wich checkpoint/restore of container I found
that we can't reuse existing controller if they were pre-created.
For example currently in PCS7 we're bindmount cgroups which belong
to a container in a form of

 /sys/fs/cgroup/<controller>/<container> ==> /sys/fs/cgroup/<controller>

so that CRIU dumps such configuration fine but on restore
it recreates controllers from the scratch which we would
like to bindmount them and ask CRIU to restore subcgroups
and their parameters.

So I extended --manage-cgroups option to take <mode> arguments.
Detailed description in docs.

Signed-off-by: Cyrill Gorcunov <gorcunov@openvz.org>
Acked-by: Tycho Andersen <tycho.andersen@canonical.com>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

Documentation/criu.txt

@@ -211,8 +211,25 @@ Restores previously checkpointed processes.
 *-r*, *--root* '<path>'::
     Change the root filesystem to <path> (when run in mount namespace).
 
-*--manage-cgroups*::
+*--manage-cgroups* [<mode>]::
     Restore cgroups configuration associated with a task from the image.
+    Controllers are always restored in optimistic way -- if already present
+    in system *criu* reuses it, otherwise it will be created.
++
+The '<mode>' may be one of below.
+
+    - *none*.   Do not restore cgroup properties but require cgroup to
+                pre-exist at the moment of *restore* procedure.
+
+    - *props*.  Restore cgroup properties and require cgroup to pre-exist.
+
+    - *soft*.   Restore cgroup properties if only cgroup has been created
+                by *criu*, otherwise do not restore properies.
+
+    - *full*.   Always restore all cgroups and their properties.
+
+    - *strict*. Restore all cgroups and their properties from the scratch,
+                requiring them to not present in the system.
 
 *--cgroup-root* '[<controller>:]/<newroot>'::
     Change the root cgroup the controller will be installed into. No controller

cgroup.c

@@ -1053,18 +1053,17 @@ static int prepare_cgroup_dirs(char **controllers, int n_controllers, char *paux
 	for (i = 0; i < n_ents; i++) {
 		size_t off2 = off;
 		e = ents[i];
-		struct stat st;
 
 		off2 += sprintf(paux + off, "/%s", e->dir_name);
 
-		/*
+		if (faccessat(cg, paux, F_OK, 0) < 0) {
-		 * Checking to see if file already exists. If not, create it. If
-		 * it does exist, prevent us from overwriting the properties
-		 * later by removing the CgroupDirEntry's properties.
-		 */
-		if (fstatat(cg, paux, &st, 0) < 0) {
 			if (errno != ENOENT) {
-				pr_perror("Failed accessing file %s", paux);
+				pr_perror("Failed accessing cgroup dir %s", paux);
+				return -1;
+			}
+
+			if (opts.manage_cgroups & (CG_MODE_NONE | CG_MODE_PROPS)) {
+				pr_err("Cgroup dir %s doesn't exist\n", paux);
 				return -1;
 			}
 
@@ -1072,7 +1071,7 @@ static int prepare_cgroup_dirs(char **controllers, int n_controllers, char *paux
 				pr_perror("Can't make cgroup dir %s", paux);
 				return -1;
 			}
-			pr_info("Created dir %s\n", paux);
+			pr_info("Created cgroup dir %s\n", paux);
 
 			for (j = 0; j < n_controllers; j++) {
 				if (strcmp(controllers[j], "cpuset") == 0) {
@@ -1083,12 +1082,21 @@ static int prepare_cgroup_dirs(char **controllers, int n_controllers, char *paux
 				}
 			}
 		} else {
-			if (e->n_properties > 0) {
+			pr_info("Determined cgroup dir %s already exist\n", paux);
-				xfree(e->properties);
+
-				e->properties = NULL;
+			if (opts.manage_cgroups & CG_MODE_STRICT) {
-				e->n_properties = 0;
+				pr_err("Abort restore of existing cgroups\n");
+				return -1;
+			}
+
+			if (opts.manage_cgroups & (CG_MODE_SOFT | CG_MODE_NONE)) {
+				pr_info("Skip restoring properties on cgroup dir %s\n", paux);
+				if (e->n_properties > 0) {
+					xfree(e->properties);
+					e->properties = NULL;
+					e->n_properties = 0;
+				}
 			}
-			pr_info("Determined dir %s already existed\n", paux);
 		}
 
 		if (prepare_cgroup_dirs(controllers, n_controllers, paux, off2,
@@ -1122,10 +1130,14 @@ static int prepare_cgroup_sfd(CgroupEntry *ce)
 	int off, i, ret;
 	char paux[PATH_MAX];
 
-	pr_info("Preparing cgroups yard\n");
+	pr_info("Preparing cgroups yard (cgroups restore mode %#x)\n",
+		opts.manage_cgroups);
+	if (!opts.manage_cgroups)
+		return 0;
 
 	cg_yard = opts.cg_yard;
 	off = strlen(opts.cg_yard);
+	strcpy(paux, opts.cg_yard);
 
 	pr_debug("Opening %s as cg yard\n", cg_yard);
 	i = open(cg_yard, O_DIRECTORY);
@@ -1139,7 +1151,6 @@ static int prepare_cgroup_sfd(CgroupEntry *ce)
 	if (ret < 0)
 		goto err;
 
-
 	paux[off++] = '/';
 
 	for (i = 0; i < ce->n_controllers; i++) {
@@ -1156,26 +1167,28 @@ static int prepare_cgroup_sfd(CgroupEntry *ce)
 				paux + ctl_off, sizeof(paux) - ctl_off,
 				opt, sizeof(opt));
 
-		pr_debug("\tMaking subdir %s (%s)\n", paux, opt);
+		/* Create controller if not yet present */
-		if (mkdir(paux, 0700)) {
+		if (access(paux, F_OK)) {
-			pr_perror("Can't make cgyard subdir %s", paux);
+			pr_debug("\tMaking controller dir %s (%s)\n", paux, opt);
-			goto err;
+			if (mkdir(paux, 0700)) {
+				pr_perror("\tCan't make controller dir %s", paux);
+				return -1;
+			}
+			if (mount("none", paux, "cgroup", 0, opt) < 0) {
+				pr_perror("\tCan't mount controller dir %s", paux);
+				return -1;
+			}
 		}
 
-		if (mount("none", paux, "cgroup", 0, opt) < 0) {
+		/*
-			pr_perror("Can't mount %s cgyard", paux);
+		 * Finally handle all cgroups for this controller.
-			goto err;
+		 */
-		}
-
-		/* We skip over the .criu.cgyard.XXXXXX/, since those will be
-		 * referred to by the cg yard service fd. */
 		yard = paux + strlen(cg_yard) + 1;
 		yard_off = ctl_off - (strlen(cg_yard) + 1);
 		if (opts.manage_cgroups &&
 		    prepare_cgroup_dirs(ctrl->cnames, ctrl->n_cnames, yard, yard_off,
 				ctrl->dirs, ctrl->n_dirs))
 			goto err;
-
 	}
 
 	return 0;

crtools.c

@@ -58,7 +58,7 @@ void init_opts(void)
 
 	opts.cg_yard = "/sys/fs/cgroup";
 	opts.cpu_cap = CPU_CAP_DEFAULT;
-	opts.manage_cgroups = false;
+	opts.manage_cgroups = CG_MODE_DEFAULT;
 	opts.ps_socket = -1;
 }
 
@@ -148,6 +148,33 @@ Esyntax:
 	return -1;
 }
 
+static int parse_manage_cgroups(struct cr_options *opts, const char *optarg)
+{
+	if (!optarg) {
+		opts->manage_cgroups = CG_MODE_SOFT;
+		return 0;
+	}
+
+	if (!strcmp(optarg, "none")) {
+		opts->manage_cgroups = CG_MODE_NONE;
+	} else if (!strcmp(optarg, "props")) {
+		opts->manage_cgroups = CG_MODE_PROPS;
+	} else if (!strcmp(optarg, "soft")) {
+		opts->manage_cgroups = CG_MODE_SOFT;
+	} else if (!strcmp(optarg, "full")) {
+		opts->manage_cgroups = CG_MODE_FULL;
+	} else if (!strcmp(optarg, "strict")) {
+		opts->manage_cgroups = CG_MODE_STRICT;
+	} else
+		goto Esyntax;
+
+	return 0;
+
+Esyntax:
+	pr_err("Unknown cgroups mode `%s' selected\n", optarg);
+	return -1;
+}
+
 int main(int argc, char *argv[], char *envp[])
 {
 	pid_t pid = 0, tree_id = 0;
@@ -200,7 +227,7 @@ int main(int argc, char *argv[], char *envp[])
 		{ "force-irmap",		no_argument,		0, 1058	},
 		{ "ext-mount-map",		required_argument,	0, 'M'	},
 		{ "exec-cmd",			no_argument,		0, 1059	},
-		{ "manage-cgroups",		no_argument,		0, 1060	},
+		{ "manage-cgroups",		optional_argument,	0, 1060	},
 		{ "cgroup-root",		required_argument,	0, 1061	},
 		{ "inherit-fd",			required_argument,	0, 1062	},
 		{ "feature",			required_argument,	0, 1063	},
@@ -394,7 +421,8 @@ int main(int argc, char *argv[], char *envp[])
 			has_exec_cmd = true;
 			break;
 		case 1060:
-			opts.manage_cgroups = true;
+			if (parse_manage_cgroups(&opts, optarg))
+				goto usage;
 			break;
 		case 1061:
 			{
@@ -674,7 +702,8 @@ usage:
 "                        allow autoresolving mounts with external sharing\n"
 "  --enable-external-masters\n"
 "                        allow autoresolving mounts with external masters\n"
-"  --manage-cgroups      dump or restore cgroups the process is in\n"
+"  --manage-cgroups [m]  dump or restore cgroups the process is in usig mode:\n"
+"                        'none', 'props', 'soft' (default), 'full' and 'strict'.\n"
 "  --cgroup-root [controller:]/newroot\n"
 "                        change the root cgroup the controller will be\n"
 "                        installed into. No controller means that root is the\n"

include/cr_options.h

@@ -21,6 +21,18 @@ struct cg_root_opt {
 	char *newroot;
 };
 
+/*
+ * Cgroup management options.
+ */
+#define CG_MODE_IGNORE		(0u << 0)	/* Zero is important here */
+#define CG_MODE_NONE		(1u << 0)
+#define CG_MODE_PROPS		(1u << 1)
+#define CG_MODE_SOFT		(1u << 2)
+#define CG_MODE_FULL		(1u << 3)
+#define CG_MODE_STRICT		(1u << 4)
+
+#define CG_MODE_DEFAULT		(CG_MODE_IGNORE)
+
 struct cr_options {
 	int			final_state;
 	char			*show_dump_file;
@@ -59,7 +71,7 @@ struct cr_options {
 	unsigned int		cpu_cap;
 	bool			force_irmap;
 	char			**exec_cmd;
-	bool			manage_cgroups;
+	unsigned int		manage_cgroups;
 	char			*cg_yard;
 	char			*new_global_cg_root;
 	struct list_head	new_cgroup_roots;