cgroup: improve support for multiple lxcpaths (v3)

Add a monitor command to get the cgroup for a running container.  This
allows container r1 started from /var/lib/lxc and container r1 started
from /home/ubuntu/lxcbase to pick unique cgroup directories (which
will be /sys/fs/cgroup/$subsys/lxc/r1 and .../r1-1), and all the lxc-*
tools to get that path over the monitor at lxcpath.

Rework the cgroup code.  Before, if /sys/fs/cgroup/$subsys/lxc/r1
already existed, it would be moved to 'deadXXXXX', and a new r1 created.
Instead, if r1 exists, use r1-1, r1-2, etc.

I ended up removing both the use of cgroup.clone_children and support
for ns cgroup.  Presumably we'll want to put support for ns cgroup
back in for older kernels.  Instead of guessing whether or not we
have clone_children support, just always explicitly do the only thing
that feature buys us - set cpuset.{cpus,mems} for newly created cgroups.

Note that upstream kernel is working toward strict hierarchical
limit enforcements, which will be good for us.

NOTE - I am changing the lxc_answer struct size.  This means that
upgrades to this version while containers are running will result
in lxc_* commands on pre-running containers will fail.

Changelog: (v3)
   implement cgroup attach
   fix a subtle bug arising when we lxc_get_cgpath() returned
     STOPPED rather than -1 (STOPPED is 0, and 0 meant success).
   Rename some functions and add detailed comments above most.
   Drop all my lxc_attach changes in favor of those by Christian
     Seiler (which are mostly the same, but improved).

Signed-off-by: Serge Hallyn <serge.hallyn@ubuntu.com>

src/lxc/attach.c

@@ -42,7 +42,6 @@
 #include "log.h"
 #include "attach.h"
 #include "caps.h"
-#include "cgroup.h"
 #include "config.h"
 #include "apparmor.h"
 

src/lxc/cgroup.h

@@ -26,13 +26,13 @@
 #define MAXPRIOLEN 24
 
 struct lxc_handler;
-extern int lxc_cgroup_create(const char *name, pid_t pid);
-extern int lxc_cgroup_destroy(const char *name);
-extern int lxc_cgroup_path_get(char **path, const char *subsystem, const char *name);
-extern int lxc_cgroup_nrtasks(const char *name);
-extern int lxc_cgroup_attach(const char *name, pid_t pid);
-extern int lxc_cgroup_prepare_attach(const char *name, void **data);
-extern int lxc_cgroup_finish_attach(void *data, pid_t pid);
-extern int lxc_cgroup_dispose_attach(void *data);
-extern int lxc_ns_is_mounted(void);
+extern int lxc_cgroup_destroy(const char *cgpath);
+extern int lxc_cgroup_path_get(char **path, const char *subsystem, const char *name,
+			      const char *lxcpath);
+extern int lxc_cgroup_nrtasks(const char *cgpath);
+extern char *lxc_cgroup_path_create(const char *lxcgroup, const char *name);
+extern int lxc_cgroup_enter(const char *cgpath, pid_t pid);
+extern int lxc_cgroup_attach(pid_t pid, const char *name, const char *lxcpath);
+extern int cgroup_path_get(char **path, const char *subsystem, const char *cgpath);
+extern int lxc_get_cgpath(const char **path, const char *subsystem, const char *name, const char *lxcpath);
 #endif

src/lxc/commands.c

@@ -84,10 +84,19 @@ static int fill_sock_name(char *path, int len, const char *name,
 static int receive_answer(int sock, struct lxc_answer *answer)
 {
 	int ret;
+	static char answerpath[MAXPATHLEN];
 
 	ret = lxc_af_unix_recv_fd(sock, &answer->fd, answer, sizeof(*answer));
 	if (ret < 0)
 		ERROR("failed to receive answer for the command");
+	if (answer->pathlen == 0)
+		return ret;
+	ret = recv(sock, answerpath, answer->pathlen, 0);
+	if (ret != answer->pathlen) {
+		ERROR("failed to receive answer for the command");
+		ret = 0;
+	} else
+		answer->path = answerpath;
 
 	return ret;
 }
@@ -202,6 +211,7 @@ extern int  lxc_stop_callback(int, struct lxc_request *, struct lxc_handler *);
 extern int  lxc_state_callback(int, struct lxc_request *, struct lxc_handler *);
 extern int  lxc_pid_callback(int, struct lxc_request *, struct lxc_handler *);
 extern int  lxc_clone_flags_callback(int, struct lxc_request *, struct lxc_handler *);
+extern int lxc_cgroup_callback(int, struct lxc_request *, struct lxc_handler *);
 
 static int trigger_command(int fd, struct lxc_request *request,
 			   struct lxc_handler *handler)
@@ -214,6 +224,7 @@ static int trigger_command(int fd, struct lxc_request *request,
 		[LXC_COMMAND_STATE]       = lxc_state_callback,
 		[LXC_COMMAND_PID]         = lxc_pid_callback,
 		[LXC_COMMAND_CLONE_FLAGS] = lxc_clone_flags_callback,
+		[LXC_COMMAND_CGROUP]      = lxc_cgroup_callback,
 	};
 
 	if (request->type < 0 || request->type >= LXC_COMMAND_MAX)

src/lxc/commands.h

@@ -29,6 +29,7 @@ enum {
 	LXC_COMMAND_STATE,
 	LXC_COMMAND_PID,
 	LXC_COMMAND_CLONE_FLAGS,
+	LXC_COMMAND_CGROUP,
 	LXC_COMMAND_MAX,
 };
 
@@ -41,6 +42,8 @@ struct lxc_answer {
 	int fd;
 	int ret; /* 0 on success, -errno on failure */
 	pid_t pid;
+	int pathlen;
+	const char *path;
 };
 
 struct lxc_command {

src/lxc/conf.c

@@ -1375,7 +1375,7 @@ static int setup_kmsg(const struct lxc_rootfs *rootfs,
 	return 0;
 }
 
-int setup_cgroup(const char *name, struct lxc_list *cgroups)
+int setup_cgroup(const char *cgpath, struct lxc_list *cgroups)
 {
 	struct lxc_list *iterator;
 	struct lxc_cgroup *cg;
@@ -1388,8 +1388,11 @@ int setup_cgroup(const char *name, struct lxc_list *cgroups)
 
 		cg = iterator->elem;
 
-		if (lxc_cgroup_set(name, cg->subsystem, cg->value))
+		if (lxc_cgroup_set_bypath(cgpath, cg->subsystem, cg->value)) {
+			ERROR("Error setting %s to %s for %s\n", cg->subsystem,
+				cg->value, cgpath);
 			goto out;
+		}
 
 		DEBUG("cgroup '%s' set to '%s'", cg->subsystem, cg->value);
 	}

src/lxc/conf.h

@@ -282,7 +282,7 @@ struct lxc_conf {
 
 int run_lxc_hooks(const char *name, char *hook, struct lxc_conf *conf);
 
-extern int setup_cgroup(const char *name, struct lxc_list *cgroups);
+extern int setup_cgroup(const char *cgpath, struct lxc_list *cgroups);
 extern int detect_shared_rootfs(void);
 
 /*

src/lxc/freezer.c

@@ -40,16 +40,11 @@
 
 lxc_log_define(lxc_freezer, lxc);
 
-static int freeze_unfreeze(const char *name, int freeze, const char *lxcpath)
+static int do_unfreeze(const char *nsgroup, int freeze, const char *name, const char *lxcpath)
 {
-	char *nsgroup;
 	char freezer[MAXPATHLEN], *f;
 	char tmpf[32];
 	int fd, ret;
-	
-	ret = lxc_cgroup_path_get(&nsgroup, "freezer", name);
-	if (ret)
-		return -1;
 
 	ret = snprintf(freezer, MAXPATHLEN, "%s/freezer.state", nsgroup);
 	if (ret >= MAXPATHLEN) {
@@ -59,7 +54,7 @@ static int freeze_unfreeze(const char *name, int freeze, const char *lxcpath)
 
 	fd = open(freezer, O_RDWR);
 	if (fd < 0) {
-		SYSERROR("failed to open freezer for '%s'", name);
+		SYSERROR("failed to open freezer at '%s'", nsgroup);
 		return -1;
 	}
 
@@ -98,7 +93,8 @@ static int freeze_unfreeze(const char *name, int freeze, const char *lxcpath)
 		ret = strncmp(f, tmpf, strlen(f));
 		if (!ret)
 		{
-			lxc_monitor_send_state(name, freeze ? FROZEN : THAWED, lxcpath);
+			if (name)
+				lxc_monitor_send_state(name, freeze ? FROZEN : THAWED, lxcpath);
 			break;		/* Success */
 		}
 
@@ -122,6 +118,18 @@ out:
 	return ret;
 }
 
+static int freeze_unfreeze(const char *name, int freeze, const char *lxcpath)
+{
+	char *nsgroup;
+	int ret;
+	
+	ret = lxc_cgroup_path_get(&nsgroup, "freezer", name, lxcpath);
+	if (ret)
+		return -1;
+
+	return do_unfreeze(nsgroup, freeze, name, lxcpath);
+}
+
 int lxc_freeze(const char *name, const char *lxcpath)
 {
 	lxc_monitor_send_state(name, FREEZING, lxcpath);
@@ -133,3 +141,14 @@ int lxc_unfreeze(const char *name, const char *lxcpath)
 	return freeze_unfreeze(name, 0, lxcpath);
 }
 
+int lxc_unfreeze_bypath(const char *cgpath)
+{
+	char *nsgroup;
+	int ret;
+	
+	ret = cgroup_path_get(&nsgroup, "freezer", cgpath);
+	if (ret)
+		return -1;
+
+	return do_unfreeze(nsgroup, 0, NULL, NULL);
+}

src/lxc/lxc.h

@@ -117,6 +117,14 @@ extern int lxc_freeze(const char *name, const char *lxcpath);
  */
 extern int lxc_unfreeze(const char *name, const char *lxcpath);
 
+/*
+ * Unfreeze all previously frozen tasks.
+ * This is the function to use from inside the monitor
+ * @name : the name of the container
+ * Return 0 on sucess, < 0 otherwise
+ */
+extern int lxc_unfreeze_bypath(const char *cgpath);
+
 /*
  * Retrieve the container state
  * @name : the name of the container
@@ -127,24 +135,36 @@ extern lxc_state_t lxc_state(const char *name, const char *lxcpath);
 /*
  * Set a specified value for a specified subsystem. The specified
  * subsystem must be fully specified, eg. "cpu.shares"
- * @name      : the name of the container
- * @filename : the cgroup attribute filename
+ * @cgpath    : the cgroup path of the container
+ * @filename  : the cgroup attribute filename
  * @value     : the value to be set
  * Returns 0 on success, < 0 otherwise
  */
-extern int lxc_cgroup_set(const char *name, const char *filename, const char *value);
+extern int lxc_cgroup_set_bypath(const char *cgpath, const char *filename, const char *value);
+
+/*
+ * Set a specified value for a specified subsystem. The specified
+ * subsystem must be fully specified, eg. "cpu.shares"
+ * @name      : the name of the container
+ * @filename  : the cgroup attribute filename
+ * @value     : the value to be set
+ * @lxcpath   : lxc config path for container
+ * Returns 0 on success, < 0 otherwise
+ */
+extern int lxc_cgroup_set(const char *name, const char *filename, const char *value, const char *lxcpath);
 
 /*
  * Get a specified value for a specified subsystem. The specified
  * subsystem must be fully specified, eg. "cpu.shares"
  * @name      : the name of the container
- * @filename : the cgroup attribute filename
+ * @filename  : the cgroup attribute filename
  * @value     : the value to be set
  * @len       : the len of the value variable
+ * @lxcpath   : lxc config path for container
  * Returns the number of bytes read, < 0 on error
  */
 extern int lxc_cgroup_get(const char *name, const char *filename,
-			  char *value, size_t len);
+			  char *value, size_t len, const char *lxcpath);
 
 /*
  * Retrieve the error string associated with the error returned by

src/lxc/lxc_attach.c

@@ -237,7 +237,7 @@ int main(int argc, char *argv[])
 		}
 
 		if (!elevated_privileges) {
-			ret = lxc_cgroup_attach(my_args.name, grandchild);
+			ret = lxc_cgroup_attach(grandchild, my_args.name, my_args.lxcpath);
 			if (ret < 0) {
 				ERROR("failed to attach process to cgroup");
 				return -1;

src/lxc/lxc_cgroup.c

@@ -78,7 +78,7 @@ int main(int argc, char *argv[])
 		value = my_args.argv[1];
 
 	if (value) {
-		if (lxc_cgroup_set(my_args.name, state_object, value)) {
+		if (lxc_cgroup_set(my_args.name, state_object, value, my_args.lxcpath)) {
 			ERROR("failed to assign '%s' value to '%s' for '%s'",
 				value, state_object, my_args.name);
 			return -1;
@@ -88,7 +88,7 @@ int main(int argc, char *argv[])
 		int ret;
 		char buffer[len];
 
-		ret = lxc_cgroup_get(my_args.name, state_object, buffer, len);
+		ret = lxc_cgroup_get(my_args.name, state_object, buffer, len, my_args.lxcpath);
 		if (ret < 0) {
 			ERROR("failed to retrieve value of '%s' for '%s'",
 			      state_object, my_args.name);

src/lxc/lxc_unshare.c

@@ -112,7 +112,6 @@ int main(int argc, char *argv[])
 {
 	int opt, status;
 	int ret;
-	char *pid_name;
 	char *namespaces = NULL;
 	char **args;
 	int flags = 0;
@@ -170,14 +169,5 @@ int main(int argc, char *argv[])
 		return -1;
 	}
 
-	if (lxc_ns_is_mounted()) {
-		if (asprintf(&pid_name, "%d", pid) == -1) {
-			ERROR("pid_name: failed to allocate memory");
-			return -1;
-		}
-		lxc_cgroup_destroy(pid_name);
-		free(pid_name);
-	}
-
 	return  lxc_error_set_and_log(pid, status);
 }

src/lxc/lxccontainer.c

@@ -945,7 +945,7 @@ static bool lxcapi_set_cgroup_item(struct lxc_container *c, const char *subsys,
 	if (is_stopped_nolock(c))
 		goto err;
 
-	ret = lxc_cgroup_set(c->name, subsys, value);
+	ret = lxc_cgroup_set(c->name, subsys, value, c->config_path);
 	if (!ret)
 		b = true;
 err:
@@ -966,7 +966,7 @@ static int lxcapi_get_cgroup_item(struct lxc_container *c, const char *subsys, c
 	if (is_stopped_nolock(c))
 		goto out;
 
-	ret = lxc_cgroup_get(c->name, subsys, retv, inlen);
+	ret = lxc_cgroup_get(c->name, subsys, retv, inlen, c->config_path);
 
 out:
 	lxcunlock(c->privlock);

src/lxc/lxcutmp.c

@@ -283,7 +283,7 @@ static int utmp_get_ntasks(struct lxc_handler *handler)
 {
 	int ntasks;
 
-	ntasks = lxc_cgroup_nrtasks(handler->name);
+	ntasks = lxc_cgroup_nrtasks(handler->cgroup);
 
 	if (ntasks < 0) {
 		ERROR("failed to get the number of tasks");

src/lxc/start.c

@@ -268,6 +268,7 @@ int lxc_pid_callback(int fd, struct lxc_request *request,
 	struct lxc_answer answer;
 	int ret;
 
+	memset(&answer, 0, sizeof(answer));
 	answer.pid = handler->pid;
 	answer.ret = 0;
 
@@ -285,12 +286,49 @@ int lxc_pid_callback(int fd, struct lxc_request *request,
 	return 0;
 }
 
+int lxc_cgroup_callback(int fd, struct lxc_request *request,
+		     struct lxc_handler *handler)
+{
+	struct lxc_answer answer;
+	int ret;
+
+	memset(&answer, 0, sizeof(answer));
+	answer.pathlen = strlen(handler->cgroup) + 1;
+	answer.path = handler->cgroup;
+	answer.ret = 0;
+
+	ret = send(fd, &answer, sizeof(answer), 0);
+	if (ret < 0) {
+		WARN("failed to send answer to the peer");
+		return -1;
+	}
+
+	if (ret != sizeof(answer)) {
+		ERROR("partial answer sent");
+		return -1;
+	}
+
+	ret = send(fd, answer.path, answer.pathlen, 0);
+	if (ret < 0) {
+		WARN("failed to send answer to the peer");
+		return -1;
+	}
+
+	if (ret != answer.pathlen) {
+		ERROR("partial answer sent");
+		return -1;
+	}
+
+	return 0;
+}
+
 int lxc_clone_flags_callback(int fd, struct lxc_request *request,
 			     struct lxc_handler *handler)
 {
 	struct lxc_answer answer;
 	int ret;
 
+	memset(&answer, 0, sizeof(answer));
 	answer.pid = 0;
 	answer.ret = handler->clone_flags;
 
@@ -467,7 +505,7 @@ out_free:
 	return NULL;
 }
 
-void lxc_fini(const char *name, struct lxc_handler *handler)
+static void lxc_fini(const char *name, struct lxc_handler *handler)
 {
 	/* The STOPPING state is there for future cleanup code
 	 * which can take awhile
@@ -487,6 +525,11 @@ void lxc_fini(const char *name, struct lxc_handler *handler)
 	close(handler->conf->maincmd_fd);
 	handler->conf->maincmd_fd = -1;
 	free(handler->name);
+	if (handler->cgroup) {
+		lxc_cgroup_destroy(handler->cgroup);
+		free(handler->cgroup);
+		handler->cgroup = NULL;
+	}
 	free(handler);
 }
 
@@ -756,7 +799,11 @@ int lxc_spawn(struct lxc_handler *handler)
 	if (lxc_sync_wait_child(handler, LXC_SYNC_CONFIGURE))
 		failed_before_rename = 1;
 
-	if (lxc_cgroup_create(name, handler->pid))
+	/* TODO - pass lxc.cgroup.dir (or user's pam cgroup) in for first argument */
+	if ((handler->cgroup = lxc_cgroup_path_create(NULL, name)) == NULL)
+		goto out_delete_net;
+	
+	if (lxc_cgroup_enter(handler->cgroup, handler->pid) < 0)
 		goto out_delete_net;
 
 	if (failed_before_rename)
@@ -786,7 +833,7 @@ int lxc_spawn(struct lxc_handler *handler)
 	if (lxc_sync_barrier_child(handler, LXC_SYNC_POST_CONFIGURE))
 		goto out_delete_net;
 
-	if (setup_cgroup(name, &handler->conf->cgroup)) {
+	if (setup_cgroup(handler->cgroup, &handler->conf->cgroup)) {
 		ERROR("failed to setup the cgroups for '%s'", name);
 		goto out_delete_net;
 	}
@@ -904,7 +951,6 @@ out_fini:
 	lxc_delete_network(handler);
 
 out_fini_nonet:
-	lxc_cgroup_destroy(name);
 	lxc_fini(name, handler);
 	return err;
 

src/lxc/start.h

@@ -51,6 +51,7 @@ struct lxc_handler {
 #endif
 	int pinfd;
 	const char *lxcpath;
+	char *cgroup;
 };
 
 extern struct lxc_handler *lxc_init(const char *name, struct lxc_conf *, const char *);
@@ -58,7 +59,6 @@ extern int lxc_spawn(struct lxc_handler *);
 
 extern int lxc_poll(const char *name, struct lxc_handler *handler);
 extern void lxc_abort(const char *name, struct lxc_handler *handler);
-extern void lxc_fini(const char *name, struct lxc_handler *handler);
 extern int lxc_set_state(const char *, struct lxc_handler *, lxc_state_t);
 extern int lxc_check_inherited(struct lxc_conf *conf, int fd_to_ignore);
 int __lxc_start(const char *, struct lxc_conf *, struct lxc_operations *,

src/lxc/state.c

@@ -66,7 +66,7 @@ lxc_state_t lxc_str2state(const char *state)
 	return -1;
 }
 
-static int freezer_state(const char *name)
+static int freezer_state(const char *name, const char *lxcpath)
 {
 	char *nsgroup;
 	char freezer[MAXPATHLEN];
@@ -74,7 +74,7 @@ static int freezer_state(const char *name)
 	FILE *file;
 	int err;
 
-	err = lxc_cgroup_path_get(&nsgroup, "freezer", name);
+	err = lxc_cgroup_path_get(&nsgroup, "freezer", name, lxcpath);
 	if (err)
 		return -1;
 
@@ -132,7 +132,7 @@ static lxc_state_t __lxc_getstate(const char *name, const char *lxcpath)
 
 lxc_state_t lxc_getstate(const char *name, const char *lxcpath)
 {
-	int state = freezer_state(name);
+	int state = freezer_state(name, lxcpath);
 	if (state != FROZEN && state != FREEZING)
 		state = __lxc_getstate(name, lxcpath);
 	return state;
@@ -148,6 +148,7 @@ extern int lxc_state_callback(int fd, struct lxc_request *request,
 	struct lxc_answer answer;
 	int ret;
 
+	memset(&answer, 0, sizeof(answer));
 	answer.ret = handler->state;
 
 	ret = send(fd, &answer, sizeof(answer), 0);

src/lxc/stop.c

@@ -83,9 +83,10 @@ extern int lxc_stop_callback(int fd, struct lxc_request *request,
 	struct lxc_answer answer;
 	int ret;
 
+	memset(&answer, 0, sizeof(answer));
 	answer.ret = kill(handler->pid, SIGKILL);
 	if (!answer.ret) {
-		ret = lxc_unfreeze(handler->name, handler->lxcpath);
+		ret = lxc_unfreeze_bypath(handler->cgroup);
 		if (!ret)
 			return 0;
 

src/tests/Makefile.am

@@ -12,6 +12,7 @@ lxc_test_shutdowntest_SOURCES = shutdowntest.c
 lxc_test_get_item_SOURCES = get_item.c
 lxc_test_getkeys_SOURCES = getkeys.c
 lxc_test_lxcpath_SOURCES = lxcpath.c
+lxc_test_cgpath_SOURCES = cgpath.c
 
 AM_CFLAGS=-I$(top_srcdir)/src \
 	-DLXCROOTFSMOUNT=\"$(LXCROOTFSMOUNT)\" \
@@ -21,6 +22,7 @@ AM_CFLAGS=-I$(top_srcdir)/src \
 
 bin_PROGRAMS = lxc-test-containertests lxc-test-locktests lxc-test-startone \
 	lxc-test-destroytest lxc-test-saveconfig lxc-test-createtest \
-	lxc-test-shutdowntest lxc-test-get_item lxc-test-getkeys lxc-test-lxcpath
+	lxc-test-shutdowntest lxc-test-get_item lxc-test-getkeys lxc-test-lxcpath \
+	lxc-test-cgpath
 
 endif

src/tests/cgpath.c

@@ -0,0 +1,164 @@
+/* liblxcapi
+ *
+ * Copyright <20> 2012 Serge Hallyn <serge.hallyn@ubuntu.com>.
+ * Copyright <20> 2012 Canonical Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#include "../lxc/lxccontainer.h"
+
+#include <unistd.h>
+#include <signal.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <errno.h>
+#include "../lxc/cgroup.h"
+#include "../lxc/lxc.h"
+
+#define MYNAME "lxctest1"
+#define MYNAME2 "lxctest2"
+
+#define TSTERR(x) do { \
+	fprintf(stderr, "%d: %s\n", __LINE__, x); \
+} while (0)
+
+int main()
+{
+	struct lxc_container *c = NULL, *c2 = NULL;
+	char *path;
+	int len;
+	int ret, retv = -1;
+
+	/* won't require privilege necessarily once users are classified by
+	 * pam_cgroup */
+	if (geteuid() != 0) {
+		TSTERR("requires privilege");
+		exit(0);
+	}
+
+	printf("Basic cgroup path tests...\n");
+	path = lxc_cgroup_path_create(NULL, MYNAME);
+	len = strlen(path);
+	if (!path || !len) {
+		TSTERR("zero result from lxc_cgroup_path_create");
+		exit(1);
+	}
+	if (!strstr(path, "lxc/" MYNAME)) {
+		TSTERR("lxc_cgroup_path_create NULL lxctest1");
+		exit(1);
+	}
+	free(path);
+
+	path = lxc_cgroup_path_create("ab", MYNAME);
+	len = strlen(path);
+	if (!path || !len) {
+		TSTERR("zero result from lxc_cgroup_path_create");
+		exit(1);
+	}
+	if (!strstr(path, "ab/" MYNAME)) {
+		TSTERR("lxc_cgroup_path_create ab lxctest1");
+		exit(1);
+	}
+	free(path);
+	printf("... passed\n");
+
+	printf("Container creation tests...\n");
+
+	if ((c = lxc_container_new(MYNAME, NULL)) == NULL) {
+		TSTERR("instantiating first container");
+		exit(1);
+	}
+	if (c->is_defined(c)) {
+		c->stop(c);
+		c->destroy(c);
+		c = lxc_container_new(MYNAME, NULL);
+	}
+	c->set_config_item(c, "lxc.network.type", "empty");
+	if (!c->createl(c, "ubuntu", NULL)) {
+		TSTERR("creating first container");
+		exit(1);
+	}
+	c->load_config(c, NULL);
+	c->want_daemonize(c);
+	if (!c->startl(c, 0, NULL)) {
+		TSTERR("starting first container");
+		goto out;
+	}
+	printf("first container passed.  Now two containers...\n");
+
+	char *nsgroup;
+#define ALTBASE "/var/lib/lxctest2"
+	ret = mkdir(ALTBASE, 0755);
+
+	ret = lxc_cgroup_path_get(&nsgroup, "freezer", MYNAME, c->get_config_path(c));
+	if (ret < 0 || !strstr(nsgroup, "lxc/" MYNAME)) {
+		TSTERR("getting first cgroup path from lxc_command");
+		goto out;
+	}
+
+	/* start second container */
+	if ((c2 = lxc_container_new(MYNAME2, ALTBASE)) == NULL) {
+		TSTERR("instantiating first container");
+		goto out;
+	}
+	if (c2->is_defined(c2)) {
+		c2->stop(c2);
+		c2->destroy(c2);
+		c2 = lxc_container_new(MYNAME2, ALTBASE);
+	}
+	c2->set_config_item(c2, "lxc.network.type", "empty");
+	if (!c2->createl(c2, "ubuntu", NULL)) {
+		TSTERR("creating first container");
+		goto out;
+	}
+
+	c2->load_config(c2, NULL);
+	c2->want_daemonize(c2);
+	if (!c2->startl(c2, 0, NULL)) {
+		TSTERR("starting first container");
+		goto out;
+	}
+
+	ret = lxc_cgroup_path_get(&nsgroup, "freezer", MYNAME2, c2->get_config_path(c2));
+	if (ret < 0 || !strstr(nsgroup, "lxc/" MYNAME2)) {
+		TSTERR("getting second cgroup path from lxc_command");
+		goto out;
+	}
+
+	const char *dirpath;
+	if (lxc_get_cgpath(&dirpath, NULL, c2->name, c2->config_path) < 0) {
+		TSTERR("getting second container's cgpath");
+		return -1;
+	}
+
+	if (lxc_cgroup_nrtasks(dirpath) < 1) {
+		TSTERR("getting nrtasks");
+		goto out;
+	}
+	printf("...passed\n");
+
+	retv = 0;
+out:
+	if (c2) {
+		c2->stop(c2);
+		c2->destroy(c2);
+	}
+	if (c) {
+		c->stop(c);
+		c->destroy(c);
+	}
+	return retv;
+}

src/tests/lxcpath.c

@@ -29,7 +29,7 @@
 #define MYNAME "lxctest1"
 
 #define TSTERR(x) do { \
-	fprintf(stderr, "%d: %s", __LINE__, x); \
+	fprintf(stderr, "%d: %s\n", __LINE__, x); \
 } while (0)
 
 int main()