Add support for btrfs subvolumes for user homes

new switch added to useradd command, --btrfs-subvolume-home. When
specified *and* the filesystem is detected as btrfs, it will create a
subvolume for user's home instead of a plain directory. This is done via
`btrfs subvolume` command.  Specifying the new switch while trying to
create home on non-btrfs will result in an error.

userdel -r will handle and remove this subvolume transparently via
`btrfs subvolume` command. Previosuly this failed as you can't rmdir a
subvolume.

usermod, when moving user's home across devices, will detect if the home
is a subvolume and issue an error messages instead of copying it. Moving
user's home (as subvolume) on same btrfs works transparently.

lib/prototypes.h

@@ -72,6 +72,12 @@ extern int expire (const struct passwd *, /*@null@*/const struct spwd *);
 /* isexpired.c */
 extern int isexpired (const struct passwd *, /*@null@*/const struct spwd *);
 
+/* btrfs.c */
+extern int btrfs_create_subvolume(const char *path);
+extern int btrfs_remove_subvolume(const char *path);
+extern int btrfs_is_subvolume(const char *path);
+extern int is_btrfs(const char *path);
+
 /* basename() renamed to Basename() to avoid libc name space confusion */
 /* basename.c */
 extern /*@observer@*/const char *Basename (const char *str);

libmisc/Makefile.am

@@ -10,6 +10,7 @@ libmisc_a_SOURCES = \
 	age.c \
 	audit_help.c \
 	basename.c \
+	btrfs.c \
 	chkname.c \
 	chkname.h \
 	chowndir.c \

libmisc/btrfs.c

@@ -0,0 +1,94 @@
+#include <linux/btrfs_tree.h>
+#include <linux/magic.h>
+#include <sys/statfs.h>
+
+#include "prototypes.h"
+
+
+static int run_btrfs_subvolume_cmd(const char *subcmd, const char *arg1, const char *arg2)
+{
+	int status = 0;
+	const char *cmd = "/sbin/btrfs";
+	const char *argv[] = {
+		strrchr(cmd, '/'),
+		"subvolume",
+		subcmd,
+		arg1,
+		arg2,
+		NULL
+	};
+
+	if (argv[0] == NULL)
+		argv[0] = cmd;
+	else
+		argv[0] = argv[0] + 1;
+
+	if (access(cmd, X_OK)) {
+		return 1;
+	}
+
+	if (run_command(cmd, argv, NULL, &status))
+		return -1;
+	return status;
+}
+
+
+int btrfs_create_subvolume(const char *path)
+{
+	return run_btrfs_subvolume_cmd("create", path, NULL);
+}
+
+
+int btrfs_remove_subvolume(const char *path)
+{
+	return run_btrfs_subvolume_cmd("delete", "-C", path);
+}
+
+
+/* Adapted from btrfsprogs */
+/*
+ * This intentionally duplicates btrfs_util_is_subvolume_fd() instead of opening
+ * a file descriptor and calling it, because fstat() and fstatfs() don't accept
+ * file descriptors opened with O_PATH on old kernels (before v3.6 and before
+ * v3.12, respectively), but stat() and statfs() can be called on a path that
+ * the user doesn't have read or write permissions to.
+ *
+ * returns:
+ *   1 - btrfs subvolume
+ *   0 - not btrfs subvolume
+ *  -1 - error
+ */
+int btrfs_is_subvolume(const char *path)
+{
+	struct stat st;
+	int ret;
+
+	ret = is_btrfs(path);
+	if (ret <= 0)
+		return ret;
+
+	ret = stat(path, &st);
+	if (ret == -1)
+		return -1;
+
+	if (st.st_ino != BTRFS_FIRST_FREE_OBJECTID || !S_ISDIR(st.st_mode)) {
+		return 0;
+	}
+
+	return 1;
+}
+
+
+/* Adapted from btrfsprogs */
+int is_btrfs(const char *path)
+{
+	struct statfs sfs;
+	int ret;
+
+	ret = statfs(path, &sfs);
+	if (ret == -1)
+		return -1;
+
+	return sfs.f_type == BTRFS_SUPER_MAGIC;
+}
+

src/useradd.c

@@ -165,6 +165,7 @@ static bool
     oflg = false,		/* permit non-unique user ID to be specified with -u */
     rflg = false,		/* create a system account */
     sflg = false,		/* shell program for new account */
+    subvolflg = false,		/* create subvolume home on BTRFS */
     uflg = false,		/* specify user ID for new account */
     Uflg = false;		/* create a group having the same name as the user */
 
@@ -822,6 +823,7 @@ static void usage (int status)
 	                Prog, Prog, Prog);
 	(void) fputs (_("  -b, --base-dir BASE_DIR       base directory for the home directory of the\n"
 	                "                                new account\n"), usageout);
+	(void) fputs (_("      --btrfs-subvolume-home    use BTRFS subvolume for home directory\n"), usageout);
 	(void) fputs (_("  -c, --comment COMMENT         GECOS field of the new account\n"), usageout);
 	(void) fputs (_("  -d, --home-dir HOME_DIR       home directory of the new account\n"), usageout);
 	(void) fputs (_("  -D, --defaults                print or change default useradd configuration\n"), usageout);
@@ -1102,6 +1104,7 @@ static void process_flags (int argc, char **argv)
 		int c;
 		static struct option long_options[] = {
 			{"base-dir",       required_argument, NULL, 'b'},
+			{"btrfs-subvolume-home", no_argument, NULL, 200},
 			{"comment",        required_argument, NULL, 'c'},
 			{"home-dir",       required_argument, NULL, 'd'},
 			{"defaults",       no_argument,       NULL, 'D'},
@@ -1148,6 +1151,9 @@ static void process_flags (int argc, char **argv)
 				def_home = optarg;
 				bflg = true;
 				break;
+			case 200:
+				subvolflg = true;
+				break;
 			case 'c':
 				if (!VALID (optarg)) {
 					fprintf (stderr,
@@ -2073,7 +2079,35 @@ static void create_home (void)
 			strcat (path, "/");
 			strcat (path, cp);
 			if (access (path, F_OK) != 0) {
-				if (mkdir (path, 0) != 0) {
+				/* Check if parent directory is BTRFS, fail if requesting
+				   subvolume but no BTRFS. The paths cound be different by the
+				   trailing slash
+				 */
+				if (subvolflg && (strlen(prefix_user_home) - (int)strlen(path)) <= 1) {
+					char *btrfs_check = strdup(path);
+
+					if (!btrfs_check) {
+						fprintf (stderr,
+						         _("%s: error while duplicating string in BTRFS check %s\n"),
+						         Prog, path);
+						fail_exit (E_HOMEDIR);
+					}
+					btrfs_check[strlen(path) - strlen(cp) - 1] = '\0';
+					if (is_btrfs(btrfs_check) <= 0) {
+						fprintf (stderr,
+						         _("%s: home directory \"%s\" must be mounted on BTRFS\n"),
+						         Prog, path);
+						fail_exit (E_HOMEDIR);
+					}
+					// make subvolume to mount for user instead of directory
+					if (btrfs_create_subvolume(path)) {
+						fprintf (stderr,
+						         _("%s: failed to create BTRFS subvolume: %s\n"),
+						         Prog, path);
+						fail_exit (E_HOMEDIR);
+					}
+				}
+				else if (mkdir (path, 0) != 0) {
 			fprintf (stderr,
 							_("%s: cannot create directory %s\n"),
 							Prog, path);

src/userdel.c

@@ -1272,7 +1272,21 @@ int main (int argc, char **argv)
 #endif				/* EXTRA_CHECK_HOME_DIR */
 
 	if (rflg) {
-		if (remove_tree (user_home, true) != 0) {
+		int is_subvolume = btrfs_is_subvolume (user_home);
+		if (is_subvolume < 0) {
+		    errors++;
+		    /* continue */
+		}
+		else if (is_subvolume > 0) {
+			if (btrfs_remove_subvolume (user_home)) {
+				fprintf (stderr,
+				         _("%s: error removing subvolume %s\n"),
+				         Prog, user_home);
+				errors++;
+				/* continue */
+			}
+		}
+		else if (remove_tree (user_home, true) != 0) {
 			fprintf (stderr,
 			         _("%s: error removing directory %s\n"),
 			         Prog, user_home);

src/usermod.c

@@ -1819,6 +1819,13 @@ static void move_home (void)
 			return;
 		} else {
 			if (EXDEV == errno) {
+				if (btrfs_is_subvolume (prefix_user_home) > 0) {
+					fprintf (stderr,
+					        _("%s: error: cannot move subvolume from %s to %s - different device\n"),
+					        Prog, prefix_user_home, prefix_user_newhome);
+					fail_exit (E_HOMEDIR);
+				}
+
 				if (copy_tree (prefix_user_home, prefix_user_newhome, true,
 				               true,
 				               user_id,