libbb: updated config_parse() from Vladimir

function                                             old     new   delta
config_read                                          385     460     +75
runsvdir_main                                       1701    1716     +15
readit                                               331     338      +7
passwd_main                                         1049    1053      +4
parse_command                                       1504    1507      +3
decode_format_string                                 822     824      +2
bb__parsespent                                       117     119      +2
udhcp_get_option                                     221     222      +1
changepath                                           196     194      -2
parse_inittab                                        400     396      -4
nameif_main                                          683     679      -4
make_device                                         1176    1172      -4
config_open                                           48      40      -8
expand_main                                          698     689      -9
readcmd                                             1012    1002     -10
config_free_data                                      37      21     -16
SynchronizeFile                                      683     643     -40
sleep_main                                           474     362    -112
------------------------------------------------------------------------------
(add/remove: 0/0 grow/shrink: 8/10 up/down: 109/-209)        Total: -100 bytes

Config.in

@@ -479,6 +479,13 @@ config INCLUDE_SUSv2
 	  will be supported in head, tail, and fold.  (Note: should
 	  affect renice too.)
 
+config PARSE
+	bool "Uniform config file parser debugging applet: parse"
+
+config FEATURE_PARSE_COPY
+	bool "Keep a copy of current line"
+	depends on PARSE
+
 endmenu
 
 menu 'Installation Options'

include/applets.h

@@ -268,6 +268,7 @@ USE_NOHUP(APPLET(nohup, _BB_DIR_USR_BIN, _BB_SUID_NEVER))
 USE_NSLOOKUP(APPLET(nslookup, _BB_DIR_USR_BIN, _BB_SUID_NEVER))
 USE_OD(APPLET(od, _BB_DIR_USR_BIN, _BB_SUID_NEVER))
 USE_OPENVT(APPLET(openvt, _BB_DIR_USR_BIN, _BB_SUID_NEVER))
+USE_PARSE(APPLET(parse, _BB_DIR_USR_BIN, _BB_SUID_NEVER))
 USE_PASSWD(APPLET(passwd, _BB_DIR_USR_BIN, _BB_SUID_ALWAYS))
 USE_PATCH(APPLET(patch, _BB_DIR_USR_BIN, _BB_SUID_NEVER))
 USE_PGREP(APPLET(pgrep, _BB_DIR_USR_BIN, _BB_SUID_NEVER))

include/libbb.h

@@ -988,16 +988,22 @@ int bb_ask_confirmation(void) FAST_FUNC;
 int bb_parse_mode(const char* s, mode_t* theMode) FAST_FUNC;
 
 /*
- * Uniform config file parser helpers
+ * Config file parser
  */
+#define PARSE_DONT_REDUCE       0x00010000 // do not treat consecutive delimiters as one
+#define PARSE_DONT_TRIM         0x00020000 // do not trim line of leading and trailing delimiters
+#define PARSE_LAST_IS_GREEDY    0x00040000 // last token takes whole remainder of the line
+//#define PARSE_DONT_NULL         0x00080000 // do not set tokens[] to NULL
 typedef struct parser_t {
 	FILE *fp;
-	char *line, *data;
+	char *line;
+	USE_FEATURE_PARSE_COPY(char *data;)
 	int lineno;
 } parser_t;
 parser_t* config_open(const char *filename) FAST_FUNC;
-/* TODO: add define magic to collapse ntokens/mintokens/comment into one int param */
+int config_read(parser_t *parser, char **tokens, unsigned flags, const char *delims) FAST_FUNC;
-int config_read(parser_t *parser, char **tokens, int ntokens, int mintokens, const char *delims, char comment) FAST_FUNC;
+#define config_read(parser, tokens, max, min, str, flags) \
+	config_read(parser, tokens, ((flags) | (((min) & 0xFF) << 8) | ((max) & 0xFF)), str)
 void config_close(parser_t *parser) FAST_FUNC;
 
 /* Concatenate path and filename to new allocated buffer.

include/usage.h

@@ -2903,6 +2903,11 @@
 #define openvt_example_usage \
        "openvt 2 /bin/ash\n"
 
+#define parse_trivial_usage \
+       "[-n maxtokens] [-m mintokens] [-d delims] [-f flags] file ..."
+#define parse_full_usage "\n\n" \
+       "[-n maxtokens] [-m mintokens] [-d delims] [-f flags] file ..."
+
 #define passwd_trivial_usage \
        "[OPTION] [name]"
 #define passwd_full_usage "\n\n" \

init/init.c

@@ -808,7 +808,7 @@ static void parse_inittab(void)
 	/* optional_tty:ignored_runlevel:action:command
 	 * Delims are not to be collapsed and need exactly 4 tokens
 	 */
-	while (config_read(parser, token, -4, 0, ":", '#') >= 0) {
+ 	while (config_read(parser, token, 4, 0, "#:", PARSE_DONT_TRIM|PARSE_DONT_REDUCE|PARSE_LAST_IS_GREEDY)) {
 		int action;
 		char *tty = token[0];
 
@@ -828,7 +828,7 @@ static void parse_inittab(void)
 			free(tty);
 		continue;
  bad_entry:
-		message(L_LOG | L_CONSOLE, "Bad inittab entry: %s", parser->line);
+		message(L_LOG | L_CONSOLE, "Bad inittab entry at line %d", parser->lineno);
 	}
 	config_close(parser);
 #endif

libbb/parse_config.c

@@ -9,23 +9,51 @@
 
 #include "libbb.h"
 
+#if ENABLE_PARSE
+int parse_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
+int parse_main(int argc UNUSED_PARAM, char **argv)
+{
+	const char *delims = "# \t";
+	unsigned flags = 0;
+	int mintokens = 0, ntokens = 128;
+	opt_complementary = "-1:n+:m+:f+";
+	getopt32(argv, "n:m:d:f:", &ntokens, &mintokens, &delims, &flags);
+	//argc -= optind;
+	argv += optind;
+	while (*argv) {
+		parser_t *p = config_open(*argv);
+		if (p) {
+			int n;
+			char **t = xmalloc(sizeof(char *) * ntokens);
+			while ((n = config_read(p, t, ntokens, mintokens, delims, flags)) > 0) {
+				for (int i = 0; i < n; ++i)
+					printf("[%s]", t[i]);
+				puts("");
+			}
+			config_close(p);
+		}
+		argv++;
+	}
+	return EXIT_SUCCESS;
+}
+#endif
+
 /*
 
 Typical usage:
 
 ----- CUT -----
 	char *t[3];	// tokens placeholder
-	parser_t p;	// parser structure
+	parser_t *p = config_open(filename);
-	// open file
+	if (p) {
-	if (config_open(filename, &p)) {
 		// parse line-by-line
-		while (*config_read(&p, t, 3, 0, delimiters, comment_char) >= 0) { // 0..3 tokens
+		while (config_read(p, t, 3, 0, delimiters, flags)) { // 1..3 tokens
 			// use tokens
 			bb_error_msg("TOKENS: [%s][%s][%s]", t[0], t[1], t[2]);
 		}
 		...
 		// free parser
-		config_close(&p);
+		config_close(p);
 	}
 ----- CUT -----
 
@@ -35,44 +63,69 @@ parser_t* FAST_FUNC config_open(const char *filename)
 {
 	parser_t *parser = xzalloc(sizeof(parser_t));
 	/* empty file configures nothing */
-	parser->fp = fopen_or_warn(filename, "r");
+	parser->fp = fopen_or_warn_stdin(filename);
 	if (parser->fp)
 		return parser;
-	config_close (parser);
 	if (ENABLE_FEATURE_CLEAN_UP)
-	  free(parser);
+		free(parser);
 	return NULL;
 }
 
 static void config_free_data(parser_t *const parser)
 {
 	free(parser->line);
-	free(parser->data);
+	parser->line = NULL;
-	parser->line = parser->data = NULL;
+	USE_FEATURE_PARSE_COPY(
+		free(parser->data);
+		parser->data = NULL;
+	)
 }
+
 void FAST_FUNC config_close(parser_t *parser)
 {
 	config_free_data(parser);
 	fclose(parser->fp);
 }
 
-int FAST_FUNC config_read(parser_t *parser, char **tokens, int ntokens, int mintokens, const char *delims, char comment)
+/*
+1. Read a line from config file. If nothing to read then bail out returning 0.
+   Handle continuation character. Advance lineno for each physical line. Cut comments.
+2. if PARSE_DONT_TRIM is not set (default) skip leading and cut trailing delimiters, if any.
+3. If resulting line is empty goto 1.
+4. Look for first delimiter. If PARSE_DONT_REDUCE or PARSE_DONT_TRIM is set then pin empty token.
+5. Else (default) if number of seen tokens is equal to max number of tokens (token is the last one)
+   and PARSE_LAST_IS_GREEDY is set then pin the remainder of the line as the last token.
+   Else (token is not last or PARSE_LAST_IS_GREEDY is not set) just replace first delimiter with '\0'
+   thus delimiting token and pin it.
+6. Advance line pointer past the end of token. If number of seen tokens is less than required number
+   of tokens then goto 4.
+7. Control the number of seen tokens is not less the min number of tokens. Die if condition is not met.
+8. Return the number of seen tokens.
+
+mintokens > 0 make config_read() exit with error message if less than mintokens
+(but more than 0) are found. Empty lines are always skipped (not warned about).
+*/
+#undef config_read
+int FAST_FUNC config_read(parser_t *parser, char **tokens, unsigned flags, const char *delims)
 {
 	char *line, *q;
-	int ii, seen;
+	char comment = *delims++;
-	/* do not treat consecutive delimiters as one delimiter */
+	int ii;
-	bool noreduce = (ntokens < 0);
+	int ntokens = flags & 0xFF;
-	if (noreduce)
+	int mintokens = (flags & 0xFF00) >> 8;
-		ntokens = -ntokens;
 
-	memset(tokens, 0, sizeof(tokens[0]) * ntokens);
+	/*
+	// N.B. this could only be used in read-in-one-go version, or when tokens use xstrdup(). TODO
+	if (!parser->lineno || !(flags & PARSE_DONT_NULL))
+	*/
+		memset(tokens, 0, sizeof(tokens[0]) * ntokens);
 	config_free_data(parser);
 
 	while (1) {
 //TODO: speed up xmalloc_fgetline by internally using fgets, not fgetc
 		line = xmalloc_fgetline(parser->fp);
 		if (!line)
-			return -1;
+			return 0;
 
 		parser->lineno++;
 		// handle continuations. Tito's code stolen :)
@@ -98,12 +151,22 @@ int FAST_FUNC config_read(parser_t *parser, char **tokens, int ntokens, int mint
 			*q = '\0';
 			ii = q - line;
 		}
-		// skip leading delimiters
+		// skip leading and trailing delimiters
-		seen = strspn(line, delims);
+		if (!(flags & PARSE_DONT_TRIM)) {
-		if (seen) {
+			// skip leading
-			ii -= seen;
+			int n = strspn(line, delims);
-			strcpy(line, line + seen);
+			if (n) {
+				ii -= n;
+				strcpy(line, line + n);
+			}
+			// cut trailing
+			if (ii) {
+				while (strchr(delims, line[--ii]))
+					continue;
+				line[++ii] = '\0';
+			}
 		}
+		// if something still remains -> return it
 		if (ii)
 			break;
 
@@ -112,36 +175,39 @@ int FAST_FUNC config_read(parser_t *parser, char **tokens, int ntokens, int mint
 		free(line);
 	}
 
-	// non-empty line found, parse and return
+	// non-empty line found, parse and return the number of tokens
 
 	// store line
 	parser->line = line = xrealloc(line, ii + 1);
-	parser->data = xstrdup(line);
+	USE_FEATURE_PARSE_COPY(
+		parser->data = xstrdup(line);
+	)
 
 	/* now split line to tokens */
-	ii = noreduce ? seen : 0;
 	ntokens--; // now it's max allowed token no
-	while (1) {
+	// N.B, non-empty remainder is also a token,
+	// so if ntokens <= 1, we just return the whole line
+	// N.B. if PARSE_LAST_IS_GREEDY is set the remainder of the line is stuck to the last token
+	for (ii = 0; *line && ii <= ntokens; ) {
+		//bb_info_msg("L[%s]", line);
 		// get next token
-		if (ii == ntokens)
+		// at the last token and need greedy token ->
-			break;
+		if ((flags & PARSE_LAST_IS_GREEDY) && (ii == ntokens)) {
-		q = line + strcspn(line, delims);
+			// ... don't cut the line
-		if (!*q)
+			q = line + strlen(line);
-			break;
+		} else {
+			// vanilla token. cut the line at the first delim
+			q = line + strcspn(line, delims);
+			*q++ = '\0';
+		}
 		// pin token
-		*q++ = '\0';
+		if ((flags & (PARSE_DONT_REDUCE|PARSE_DONT_TRIM)) || *line) {
-		if (noreduce || *line) {
+			//bb_info_msg("N[%d] T[%s]", ii, line);
 			tokens[ii++] = line;
-//bb_info_msg("L[%d] T[%s]\n", ii, line);
 		}
 		line = q;
 	}
 
-	// non-empty remainder is also a token,
-	// so if ntokens <= 1, we just return the whole line
-	if (noreduce || *line)
-		tokens[ii++] = line;
-
 	if (ii < mintokens)
 		bb_error_msg_and_die("bad line %u: %d tokens found, %d needed",
 				parser->lineno, ii, mintokens);

miscutils/crond.c

@@ -469,12 +469,15 @@ static void SynchronizeFile(const char *fileName)
 		file->cf_User = xstrdup(fileName);
 		pline = &file->cf_LineBase;
 
-		while (--maxLines && (n=config_read(parser, tokens, 6, 0, " \t", '#')) >= 0) {
+		while (--maxLines
+		 && (n = config_read(parser, tokens, 6, 1, "# \t", PARSE_LAST_IS_GREEDY))
+		) {
 			CronLine *line;
 
-			if (DebugOpt) {
+			USE_FEATURE_PARSE_COPY(
-				crondlog(LVL5 "user:%s entry:%s", fileName, parser->data);
+				if (DebugOpt)
-			}
+					crondlog(LVL5 "user:%s entry:%s", fileName, parser->data);
+			)
 
 			/* check if line is setting MAILTO= */
 			if (0 == strncmp(tokens[0], "MAILTO=", 7)) {
@@ -485,7 +488,7 @@ static void SynchronizeFile(const char *fileName)
 				continue;
 			}
 			/* check if a minimum of tokens is specified */
-			if (n < 5)
+			if (n < 6)
 				continue;
 			*pline = line = xzalloc(sizeof(CronLine));
 			/* parse date ranges */

networking/nameif.c

@@ -163,7 +163,7 @@ int nameif_main(int argc, char **argv)
 		struct parser_t *parser = config_open(fname);
 		if (parser) {
 			char *tokens[2];
-			while (config_read(parser, tokens, 2, 2, " \t", '#') >= 0)
+			while (config_read(parser, tokens, 2, 2, "# \t", 0))
 				prepend_new_eth_table(&clist, tokens[0], tokens[1]);
 			config_close(parser);
 		}

testsuite/parse.tests

@@ -0,0 +1,64 @@
+#!/bin/sh
+
+# Copyright 2008 by Denys Vlasenko <vda.linux@googlemail.com>
+# Licensed under GPL v2, see file LICENSE for details.
+
+. testing.sh
+
+NO_REDUCE=65536
+NO_TRIM=131072
+GREEDY=262144
+
+# testing "description" "command" "result" "infile" "stdin"
+
+testing "mdev.conf" \
+	"parse -n 4 -m 3 -f $GREEDY -" \
+	"[sda][0:0][644][@echo @echo TEST]\n" \
+	"-" \
+	" sda 0:0 644 @echo @echo TEST # echo trap\n"
+
+testing "notrim" \
+	"parse -n 4 -m 3 -f $(($GREEDY+$NO_TRIM)) -" \
+	"[][sda][0:0][644 @echo @echo TEST ]\n" \
+	"-" \
+	" sda 0:0 644 @echo @echo TEST \n"
+
+FILE=__parse.fstab
+cat >$FILE <<EOF
+#
+# Device         Point               System                       Options
+#_______________________________________________________________
+/dev/hdb3       /                       ext2                 defaults      1          0
+   /dev/hdb1       /dosc               hpfs                 ro      1          0
+ /dev/fd0          /dosa              vfat                  rw,user,noauto,nohide	 0		0
+	/dev/fd1          /dosb              vfat                  rw,user,noauto,nohide	 0		0
+#
+ /dev/cdrom     /cdrom            iso9660          ro,user,noauto,nohide	 0		0
+/dev/hdb5       /redhat            ext2                 rw,root,noauto,nohide	 0		0 #sssd
+	/dev/hdb6       /win2home     ntfs                  rw,root,noauto,nohide	 0		0# ssdsd
+/dev/hdb7       /win2skul        ntfs                  rw,root,noauto,nohide none	 0		0
+none     /dev/pts           devpts             gid=5,mode=620                 0    0 
+     none                /proc               proc                defaults     0          0
+EOF
+
+cat >$FILE.res <<EOF
+[/dev/hdb3][/][ext2][defaults][1][0]
+[/dev/hdb1][/dosc][hpfs][ro][1][0]
+[/dev/fd0][/dosa][vfat][rw,user,noauto,nohide][0][0]
+[/dev/fd1][/dosb][vfat][rw,user,noauto,nohide][0][0]
+[/dev/cdrom][/cdrom][iso9660][ro,user,noauto,nohide][0][0]
+[/dev/hdb5][/redhat][ext2][rw,root,noauto,nohide][0][0]
+[/dev/hdb6][/win2home][ntfs][rw,root,noauto,nohide][0][0]
+[/dev/hdb7][/win2skul][ntfs][rw,root,noauto,nohide][none][0]
+[none][/dev/pts][devpts][gid=5,mode=620][0][0]
+[none][/proc][proc][defaults][0][0]
+EOF
+
+testing "polluted fstab" \
+	"parse -n 6 -m 6 $FILE" \
+	"`cat $FILE.res`\n" \
+	"" \
+	""
+rm -f $FILE $FILE.res
+
+exit $FAILCOUNT

util-linux/mdev.c

@@ -101,7 +101,7 @@ static void make_device(char *path, int delete)
 		if (!parser)
 			goto end_parse;
 
-		while (config_read(parser, tokens, 4, 3, " \t", '#') >= 0) {
+		while (config_read(parser, tokens, 4, 3, "# \t", PARSE_LAST_IS_GREEDY)) {
 			regmatch_t off[1+9*ENABLE_FEATURE_MDEV_RENAME_REGEXP];
 			char *val;