wget: implement -T SEC; rework progress meter to not use signals (it was unsafe)

function                                             old     new   delta
retrieve_file_data                                   364     450     +86
bb_progress_update                                   615     682     +67
packed_usage                                       27406   27422     +16
wget_main                                           2440    2453     +13
static.wget_longopts                                 145     155     +10
progress_meter                                       199     159     -40
------------------------------------------------------------------------------
(add/remove: 0/0 grow/shrink: 5/1 up/down: 192/-40)           Total: 152 bytes

Signed-off-by: Bradley M. Kuhn <bkuhn@ebb.org>
Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>

TEST_config_rh9

@@ -803,6 +803,7 @@ CONFIG_WGET=y
 CONFIG_FEATURE_WGET_STATUSBAR=y
 CONFIG_FEATURE_WGET_AUTHENTICATION=y
 CONFIG_FEATURE_WGET_LONG_OPTIONS=y
+CONFIG_FEATURE_WGET_TIMEOUT=y
 CONFIG_ZCIP=y
 
 #

include/usage.src.h

@@ -4779,10 +4779,12 @@ INSERT
 	IF_FEATURE_WGET_LONG_OPTIONS( \
        "[-c|--continue] [-s|--spider] [-q|--quiet] [-O|--output-document FILE]\n" \
        "	[--header 'header: value'] [-Y|--proxy on/off] [-P DIR]\n" \
-       "	[--no-check-certificate] [-U|--user-agent AGENT] URL" \
+       "	[--no-check-certificate] [-U|--user-agent AGENT]" \
+			IF_FEATURE_WGET_TIMEOUT("[-T SEC] ") " URL" \
 	) \
 	IF_NOT_FEATURE_WGET_LONG_OPTIONS( \
-       "[-csq] [-O FILE] [-Y on/off] [-P DIR] [-U AGENT] URL" \
+       "[-csq] [-O FILE] [-Y on/off] [-P DIR] [-U AGENT]" \
+			IF_FEATURE_WGET_TIMEOUT("[-T SEC] ") " URL" \
 	)
 #define wget_full_usage "\n\n" \
        "Retrieve files via HTTP or FTP\n" \
@@ -4790,7 +4792,10 @@ INSERT
      "\n	-s	Spider mode - only check file existence" \
      "\n	-c	Continue retrieval of aborted transfer" \
      "\n	-q	Quiet" \
-     "\n	-P	Set directory prefix to DIR" \
+     "\n	-P DIR	Save to DIR (default .)" \
+	IF_FEATURE_WGET_TIMEOUT( \
+     "\n	-T SEC	Network read timeout is SEC seconds" \
+	) \
      "\n	-O FILE	Save to FILE ('-' for stdout)" \
      "\n	-U STR	Use STR for User-Agent header" \
      "\n	-Y	Use proxy ('on' or 'off')" \

libbb/progress.c

@@ -66,16 +66,29 @@ void FAST_FUNC bb_progress_update(bb_progress_t *p,
 		off_t transferred,
 		off_t totalsize)
 {
-	off_t abbrevsize;
+	uoff_t beg_and_transferred;
 	unsigned since_last_update, elapsed;
 	unsigned ratio;
 	int barlength, i;
 
+	/* totalsize == 0 if it is unknown */
+
+	elapsed = monotonic_sec();
+	since_last_update = elapsed - p->lastupdate_sec;
+	/* Do not update on every call
+	 * (might be: on every network read!) */
+	if (since_last_update == 0 && !totalsize)
+			return;
+
+	beg_and_transferred = beg_range + transferred;
 	ratio = 100;
-	if (totalsize) {
+	if (beg_and_transferred < totalsize) {
+		/* Do not update on every call
+		 * (might be: on every network read!) */
+		if (since_last_update == 0)
+			return;
 		/* long long helps to have it working even if !LFS */
-		ratio = (unsigned) (100ULL * (transferred+beg_range) / totalsize);
+		ratio = 100ULL * beg_and_transferred / (uoff_t)totalsize;
-		if (ratio > 100) ratio = 100;
 	}
 
 #if ENABLE_UNICODE_SUPPORT
@@ -95,11 +108,11 @@ void FAST_FUNC bb_progress_update(bb_progress_t *p,
 		/* back to multibyte; cant overflow */
 		wcstombs(buf, wbuf21, INT_MAX);
 		len = (len > 20) ? 0 : 20 - len;
-		fprintf(stderr, "\r%s%*s%4d%% ", buf, len, "", ratio);
+		fprintf(stderr, "\r%s%*s%4u%% ", buf, len, "", ratio);
 		free(buf);
 	}
 #else
-	fprintf(stderr, "\r%-20.20s%4d%% ", curfile, ratio);
+	fprintf(stderr, "\r%-20.20s%4u%% ", curfile, ratio);
 #endif
 
 	barlength = get_tty2_width() - 49;
@@ -114,16 +127,14 @@ void FAST_FUNC bb_progress_update(bb_progress_t *p,
 		}
 	}
 	i = 0;
-	abbrevsize = transferred + beg_range;
+	while (beg_and_transferred >= 100000) {
-	while (abbrevsize >= 100000) {
 		i++;
-		abbrevsize >>= 10;
+		beg_and_transferred >>= 10;
 	}
 	/* see http://en.wikipedia.org/wiki/Tera */
-	fprintf(stderr, "%6d%c ", (int)abbrevsize, " kMGTPEZY"[i]);
+	fprintf(stderr, "%6u%c ", (unsigned)beg_and_transferred, " kMGTPEZY"[i]);
+#define beg_and_transferred dont_use_beg_and_transferred_below
 
-	elapsed = monotonic_sec();
-	since_last_update = elapsed - p->lastupdate_sec;
 	if (transferred > p->lastsize) {
 		p->lastupdate_sec = elapsed;
 		p->lastsize = transferred;

networking/Config.src

@@ -1012,6 +1012,18 @@ config FEATURE_WGET_LONG_OPTIONS
 	help
 	  Support long options for the wget applet.
 
+config FEATURE_WGET_TIMEOUT
+	bool "Enable timeout options"
+	default n
+	depends on WGET
+	help
+	  Supports network read timeout for wget, so that wget will give
+	  up and timeout when reading network data, through the -T command
+	  line option.  Currently only network data read timeout is
+	  supported (i.e., timeout is not applied to the DNS nor TCP
+	  connection initialization).  When FEATURE_WGET_LONG_OPTIONS is
+	  also enabled, the --timeout option will work in addition to -T.
+
 config ZCIP
 	bool "zcip"
 	default y

networking/wget.c

@@ -3,8 +3,10 @@
  * wget - retrieve a file using HTTP or FTP
  *
  * Chip Rosenthal Covad Communications <chip@laserlink.net>
- *
  * Licensed under GPLv2, see file LICENSE in this tarball for details.
+ *
+ * Copyright (C) 2010 Bradley M. Kuhn <bkuhn@ebb.org>
+ *   Kuhn's copyrights are licensed GPLv2-or-later.  File as a whole remains GPLv2.
  */
 #include "libbb.h"
 
@@ -19,7 +21,7 @@ struct host_info {
 };
 
 
-/* Globals (can be accessed from signal handlers) */
+/* Globals */
 struct globals {
 	off_t content_len;        /* Content-length of the file */
 	off_t beg_range;          /* Range at which continue begins */
@@ -27,6 +29,9 @@ struct globals {
 	off_t transferred;        /* Number of bytes transferred so far */
 	const char *curfile;      /* Name of current file being transferred */
 	bb_progress_t pmt;
+#endif
+#if ENABLE_FEATURE_WGET_TIMEOUT
+	unsigned timeout_seconds;
 #endif
 	smallint chunked;         /* chunked transfer encoding */
 	smallint got_clen;        /* got content-length: from server  */
@@ -35,42 +40,51 @@ struct globals {
 struct BUG_G_too_big {
 	char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1];
 };
-#define INIT_G() do { } while (0)
+#define INIT_G() do { \
+	IF_FEATURE_WGET_TIMEOUT(G.timeout_seconds = 900;) \
+} while (0)
 
 
+/* Must match option string! */
+enum {
+	WGET_OPT_CONTINUE   = (1 << 0),
+	WGET_OPT_SPIDER	    = (1 << 1),
+	WGET_OPT_QUIET      = (1 << 2),
+	WGET_OPT_OUTNAME    = (1 << 3),
+	WGET_OPT_PREFIX     = (1 << 4),
+	WGET_OPT_PROXY      = (1 << 5),
+	WGET_OPT_USER_AGENT = (1 << 6),
+	WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 7),
+	WGET_OPT_RETRIES    = (1 << 8),
+	WGET_OPT_PASSIVE    = (1 << 9),
+	WGET_OPT_HEADER     = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
+	WGET_OPT_POST_DATA  = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
+};
+
+enum {
+	PROGRESS_START = -1,
+	PROGRESS_END   = 0,
+	PROGRESS_BUMP  = 1,
+};
 #if ENABLE_FEATURE_WGET_STATUSBAR
-
 static void progress_meter(int flag)
 {
-	/* We can be called from signal handler */
+	if (option_mask32 & WGET_OPT_QUIET)
-	int save_errno = errno;
+		return;
 
-	if (flag == -1) { /* first call to progress_meter */
+	if (flag == PROGRESS_START)
 		bb_progress_init(&G.pmt);
-	}
 
 	bb_progress_update(&G.pmt, G.curfile, G.beg_range, G.transferred,
 			   G.chunked ? 0 : G.beg_range + G.transferred + G.content_len);
 
-	if (flag == 0) {
+	if (flag == PROGRESS_END) {
-		/* last call to progress_meter */
-		alarm(0);
 		bb_putchar_stderr('\n');
 		G.transferred = 0;
-	} else {
-		if (flag == -1) { /* first call to progress_meter */
-			signal_SA_RESTART_empty_mask(SIGALRM, progress_meter);
 	}
-		alarm(1);
 }
-
+#else
-	errno = save_errno;
-}
-
-#else /* FEATURE_WGET_STATUSBAR */
-
 static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
-
 #endif
 
 
@@ -430,28 +444,20 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_
 	return sfp;
 }
 
-/* Must match option string! */
-enum {
-	WGET_OPT_CONTINUE   = (1 << 0),
-	WGET_OPT_SPIDER	    = (1 << 1),
-	WGET_OPT_QUIET      = (1 << 2),
-	WGET_OPT_OUTNAME    = (1 << 3),
-	WGET_OPT_PREFIX     = (1 << 4),
-	WGET_OPT_PROXY      = (1 << 5),
-	WGET_OPT_USER_AGENT = (1 << 6),
-	WGET_OPT_RETRIES    = (1 << 7),
-	WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
-	WGET_OPT_PASSIVE    = (1 << 9),
-	WGET_OPT_HEADER     = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
-	WGET_OPT_POST_DATA  = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
-};
-
 static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
 {
 	char buf[512];
+#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
+# if ENABLE_FEATURE_WGET_TIMEOUT
+	unsigned second_cnt;
+# endif
+	struct pollfd polldata;
 
-	if (!(option_mask32 & WGET_OPT_QUIET))
+	polldata.fd = fileno(dfp);
-		progress_meter(-1);
+	polldata.events = POLLIN | POLLPRI;
+	ndelay(polldata.fd);
+#endif
+	progress_meter(PROGRESS_START);
 
 	if (G.chunked)
 		goto get_clen;
@@ -470,6 +476,23 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
 					rdsz = (unsigned)G.content_len;
 				}
 			}
+#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
+# if ENABLE_FEATURE_WGET_TIMEOUT
+			second_cnt = G.timeout_seconds;
+# endif
+			while (1) {
+				if (safe_poll(&polldata, 1, 1000) != 0)
+					break; /* error, EOF, or data is available */
+# if ENABLE_FEATURE_WGET_TIMEOUT
+				if (second_cnt != 0 && --second_cnt == 0) {
+					progress_meter(PROGRESS_END);
+					bb_perror_msg_and_die("download timed out");
+				}
+# endif
+				/* Needed for "stalled" indicator */
+				progress_meter(PROGRESS_BUMP);
+			}
+#endif
 			n = safe_fread(buf, rdsz, dfp);
 			if (n <= 0) {
 				if (ferror(dfp)) {
@@ -481,6 +504,7 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
 			xwrite(output_fd, buf, n);
 #if ENABLE_FEATURE_WGET_STATUSBAR
 			G.transferred += n;
+			progress_meter(PROGRESS_BUMP);
 #endif
 			if (G.got_clen)
 				G.content_len -= n;
@@ -499,8 +523,7 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
 		G.got_clen = 1;
 	}
 
-	if (!(option_mask32 & WGET_OPT_QUIET))
+	progress_meter(PROGRESS_END);
-		progress_meter(0);
 }
 
 int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
@@ -541,9 +564,11 @@ int wget_main(int argc UNUSED_PARAM, char **argv)
 		"directory-prefix\0" Required_argument "P"
 		"proxy\0"            Required_argument "Y"
 		"user-agent\0"       Required_argument "U"
+#if ENABLE_FEATURE_WGET_TIMEOUT
+		"timeout\0"          Required_argument "T"
+#endif
 		/* Ignored: */
 		// "tries\0"            Required_argument "t"
-		// "timeout\0"          Required_argument "T"
 		/* Ignored (we always use PASV): */
 		"passive-ftp\0"      No_argument       "\xff"
 		"header\0"           Required_argument "\xfe"
@@ -559,12 +584,12 @@ int wget_main(int argc UNUSED_PARAM, char **argv)
 	applet_long_options = wget_longopts;
 #endif
 	/* server.allocated = target.allocated = NULL; */
-	opt_complementary = "-1" IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::");
+	opt_complementary = "-1" IF_FEATURE_WGET_TIMEOUT(":T+") IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::");
-	opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:",
+	opt = getopt32(argv, "csqO:P:Y:U:T:" /*ignored:*/ "t:",
 				&fname_out, &dir_prefix,
 				&proxy_flag, &user_agent,
-				NULL, /* -t RETRIES */
+				IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
-				NULL /* -T NETWORK_READ_TIMEOUT */
+				NULL /* -t RETRIES */
 				IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
 				IF_FEATURE_WGET_LONG_OPTIONS(, &post_data)
 				);