[prev in list] [next in list] [prev in thread] [next in thread]
List: busybox
Subject: [PATCH v2 2/2] wget: add support for retries in http requests
From: Martin Lewis <martin.lewis.x84 () gmail ! com>
Date: 2019-01-15 11:05:11
Message-ID: CA+nwWBvnJqS5iVZvSbwY0bu1rPFU3DkvwHLSy4=auW-25buSEA () mail ! gmail ! com
[Download RAW message or body]
[Attachment #2 (multipart/alternative)]
Replace die handlers with error returning so download_one_url can retry
from the beginning.
When retries is 1 (default) the behaviour should be the same as before.
v2: updated diff to mind the -o option, prettified help
Signed-off-by: Martin Lewis <martin.lewis.x84@gmail.com>
---
networking/wget.c | 125
++++++++++++++++++++++++++++++++++++++++++------------
1 file changed, 98 insertions(+), 27 deletions(-)
diff --git a/networking/wget.c b/networking/wget.c
index d11b201..b41f8ed 100644
--- a/networking/wget.c
+++ b/networking/wget.c
@@ -125,13 +125,14 @@
//usage: "[-c|--continue] [--spider] [-q|--quiet]
[-O|--output-document FILE]\n"
//usage: " [-o|--output-file FILE] [--header 'header: value']
[-Y|--proxy on/off]\n"
/* Since we ignore these opts, we don't show them in --help */
-/* //usage: " [--no-check-certificate] [--no-cache] [--passive-ftp]
[-t TRIES]" */
+/* //usage: " [--no-check-certificate] [--no-cache] [--passive-ftp]"
*/
/* //usage: " [-nv] [-nc] [-nH] [-np]" */
-//usage: " [-P DIR] [-S|--server-response] [-U|--user-agent
AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
+//usage: " [-t|--tries TRIES] [-P DIR] [-S|--server-response]\n"
+//usage: " [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T
SEC]") " URL..."
//usage: )
//usage: IF_NOT_FEATURE_WGET_LONG_OPTIONS(
-//usage: "[-cq] [-O FILE] [-o FILE] [-Y on/off] [-P DIR] [-S] [-U
AGENT]"
-//usage: IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
+//usage: "[-cq] [-O FILE] [-o FILE] [-Y on/off] [-P DIR] [-S] [-U
AGENT]\n"
+//usage: " [-t TRIES]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") "
URL..."
//usage: )
//usage:#define wget_full_usage "\n\n"
//usage: "Retrieve files via HTTP or FTP\n"
@@ -150,6 +151,7 @@
//usage: "\n -o FILE Log messages to FILE"
//usage: "\n -U STR Use STR for User-Agent header"
//usage: "\n -Y on/off Use proxy"
+//usage: "\n -t TRIES Set number of retries to TRIES (0
unlimits)"
#include "libbb.h"
@@ -235,6 +237,7 @@ struct globals {
char *fname_log; /* where to direct log (-o) */
const char *proxy_flag; /* Use proxies if env vars are set */
const char *user_agent; /* "User-Agent" header field */
+ unsigned tries; /* For -t option */
int output_fd;
int log_fd;
int o_flags;
@@ -410,6 +413,7 @@ static int is_ip_address(const char *string)
}
#endif
+/* Return NULL if connect() fails */
static FILE *open_socket(len_and_sockaddr *lsa)
{
int fd;
@@ -417,13 +421,19 @@ static FILE *open_socket(len_and_sockaddr *lsa)
#if ENABLE_FEATURE_WGET_TIMEOUT
struct timeval timeout = {G.timeout_seconds, 0};
#endif
+
fd = xsocket(lsa->u.sa.sa_family, SOCK_STREAM, 0);
#if ENABLE_FEATURE_WGET_TIMEOUT
if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof
(timeout)) < 0) {
bb_perror_msg_and_die("setsockopt failed\n");
}
#endif
- xconnect(fd, &lsa->u.sa, lsa->len);
+ if (connect(fd, &lsa->u.sa, lsa->len) < 0) {
+ /* Failure */
+ bb_perror_msg("connect failed");
+ close(fd);
+ return NULL;
+ }
/* glibc 2.4 seems to try seeking on it - ??! */
/* hopefully it understands what ESPIPE means... */
@@ -592,14 +602,16 @@ static int fread_buffered(char *buffer, size_t len,
FILE *fp)
return fread(buffer, 1, len, fp);
}
-/* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */
-static char fgets_trim_sanitize(FILE *fp, const char *fmt)
+/* Returns '\n' if it was seen, -1 if timeout occured, else '\0'. Trims at
first '\r' or '\n' */
+static signed char fgets_trim_sanitize(FILE *fp, const char *fmt)
{
char c;
char *buf_ptr;
- if (fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
- bb_perror_msg_and_die("error getting response");
+ if (fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
{
+ bb_perror_msg("error getting response");
+ return -1;
+ }
buf_ptr = strchrnul(G.wget_buf, '\n');
c = *buf_ptr;
@@ -637,7 +649,9 @@ static int ftpcmd(const char *s1, const char *s2, FILE
*fp)
/* Read until "Nxx something" is received */
G.wget_buf[3] = 0;
do {
- fgets_trim_sanitize(fp, "%s\n");
+ if (fgets_trim_sanitize(fp, "%s\n") == -1) {
+ xfunc_die();
+ }
} while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
G.wget_buf[3] = '\0';
@@ -742,6 +756,11 @@ static char *get_sanitized_hdr(FILE *fp)
/* retrieve header line */
c = fgets_trim_sanitize(fp, " %s\n");
+ if (c == -1) {
+ /* Timed out */
+ return NULL;
+ }
+
/* end of the headers? */
if (G.wget_buf[0] == '\0')
return NULL;
@@ -924,7 +943,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct
host_info *target, len_and_
char *pass;
int port;
+ /* TODO: Add retry support for ftp */
sfp = open_socket(lsa);
+ if (!sfp) {
+ xfunc_die();
+ }
#if ENABLE_FEATURE_WGET_HTTPS
if (target->protocol == P_FTPS)
spawn_ssl_client(target->host, fileno(sfp),
TLSLOOP_EXIT_ON_LOCAL_EOF);
@@ -979,7 +1002,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct
host_info *target, len_and_
set_nport(&lsa->u.sa, htons(port));
+ /* TODO: Add retry support for ftp */
*dfpp = open_socket(lsa);
+ if (!*dfpp) {
+ xfunc_die();
+ }
#if ENABLE_FEATURE_WGET_HTTPS
if (target->protocol == P_FTPS) {
@@ -1008,7 +1035,8 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct
host_info *target, len_and_
return sfp;
}
-static void NOINLINE retrieve_file_data(FILE *dfp)
+/* Return -1 if times out so we can retry */
+static int NOINLINE retrieve_file_data(FILE *dfp)
{
#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
# if ENABLE_FEATURE_WGET_TIMEOUT
@@ -1065,6 +1093,8 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
rdsz = (unsigned)G.content_len;
}
}
+ /* We probably have some data in fgets_buffer, so we need to
+ flush it first */
n = fread_buffered(G.wget_buf, rdsz, dfp);
if (n > 0) {
@@ -1105,7 +1135,7 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
# if ENABLE_FEATURE_WGET_TIMEOUT
if (second_cnt != 0 && --second_cnt == 0) {
progress_meter(PROGRESS_END);
- bb_error_msg_and_die("download timed out");
+ return -1;
}
# endif
/* We used to loop back to poll here,
@@ -1129,7 +1159,9 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
break;
/* Each chunk ends with "\r\n" - eat it */
- fgets_trim_sanitize(dfp, NULL);
+ if (fgets_trim_sanitize(dfp, NULL) == -1 ) {
+ return -1;
+ }
get_clen:
/* chunk size format is "HEXNUM[;name[=val]]\r\n" */
fgets_trim_sanitize(dfp, NULL);
@@ -1172,6 +1204,8 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
fprintf(stderr, "written to stdout\n");
else
fprintf(stderr, "'%s' saved\n", G.fname_out);
+
+ return 0;
}
static void download_one_url(const char *url)
@@ -1181,6 +1215,7 @@ static void download_one_url(const char *url)
len_and_sockaddr *lsa;
FILE *sfp; /* socket to web/ftp server */
FILE *dfp; /* socket to ftp server (data) */
+ unsigned cur_tries = 0; /* number of tries so far */
char *fname_out_alloc;
char *redirected_path = NULL;
struct host_info server;
@@ -1246,9 +1281,21 @@ static void download_one_url(const char *url)
* We are not sure it exists on remote side */
}
+ retry:
+ cur_tries++;
+ if (G.tries != 0 && cur_tries > G.tries) {
+ if (G.tries != 1) /* Show message about the tries only if was set
to more than one */
+ bb_error_msg_and_die("Gave up after %u tries", G.tries);
+ else
+ xfunc_die();
+ }
redir_limit = 5;
resolve_lsa:
- lsa = xhost2sockaddr(server.host, server.port);
+ /* If DNS resolution fails, retry, don't die */
+ lsa = host2sockaddr(server.host, server.port);
+ if (!lsa)
+ goto retry;
+
if (!(option_mask32 & WGET_OPT_QUIET)) {
char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
@@ -1278,6 +1325,8 @@ static void download_one_url(const char *url)
# if ENABLE_FEATURE_WGET_HTTPS
if (fd < 0) { /* no openssl? try internal */
sfp = open_socket(lsa);
+ if (!sfp)
+ goto retry;
spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0);
goto socket_opened;
}
@@ -1290,15 +1339,22 @@ static void download_one_url(const char *url)
goto socket_opened;
}
sfp = open_socket(lsa);
+ if (!sfp) {
+ goto retry;
+ }
socket_opened:
#elif ENABLE_FEATURE_WGET_HTTPS
/* Only internal TLS support is configured */
sfp = open_socket(lsa);
- if (server.protocol == P_HTTPS)
+ if (!sfp)
+ goto retry;
+ if (server.protocol == P_HTTPS) {
spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0);
#else
/* ssl (https) support is not configured */
sfp = open_socket(lsa);
+ if (!sfp)
+ goto retry;
#endif
/* Send HTTP request */
if (use_proxy) {
@@ -1374,7 +1430,11 @@ static void download_one_url(const char *url)
* Retrieve HTTP response line and check for "200" status code.
*/
read_response:
- fgets_trim_sanitize(sfp, " %s\n");
+ if (fgets_trim_sanitize(sfp, " %s\n") == -1) {
+ /* Timed out */
+ bb_error_msg("timed out");
+ goto retry;
+ }
str = G.wget_buf;
str = skip_non_whitespace(str);
@@ -1449,7 +1509,8 @@ However, in real world it was observed that some web
servers
/* Partial Content even though we did not ask for it??? */
/* fall through */
default:
- bb_error_msg_and_die("server returned error: %s", G.wget_buf);
+ bb_error_msg("server returned error: %s", G.wget_buf);
+ goto retry;
}
/*
@@ -1475,19 +1536,24 @@ However, in real world it was observed that some
web servers
if (key == KEY_content_length) {
G.content_len = BB_STRTOOFF(str, NULL, 10);
if (G.content_len < 0 || errno) {
- bb_error_msg_and_die("content-length %s is garbage",
str);
+ bb_error_msg("content-length %s is garbage", str);
+ goto retry;
}
G.got_clen = 1;
continue;
}
if (key == KEY_transfer_encoding) {
- if (strcmp(str_tolower(str), "chunked") != 0)
- bb_error_msg_and_die("transfer encoding '%s' is not
supported", str);
+ if (strcmp(str_tolower(str), "chunked") != 0) {
+ bb_error_msg("transfer encoding '%s' is not
supported", str);
+ goto retry;
+ }
G.chunked = 1;
}
if (key == KEY_location && status >= 300) {
- if (--redir_limit == 0)
- bb_error_msg_and_die("too many redirections");
+ if (--redir_limit == 0) {
+ bb_error_msg("too many redirections");
+ goto retry;
+ }
fclose(sfp);
if (str[0] == '/') {
free(redirected_path);
@@ -1526,13 +1592,18 @@ However, in real world it was observed that some
web servers
free(lsa);
if (!(option_mask32 & WGET_OPT_SPIDER)) {
+ int retrieve_retval;
if (G.output_fd < 0)
G.output_fd = xopen(G.fname_out, G.o_flags);
- retrieve_file_data(dfp);
+ retrieve_retval = retrieve_file_data(dfp);
if (!(option_mask32 & WGET_OPT_OUTNAME)) {
xclose(G.output_fd);
G.output_fd = -1;
}
+ if (retrieve_retval < 0) { /* We timed out, if we retry so the fd
is closed */
+ bb_error_msg("download timed out");
+ goto retry;
+ }
} else {
fprintf(stderr, "remote file exists\n");
}
@@ -1570,8 +1641,8 @@ int wget_main(int argc UNUSED_PARAM, char **argv)
"user-agent\0" Required_argument "U"
IF_FEATURE_WGET_TIMEOUT(
"timeout\0" Required_argument "T")
+ "tries\0" Required_argument "t"
/* Ignored: */
-IF_DESKTOP( "tries\0" Required_argument "t")
"header\0" Required_argument "\xff"
"post-data\0" Required_argument "\xfe"
"spider\0" No_argument "\xfd"
@@ -1601,14 +1672,14 @@ IF_DESKTOP( "no-parent\0"
No_argument "\xf0")
#if ENABLE_FEATURE_WGET_TIMEOUT
G.timeout_seconds = 900;
#endif
+ G.tries = 1;
G.proxy_flag = "on"; /* use proxies if env vars are set */
G.user_agent = "Wget"; /* "User-Agent" header field */
#if ENABLE_FEATURE_WGET_LONG_OPTIONS
#endif
GETOPT32(argv, "^"
- "cqSO:o:P:Y:U:T:+"
- /*ignored:*/ "t:"
+ "cqSO:o:P:Y:U:T:+t:+"
/*ignored:*/ "n::"
/* wget has exactly four -n<letter> opts, all of which we can
ignore:
* -nv --no-verbose: be moderately quiet (-q is full quiet)
@@ -1625,7 +1696,7 @@ IF_DESKTOP( "no-parent\0" No_argument
"\xf0")
, &G.fname_out, &G.fname_log, &G.dir_prefix,
&G.proxy_flag, &G.user_agent,
IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds)
IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
- NULL, /* -t RETRIES */
+ &G.tries, /* -t RETRIES */
NULL /* -n[ARG] */
IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
--
1.9.1
[Attachment #5 (text/html)]
<div dir="ltr"><div dir="ltr"><div dir="ltr">Replace die handlers with error \
returning so download_one_url can retry from the beginning.<br>When retries is 1 \
(default) the behaviour should be the same as before.<br><br>v2: updated diff to mind \
the -o option, prettified help<br><br>Signed-off-by: Martin Lewis <<a \
href="mailto:martin.lewis.x84@gmail.com">martin.lewis.x84@gmail.com</a>><br>---<br> \
networking/wget.c | 125 ++++++++++++++++++++++++++++++++++++++++++------------<br> 1 \
file changed, 98 insertions(+), 27 deletions(-)<br><br>diff --git a/networking/wget.c \
b/networking/wget.c<br>index d11b201..b41f8ed 100644<br>--- \
a/networking/wget.c<br>+++ b/networking/wget.c<br>@@ -125,13 +125,14 @@<br> //usage: \
"[-c|--continue] [--spider] [-q|--quiet] [-O|--output-document FILE]\n"<br> \
//usage: " [-o|--output-file FILE] [--header 'header: value'] \
[-Y|--proxy on/off]\n"<br> /* Since we ignore these opts, we don't show them \
in --help */<br>-/* //usage: " [--no-check-certificate] [--no-cache] \
[--passive-ftp] [-t TRIES]" */<br>+/* //usage: " \
[--no-check-certificate] [--no-cache] [--passive-ftp]" */<br> /* //usage: \
" [-nv] [-nc] [-nH] [-np]" */<br>-//usage: " [-P DIR] \
[-S|--server-response] [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" \
[-T SEC]") " URL..."<br>+//usage: " [-t|--tries TRIES] \
[-P DIR] [-S|--server-response]\n"<br>+//usage: " [-U|--user-agent \
AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."<br> \
//usage: )<br> //usage: IF_NOT_FEATURE_WGET_LONG_OPTIONS(<br>-//usage: \
"[-cq] [-O FILE] [-o FILE] [-Y on/off] [-P DIR] [-S] [-U \
AGENT]"<br>-//usage: IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") \
" URL..."<br>+//usage: "[-cq] [-O FILE] [-o FILE] [-Y on/off] \
[-P DIR] [-S] [-U AGENT]\n"<br>+//usage: " [-t TRIES]" \
IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."<br> //usage: \
)<br> //usage:#define wget_full_usage "\n\n"<br> //usage: \
"Retrieve files via HTTP or FTP\n"<br>@@ -150,6 +151,7 @@<br> //usage: \
"\n -o FILE Log messages to FILE"<br> //usage: "\n -U \
STR Use STR for User-Agent header"<br> //usage: "\n -Y on/off \
Use proxy"<br>+//usage: "\n -t TRIES Set number of retries to \
TRIES (0 unlimits)"<br> <br> #include "libbb.h"<br> <br>@@ -235,6 \
+237,7 @@ struct globals {<br> char *fname_log; /* where to direct log \
(-o) */<br> const char *proxy_flag; /* Use proxies if env vars are set */<br> \
const char *user_agent; /* "User-Agent" header field */<br>+ unsigned \
tries; /* For -t option */<br> int output_fd;<br> int log_fd;<br> int \
o_flags;<br>@@ -410,6 +413,7 @@ static int is_ip_address(const char *string)<br> \
}<br> #endif<br> <br>+/* Return NULL if connect() fails */<br> static FILE \
*open_socket(len_and_sockaddr *lsa)<br> {<br> int fd;<br>@@ -417,13 +421,19 @@ \
static FILE *open_socket(len_and_sockaddr *lsa)<br> #if \
ENABLE_FEATURE_WGET_TIMEOUT<br> struct timeval timeout = {G.timeout_seconds, \
0};<br> #endif<br>+<br> fd = xsocket(lsa->u.sa.sa_family, SOCK_STREAM, 0);<br> \
#if ENABLE_FEATURE_WGET_TIMEOUT<br> if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, \
&timeout, sizeof (timeout)) < 0) {<br> \
bb_perror_msg_and_die("setsockopt failed\n");<br> }<br> #endif<br>- \
xconnect(fd, &lsa-><a href="http://u.sa">u.sa</a>, lsa->len);<br>+ if \
(connect(fd, &lsa-><a href="http://u.sa">u.sa</a>, lsa->len) < 0) {<br>+ \
/* Failure */<br>+ bb_perror_msg("connect failed");<br>+ \
close(fd);<br>+ return NULL;<br>+ }<br> <br> /* glibc 2.4 seems to try \
seeking on it - ??! */<br> /* hopefully it understands what ESPIPE means... \
*/<br>@@ -592,14 +602,16 @@ static int fread_buffered(char *buffer, size_t len, FILE \
*fp)<br> return fread(buffer, 1, len, fp);<br> }<br> <br>-/* Returns '\n' \
if it was seen, else '\0'. Trims at first '\r' or '\n' \
*/<br>-static char fgets_trim_sanitize(FILE *fp, const char *fmt)<br>+/* Returns \
'\n' if it was seen, -1 if timeout occured, else '\0'. Trims at first \
'\r' or '\n' */<br>+static signed char fgets_trim_sanitize(FILE *fp, \
const char *fmt)<br> {<br> char c;<br> char *buf_ptr;<br> <br>- if \
(fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)<br>- \
bb_perror_msg_and_die("error getting response");<br>+ if \
(fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL) {<br>+ \
bb_perror_msg("error getting response");<br>+ return -1;<br>+ \
}<br> <br> buf_ptr = strchrnul(G.wget_buf, '\n');<br> c = \
*buf_ptr;<br>@@ -637,7 +649,9 @@ static int ftpcmd(const char *s1, const char *s2, \
FILE *fp)<br> /* Read until "Nxx something" is received */<br> \
G.wget_buf[3] = 0;<br> do {<br>- fgets_trim_sanitize(fp, \
"%s\n");<br>+ if (fgets_trim_sanitize(fp, "%s\n") == -1) \
{<br>+ xfunc_die();<br>+ }<br> } while (!isdigit(G.wget_buf[0]) \
|| G.wget_buf[3] != ' ');<br> <br> G.wget_buf[3] = '\0';<br>@@ \
-742,6 +756,11 @@ static char *get_sanitized_hdr(FILE *fp)<br> /* retrieve header \
line */<br> c = fgets_trim_sanitize(fp, " %s\n");<br> <br>+ if (c \
== -1) {<br>+ /* Timed out */<br>+ return NULL;<br>+ }<br>+<br> \
/* end of the headers? */<br> if (G.wget_buf[0] == '\0')<br> \
return NULL;<br>@@ -924,7 +943,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, \
struct host_info *target, len_and_<br> char *pass;<br> int port;<br> <br>+ \
/* TODO: Add retry support for ftp */<br> sfp = open_socket(lsa);<br>+ if \
(!sfp) {<br>+ xfunc_die();<br>+ }<br> #if ENABLE_FEATURE_WGET_HTTPS<br> \
if (target->protocol == P_FTPS)<br> spawn_ssl_client(target->host, \
fileno(sfp), TLSLOOP_EXIT_ON_LOCAL_EOF);<br>@@ -979,7 +1002,11 @@ static FILE* \
prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_<br> <br> \
set_nport(&lsa-><a href="http://u.sa">u.sa</a>, htons(port));<br> <br>+ /* \
TODO: Add retry support for ftp */<br> *dfpp = open_socket(lsa);<br>+ if \
(!*dfpp) {<br>+ xfunc_die();<br>+ }<br> <br> #if \
ENABLE_FEATURE_WGET_HTTPS<br> if (target->protocol == P_FTPS) {<br>@@ -1008,7 \
+1035,8 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, \
len_and_<br> return sfp;<br> }<br> <br>-static void NOINLINE \
retrieve_file_data(FILE *dfp)<br>+/* Return -1 if times out so we can retry \
*/<br>+static int NOINLINE retrieve_file_data(FILE *dfp)<br> {<br> #if \
ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT<br> # if \
ENABLE_FEATURE_WGET_TIMEOUT<br>@@ -1065,6 +1093,8 @@ static void NOINLINE \
retrieve_file_data(FILE *dfp)<br> rdsz = \
(unsigned)G.content_len;<br> }<br> }<br>+ /* \
We probably have some data in fgets_buffer, so we need to<br>+ flush it \
first */<br> n = fread_buffered(G.wget_buf, rdsz, dfp);<br> <br> \
if (n > 0) {<br>@@ -1105,7 +1135,7 @@ static void NOINLINE retrieve_file_data(FILE \
*dfp)<br> # if ENABLE_FEATURE_WGET_TIMEOUT<br> if (second_cnt != 0 \
&& --second_cnt == 0) {<br> \
progress_meter(PROGRESS_END);<br>- \
bb_error_msg_and_die("download timed out");<br>+ return \
-1;<br> }<br> # endif<br> /* We used to loop back to \
poll here,<br>@@ -1129,7 +1159,9 @@ static void NOINLINE retrieve_file_data(FILE \
*dfp)<br> break;<br> <br> /* Each chunk ends with \
"\r\n" - eat it */<br>- fgets_trim_sanitize(dfp, NULL);<br>+ \
if (fgets_trim_sanitize(dfp, NULL) == -1 ) {<br>+ return -1;<br>+ \
}<br> get_clen:<br> /* chunk size format is \
"HEXNUM[;name[=val]]\r\n" */<br> fgets_trim_sanitize(dfp, \
NULL);<br>@@ -1172,6 +1204,8 @@ static void NOINLINE retrieve_file_data(FILE \
*dfp)<br> fprintf(stderr, "written to stdout\n");<br> else<br> \
fprintf(stderr, "'%s' saved\n", G.fname_out);<br>+<br>+ return \
0;<br> }<br> <br> static void download_one_url(const char *url)<br>@@ -1181,6 +1215,7 \
@@ static void download_one_url(const char *url)<br> len_and_sockaddr *lsa;<br> \
FILE *sfp; /* socket to web/ftp server */<br> FILE \
*dfp; /* socket to ftp server (data) */<br>+ unsigned \
cur_tries = 0; /* number of tries so far */<br> char \
*fname_out_alloc;<br> char *redirected_path = NULL;<br> struct host_info \
server;<br>@@ -1246,9 +1281,21 @@ static void download_one_url(const char *url)<br> \
* We are not sure it exists on remote side */<br> }<br> <br>+ retry:<br>+ \
cur_tries++;<br>+ if (G.tries != 0 && cur_tries > G.tries) {<br>+ \
if (G.tries != 1) /* Show message about the tries only if was set to more than one \
*/<br>+ bb_error_msg_and_die("Gave up after %u tries", \
G.tries);<br>+ else<br>+ xfunc_die();<br>+ }<br> redir_limit \
= 5;<br> resolve_lsa:<br>- lsa = xhost2sockaddr(server.host, server.port);<br>+ \
/* If DNS resolution fails, retry, don't die */<br>+ lsa = \
host2sockaddr(server.host, server.port);<br>+ if (!lsa)<br>+ goto \
retry;<br>+<br> if (!(option_mask32 & WGET_OPT_QUIET)) {<br> char *s \
= xmalloc_sockaddr2dotted(&lsa-><a href="http://u.sa">u.sa</a>);<br> \
fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);<br>@@ -1278,6 \
+1325,8 @@ static void download_one_url(const char *url)<br> # if \
ENABLE_FEATURE_WGET_HTTPS<br> if (fd < 0) { /* no openssl? try \
internal */<br> sfp = open_socket(lsa);<br>+ if \
(!sfp)<br>+ goto retry;<br> \
spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0);<br> goto \
socket_opened;<br> }<br>@@ -1290,15 +1339,22 @@ static void \
download_one_url(const char *url)<br> goto socket_opened;<br> \
}<br> sfp = open_socket(lsa);<br>+ if (!sfp) {<br>+ goto \
retry;<br>+ }<br> socket_opened:<br> #elif ENABLE_FEATURE_WGET_HTTPS<br> \
/* Only internal TLS support is configured */<br> sfp = \
open_socket(lsa);<br>- if (server.protocol == P_HTTPS)<br>+ if \
(!sfp)<br>+ goto retry;<br>+ if (server.protocol == P_HTTPS) {<br> \
spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0);<br> #else<br> /* ssl \
(https) support is not configured */<br> sfp = open_socket(lsa);<br>+ \
if (!sfp)<br>+ goto retry;<br> #endif<br> /* Send HTTP request \
*/<br> if (use_proxy) {<br>@@ -1374,7 +1430,11 @@ static void \
download_one_url(const char *url)<br> * Retrieve HTTP response line and \
check for "200" status code.<br> */<br> read_response:<br>- \
fgets_trim_sanitize(sfp, " %s\n");<br>+ if \
(fgets_trim_sanitize(sfp, " %s\n") == -1) {<br>+ /* Timed out \
*/<br>+ bb_error_msg("timed out");<br>+ goto \
retry;<br>+ }<br> <br> str = G.wget_buf;<br> str = \
skip_non_whitespace(str);<br>@@ -1449,7 +1509,8 @@ However, in real world it was \
observed that some web servers<br> /* Partial Content even though we did \
not ask for it??? */<br> /* fall through */<br> default:<br>- \
bb_error_msg_and_die("server returned error: %s", G.wget_buf);<br>+ \
bb_error_msg("server returned error: %s", G.wget_buf);<br>+ goto \
retry;<br> }<br> <br> /*<br>@@ -1475,19 +1536,24 @@ However, in real \
world it was observed that some web servers<br> if (key == \
KEY_content_length) {<br> G.content_len = BB_STRTOOFF(str, NULL, \
10);<br> if (G.content_len < 0 || errno) {<br>- \
bb_error_msg_and_die("content-length %s is garbage", str);<br>+ \
bb_error_msg("content-length %s is garbage", str);<br>+ \
goto retry;<br> }<br> G.got_clen = 1;<br> \
continue;<br> }<br> if (key == KEY_transfer_encoding) {<br>- \
if (strcmp(str_tolower(str), "chunked") != 0)<br>- \
bb_error_msg_and_die("transfer encoding '%s' is not supported", \
str);<br>+ if (strcmp(str_tolower(str), "chunked") != 0) \
{<br>+ bb_error_msg("transfer encoding '%s' is not \
supported", str);<br>+ goto retry;<br>+ }<br> \
G.chunked = 1;<br> }<br> if (key == KEY_location && \
status >= 300) {<br>- if (--redir_limit == 0)<br>- \
bb_error_msg_and_die("too many redirections");<br>+ if \
(--redir_limit == 0) {<br>+ bb_error_msg("too many \
redirections");<br>+ goto retry;<br>+ }<br> \
fclose(sfp);<br> if (str[0] == '/') {<br> \
free(redirected_path);<br>@@ -1526,13 +1592,18 @@ However, in real world it was \
observed that some web servers<br> free(lsa);<br> <br> if (!(option_mask32 \
& WGET_OPT_SPIDER)) {<br>+ int retrieve_retval;<br> if \
(G.output_fd < 0)<br> G.output_fd = xopen(G.fname_out, \
G.o_flags);<br>- retrieve_file_data(dfp);<br>+ retrieve_retval = \
retrieve_file_data(dfp);<br> if (!(option_mask32 & WGET_OPT_OUTNAME)) \
{<br> xclose(G.output_fd);<br> G.output_fd = -1;<br> \
}<br>+ if (retrieve_retval < 0) { /* We timed out, if we retry so the fd is \
closed */<br>+ bb_error_msg("download timed out");<br>+ \
goto retry;<br>+ }<br> } else {<br> fprintf(stderr, "remote \
file exists\n");<br> }<br>@@ -1570,8 +1641,8 @@ int wget_main(int argc \
UNUSED_PARAM, char **argv)<br> "user-agent\0" \
Required_argument "U"<br> IF_FEATURE_WGET_TIMEOUT(<br> \
"timeout\0" Required_argument "T")<br>+ \
"tries\0" Required_argument "t"<br> /* \
Ignored: */<br>-IF_DESKTOP( "tries\0" Required_argument \
"t")<br> "header\0" Required_argument \
"\xff"<br> "post-data\0" Required_argument \
"\xfe"<br> "spider\0" No_argument \
"\xfd"<br>@@ -1601,14 +1672,14 @@ IF_DESKTOP( "no-parent\0" \
No_argument "\xf0")<br> #if ENABLE_FEATURE_WGET_TIMEOUT<br> \
G.timeout_seconds = 900;<br> #endif<br>+ G.tries = 1;<br> G.proxy_flag = \
"on"; /* use proxies if env vars are set */<br> G.user_agent = \
"Wget"; /* "User-Agent" header field */<br> <br> #if \
ENABLE_FEATURE_WGET_LONG_OPTIONS<br> #endif<br> GETOPT32(argv, "^"<br>- \
"cqSO:o:P:Y:U:T:+"<br>- /*ignored:*/ "t:"<br>+ \
"cqSO:o:P:Y:U:T:+t:+"<br> /*ignored:*/ "n::"<br> \
/* wget has exactly four -n<letter> opts, all of which we can ignore:<br> \
* -nv --no-verbose: be moderately quiet (-q is full quiet)<br>@@ -1625,7 +1696,7 @@ \
IF_DESKTOP( "no-parent\0" No_argument "\xf0")<br> \
_______________________________________________
busybox mailing list
busybox@busybox.net
http://lists.busybox.net/mailman/listinfo/busybox
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic