[prev in list] [next in list] [prev in thread] [next in thread] 

List:       wget
Subject:    Throttle patch for wget 1.5.3
From:       Keith Winstein <keithw () mit ! edu>
Date:       2000-02-22 14:48:02
[Download RAW message or body]

Attached is a patch to do receive throttling in wget 1.5.3. The throttling
is implemented by playing games with the MSS and window sizes, and
there are some subtle blocking and silly-window issues that I don't
fully understand. I also haven't tested the patch extensively and so I
wouldn't be that surprised if it broke compilation on some platforms.
However, it does seem to work quite well for me and it would be pretty
cool if it made it into the main distribution.

Keith Winstein
keithw@mit.edu


diff -urN wget-1.5.3/src/ChangeLog wget-1.5.3-throttle/src/ChangeLog
--- wget-1.5.3/src/ChangeLog	Mon Sep 21 13:01:05 1998
+++ wget-1.5.3-throttle/src/ChangeLog	Mon Feb 21 17:10:10 2000
@@ -1,3 +1,8 @@
+2000-02-21  Keith Winstein <keithw@mit.edu>
+
+	* connect.c, init.c, main.c, options.h, retr.c: Added support
+	for --kbpersec (throttle) option
+
 1998-09-21  Hrvoje Niksic  <hniksic@srce.hr>
 
 	* version.c: Wget 1.5.3 is released.
diff -urN wget-1.5.3/src/connect.c wget-1.5.3-throttle/src/connect.c
--- wget-1.5.3/src/connect.c	Sun Mar 29 16:28:15 1998
+++ wget-1.5.3-throttle/src/connect.c	Mon Feb 21 17:02:26 2000
@@ -28,6 +28,7 @@
 #ifdef WINDOWS
 # include <winsock.h>
 #else
+# include <netinet/tcp.h>
 # include <sys/socket.h>
 # include <netdb.h>
 # include <netinet/in.h>
@@ -86,6 +87,14 @@
   /* Make an internet socket, stream type.  */
   if ((*sock = socket (AF_INET, SOCK_STREAM, 0)) == -1)
     return CONSOCKERR;
+
+  /* Throttle, if necessary */
+  if (opt.kb_per_sec) {
+    int rcvbuf = 512;
+    int maxseg = 512;
+    setsockopt( *sock, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf) );
+    setsockopt( *sock, SOL_TCP, TCP_MAXSEG, &maxseg, sizeof(maxseg) );
+  }
 
   /* Connect the socket to the remote host.  */
   if (connect (*sock, (struct sockaddr *) &sock_name, sizeof (sock_name)))
diff -urN wget-1.5.3/src/init.c wget-1.5.3-throttle/src/init.c
--- wget-1.5.3/src/init.c	Thu Sep 10 20:20:23 1998
+++ wget-1.5.3-throttle/src/init.c	Mon Feb 21 17:02:26 2000
@@ -115,6 +115,7 @@
   { "ignorelength",	&opt.ignore_length,	cmd_boolean },
   { "includedirectories", &opt.includes,	cmd_directory_vector },
   { "input",		&opt.input_filename,	cmd_string },
+  { "kbpersec",        &opt.kb_per_sec,        cmd_number },
   { "killlonger",	&opt.kill_longer,	cmd_boolean },
   { "logfile",		&opt.lfilename,		cmd_string },
   { "login",		&opt.ftp_acc,		cmd_string },
diff -urN wget-1.5.3/src/main.c wget-1.5.3-throttle/src/main.c
--- wget-1.5.3/src/main.c	Thu Sep 10 21:41:53 1998
+++ wget-1.5.3-throttle/src/main.c	Mon Feb 21 17:02:26 2000
@@ -143,6 +143,7 @@
   -T,  --timeout=SECONDS        set the read timeout to SECONDS.\n\
   -w,  --wait=SECONDS           wait SECONDS between retrievals.\n\
   -Y,  --proxy=on/off           turn proxy on or off.\n\
+  -K,  --kbpersec=NUMBER        number of throttled kilobytes per second.\n\
   -Q,  --quota=NUMBER           set retrieval quota to NUMBER.\n\
 \n"),  _("\
 Directories:\n\
@@ -265,6 +266,7 @@
     { "user-agent", required_argument, NULL, 'U' },
     { "use-proxy", required_argument, NULL, 'Y' },
     { "wait", required_argument, NULL, 'w' },
+    { "kbpersec", required_argument, NULL, 'K' },
     { 0, 0, 0, 0 }
   };
 
@@ -286,7 +288,7 @@
   initialize ();
 
   while ((c = getopt_long (argc, argv, "\
-hVqvdksxmNWrHSLcFbEY:g:T:U:O:l:n:i:o:a:t:D:A:R:P:B:e:Q:X:I:w:",
+hVqvdksxmNWrHSLcFbEY:g:T:U:O:l:n:i:o:a:t:D:A:R:P:B:e:Q:X:I:w:K:",
 			   long_options, (int *)0)) != EOF)
     {
       switch (c)
@@ -546,6 +548,9 @@
 	  break;
 	case 'w':
 	  setval ("wait", optarg);
+	  break;
+	case 'K':
+	  setval ("kbpersec", optarg);
 	  break;
 	case 'X':
 	  setval ("excludedirectories", optarg);
diff -urN wget-1.5.3/src/options.h wget-1.5.3-throttle/src/options.h
--- wget-1.5.3/src/options.h	Tue Apr 28 17:29:40 1998
+++ wget-1.5.3-throttle/src/options.h	Mon Feb 21 17:02:26 2000
@@ -132,6 +132,9 @@
 
   int delete_after;		/* Whether the files will be deleted
 				   after download. */
+
+  int kb_per_sec;               /* The number of throttled kilobytes
+                                   per second */
 };
 
 #ifndef OPTIONS_DEFINED_HERE
diff -urN wget-1.5.3/src/retr.c wget-1.5.3-throttle/src/retr.c
--- wget-1.5.3/src/retr.c	Sat May 16 21:08:38 1998
+++ wget-1.5.3-throttle/src/retr.c	Mon Feb 21 17:18:18 2000
@@ -22,6 +22,15 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/types.h>
+
+#ifdef WINDOWS
+# include <winsock.h>
+#else
+# include <sys/socket.h>
+# include <netdb.h>
+# include <netinet/in.h>
+# include <arpa/inet.h>
+#endif /* WINDOWS */
 #ifdef HAVE_UNISTD_H
 # include <unistd.h>
 #endif /* HAVE_UNISTD_H */
@@ -81,15 +90,24 @@
 get_contents (int fd, FILE *fp, long *len, long restval, long expected,
 	      struct rbuf *rbuf)
 {
-  int res;
+  int res, sres, len_init;
   static char c[8192];
+  struct timeval start_time, this_read;
+  struct timespec sleep_spec, sleep_temp;
+  double seconds_elapsed, sleep_seconds;
+  int rcvbuf = opt.kb_per_sec ?
+    THROTTLE_GRAIN : sizeof(c); /* Use smaller buffer for
+				   better granularity if
+				   throttle requested. */
 
+  gettimeofday( &start_time, NULL );
+  
   *len = restval;
   if (opt.verbose)
     show_progress (restval, expected, SP_INIT);
   if (rbuf && RBUF_FD (rbuf) == fd)
     {
-      while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
+      while ((res = rbuf_flush (rbuf, c, rcvbuf)) != 0)
 	{
 	  if (fwrite (c, sizeof (char), res, fp) < res)
 	    return -2;
@@ -101,22 +119,43 @@
 	  *len += res;
 	}
     }
+
+  len_init = *len;
   /* Read from fd while there is available data.  */
-  do
-    {
-      res = iread (fd, c, sizeof (c));
-      if (res > 0)
-	{
-	  if (fwrite (c, sizeof (char), res, fp) < res)
-	    return -2;
-	  if (opt.verbose)
-	    {
-	      if (show_progress (res, expected, SP_NONE))
-		fflush (fp);
-	    }
-	  *len += res;
-	}
-    } while (res > 0);
+  do {
+    if (opt.kb_per_sec) {
+      gettimeofday( &this_read, NULL );
+      seconds_elapsed = this_read.tv_sec - start_time.tv_sec
+	+ ((float) (this_read.tv_usec - start_time.tv_usec)) / 1000000.0;
+      
+      if (1024 * opt.kb_per_sec * seconds_elapsed < *len - len_init) {
+	sleep_seconds = ((float)(*len - len_init) / (1024 * opt.kb_per_sec)
+			 - seconds_elapsed);
+	sleep_temp.tv_sec = (int)sleep_seconds;
+	sleep_temp.tv_nsec = 1000000000 * (sleep_seconds - sleep_temp.tv_sec);
+	
+	do {
+	  memcpy( &sleep_spec, &sleep_temp, sizeof( struct timespec ) );
+	  sres = nanosleep( &sleep_spec, &sleep_temp );
+	} while( sres == -1 && errno == EINTR );
+      }
+      rcvbuf = THROTTLE_GRAIN;
+      setsockopt( fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf) );
+    }
+    res = iread (fd, c, rcvbuf);
+    if (res > 0)
+      {
+	if (fwrite (c, sizeof (char), res, fp) < res)
+	  return -2;
+	if (opt.verbose)
+	  {
+	    if (show_progress (res, expected, SP_NONE))
+	      fflush (fp);
+	  }
+	*len += res;
+	
+      }
+  } while (res > 0);
   if (res < -1)
     res = -1;
   if (opt.verbose)
diff -urN wget-1.5.3/src/retr.h wget-1.5.3-throttle/src/retr.h
--- wget-1.5.3/src/retr.h	Fri Jan 30 20:22:34 1998
+++ wget-1.5.3-throttle/src/retr.h	Mon Feb 21 17:17:46 2000
@@ -34,4 +34,6 @@
 
 void printwhat PARAMS ((int, int));
 
+#define THROTTLE_GRAIN 512
+
 #endif /* RETR_H */


[prev in list] [next in list] [prev in thread] [next in thread]