cmd/{save,split}: add a --bwlimit option.

This allows you to limit how much upstream bandwidth 'bup save' and 'bup
split' will use.  Specify it as a number of bytes/second, or with the 'k' or
'M' or (lucky you!) 'G' suffixes for larger values.

Signed-off-by: Avery Pennarun <apenwarr@gmail.com>

diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py
index e840e557bba85e9bb87f5bc238cb3a750f37ff2c..59b9cb06d3dd0fe6c1dce77e399db63a39e8762f 100755 (executable)
--- a/cmd/save-cmd.py
+++ b/cmd/save-cmd.py
@@ -14,6 +14,7 @@ n,name=    name of backup set to update (if any)
 v,verbose  increase log output (can be used more than once)
 q,quiet    don't show progress meter
 smaller=   only back up files smaller than n bytes
+bwlimit=   maximum bytes/sec to transmit to server
 """
 o = options.Options('bup save', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -26,6 +27,8 @@ if not extra:
 
 opt.progress = (istty and not opt.quiet)
 opt.smaller = parse_num(opt.smaller or 0)
+if opt.bwlimit:
+    client.bwlimit = parse_num(opt.bwlimit)
 
 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
 if is_reverse and opt.remote:

diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py
index ff73c3f25fec0ddac8b6f66a1839cedf7fc6c3d5..b065892f12af0e1c3c6b6c66464bfb8f6957c30a 100755 (executable)
--- a/cmd/split-cmd.py
+++ b/cmd/split-cmd.py
@@ -20,7 +20,8 @@ copy       just copy input to output, hashsplitting along the way
 bench      print benchmark timings to stderr
 max-pack-size=  maximum bytes in a single pack
 max-pack-objects=  maximum number of objects in a single pack
-fanout=  maximum number of blobs in a single tree
+fanout=    maximum number of blobs in a single tree
+bwlimit=   maximum bytes/sec to transmit to server
 """
 o = options.Options('bup split', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -44,6 +45,8 @@ if opt.fanout:
     hashsplit.fanout = parse_num(opt.fanout)
 if opt.blobs:
     hashsplit.fanout = 0
+if opt.bwlimit:
+    client.bwlimit = parse_num(opt.bwlimit)
 
 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
 if is_reverse and opt.remote:

diff --git a/lib/bup/client.py b/lib/bup/client.py
index 32e0e0d1f319132a00f17b1b7b354b5edaf66821..e471cf59b0aac3199d577fe2b928480518450081 100644 (file)
--- a/lib/bup/client.py
+++ b/lib/bup/client.py
@@ -1,12 +1,37 @@
-import re, struct, errno, select
+import re, struct, errno, select, time
 from bup import git, ssh
 from bup.helpers import *
 
+bwlimit = None
+
 
 class ClientError(Exception):
     pass
 
 
+def _raw_write_bwlimit(f, buf, bwcount, bwtime):
+    if not bwlimit:
+        f.write(buf)
+        return (len(buf), time.time())
+    else:
+        # We want to write in reasonably large blocks, but not so large that
+        # they're likely to overflow a router's queue.  So our bwlimit timing
+        # has to be pretty granular.  Also, if it takes too long from one
+        # transmit to the next, we can't just make up for lost time to bring
+        # the average back up to bwlimit - that will risk overflowing the
+        # outbound queue, which defeats the purpose.  So if we fall behind
+        # by more than one block delay, we shouldn't ever try to catch up.
+        for i in xrange(0,len(buf),4096):
+            now = time.time()
+            next = max(now, bwtime + 1.0*bwcount/bwlimit)
+            time.sleep(next-now)
+            sub = buf[i:i+4096]
+            f.write(sub)
+            bwcount = len(sub)  # might be less than 4096
+            bwtime = next
+        return (bwcount, bwtime)
+                       
+
 class Client:
     def __init__(self, remote, create=False):
         self._busy = self.conn = self.p = self.pout = self.pin = None
@@ -208,6 +233,8 @@ class PackWriter_Remote(git.PackWriter):
         self.onclose = onclose
         self.ensure_busy = ensure_busy
         self._packopen = False
+        self._bwcount = 0
+        self._bwtime = time.time()
 
     def _open(self):
         if not self._packopen:
@@ -251,7 +278,9 @@ class PackWriter_Remote(git.PackWriter):
             self.ensure_busy()
         data = ''.join(datalist)
         assert(len(data))
-        self.file.write(struct.pack('!I', len(data)) + data)
+        outbuf = struct.pack('!I', len(data)) + data
+        (self._bwcount, self._bwtime) = \
+            _raw_write_bwlimit(self.file, outbuf, self._bwcount, self._bwtime)
         self.outbytes += len(data)
         self.count += 1