fsck: add a -j# (run multiple threads) option.

Sort of like make -j.  par2 can be pretty slow, so this lets us verify
multiple files in parallel.  Since the files are so big, though, this might
actually make performance *worse* if you don't have a lot of RAM.  I haven't
benchmarked this too much, but on my quad-core with 6 gigs of RAM, -j4 does
definitely make it go "noticeably" faster.

diff --git a/cmd-fsck.py b/cmd-fsck.py
index 411e499a9c85b15018a5cb5057e9a75a47cec53c..1818a9f0eec1343a276f7cb968ca7a14c8964633 100755 (executable)
--- a/cmd-fsck.py
+++ b/cmd-fsck.py
@@ -78,6 +78,39 @@ def git_verify(base):
         return 0
     else:
         return run(['git', 'verify-pack', '--', base])
+    
+    
+def do_pack(base, last):
+    code = 0
+    if par2_ok and par2_exists and (opt.repair or not opt.generate):
+        vresult = par2_verify(base)
+        if vresult != 0:
+            if opt.repair:
+                rresult = par2_repair(base)
+                if rresult != 0:
+                    print '%s par2 repair: failed (%d)' % (last, rresult)
+                    code = rresult
+                else:
+                    print '%s par2 repair: succeeded (0)' % last
+                    code = 100
+            else:
+                print '%s par2 verify: failed (%d)' % (last, vresult)
+                code = vresult
+        else:
+            print '%s ok' % last
+    elif not opt.generate or (par2_ok and not par2_exists):
+        gresult = git_verify(base)
+        if gresult != 0:
+            print '%s git verify: failed (%d)' % (last, gresult)
+            code = gresult
+        else:
+            if par2_ok and opt.generate:
+                par2_generate(base)
+            print '%s ok' % last
+    else:
+        assert(opt.generate and (not par2_ok or par2_exists))
+        debug('    skipped: par2 file already generated.\n')
+    return code
 
 
 optspec = """
@@ -87,6 +120,7 @@ r,repair    attempt to repair errors using par2 (dangerous!)
 g,generate  generate auto-repair information using par2
 v,verbose   increase verbosity (can be used more than once)
 quick       just check pack sha1sum, don't use git verify-pack
+j,jobs=     run 'n' jobs in parallel
 par2-ok     immediately return 0 if par2 is ok, 1 if not
 disable-par2  ignore par2 even if it is available
 """
@@ -110,6 +144,7 @@ if not extra:
 
 code = 0
 count = 0
+outstanding = {}
 for name in extra:
     if name.endswith('.pack'):
         base = name[:-5]
@@ -130,36 +165,38 @@ for name in extra:
           % (last, par2_ok and par2_exists and 'par2' or 'git'))
     if not opt.verbose and istty:
         log('fsck (%d/%d)\r' % (count, len(extra)))
-
-    if par2_ok and par2_exists and (opt.repair or not opt.generate):
-        vresult = par2_verify(base)
-        if vresult != 0:
-            if opt.repair:
-                rresult = par2_repair(base)
-                if rresult != 0:
-                    print '%s par2 repair: failed (%d)' % (last, rresult)
-                    code = code or rresult
-                else:
-                    print '%s par2 repair: succeeded (0)' % last
-                    code = code or 100
-            else:
-                print '%s par2 verify: failed (%d)' % (last, vresult)
-                code = code or vresult
-        else:
-            print '%s ok' % last
-    elif not opt.generate or (par2_ok and not par2_exists):
-        gresult = git_verify(base)
-        if gresult != 0:
-            print '%s git verify: failed (%d)' % (last, gresult)
-            code = code or gresult
-        else:
-            if par2_ok and opt.generate:
-                par2_generate(base)
-            print '%s ok' % last
+    
+    if not opt.jobs:
+        nc = do_pack(base, last)
+        code = code or nc
+        count += 1
     else:
-        assert(opt.generate and (not par2_ok or par2_exists))
-        debug('    skipped: par2 file already generated.\n')
-    count += 1
+        while len(outstanding) >= opt.jobs:
+            (pid,nc) = os.wait()
+            nc >>= 8
+            if pid in outstanding:
+                del outstanding[pid]
+                code = code or nc
+                count += 1
+        pid = os.fork()
+        if pid:  # parent
+            outstanding[pid] = 1
+        else: # child
+            try:
+                sys.exit(do_pack(base, last))
+            except Exception, e:
+                log('exception: %r\n' % e)
+                sys.exit(99)
+                
+while len(outstanding):
+    (pid,nc) = os.wait()
+    nc >>= 8
+    if pid in outstanding:
+        del outstanding[pid]
+        code = code or nc
+        count += 1
+    if not opt.verbose and istty:
+        log('fsck (%d/%d)\r' % (count, len(extra)))
 
 if not opt.verbose and istty:
     log('fsck done.           \n')

diff --git a/t/test.sh b/t/test.sh
index fa1afaa36988bfa3fa5d659bbb43f923194752c8..eade5f89a593d0a5b320d118fe46ae56f708a455 100755 (executable)
--- a/t/test.sh
+++ b/t/test.sh
@@ -115,10 +115,10 @@ WVFAIL bup fsck
 WVFAIL bup fsck --disable-par2
 chmod u+w $BUP_DIR/objects/pack/*.idx
 WVPASS bup damage $BUP_DIR/objects/pack/*.idx -n10 -s1 -S0
-WVFAIL bup fsck
+WVFAIL bup fsck -j4
 WVPASS bup damage $BUP_DIR/objects/pack/*.pack -n10 -s1024 --percent 0.4 -S0
 WVFAIL bup fsck
-WVFAIL bup fsck -rvv   # fails because repairs were needed
+WVFAIL bup fsck -rvv -j99   # fails because repairs were needed
 if bup fsck --par2-ok; then
     WVPASS bup fsck -r # ok because of repairs from last time
     WVPASS bup damage $BUP_DIR/objects/pack/*.pack -n201 -s1 --equal -S0