cmd-midx: add --auto and --force options.

Rather than having to list the indexes you want to merge, now it can do it
for you automatically.  The output filename is now also optional; it'll
generate it in the right place in the git repo automatically.

diff --git a/cmd-midx.py b/cmd-midx.py
index 63000e050620f9ecf09beb1caeea50819cb45f95..94648d4a1603a7551686cc62602710d0cf627b27 100755 (executable)
--- a/cmd-midx.py
+++ b/cmd-midx.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import sys, math, struct
+import sys, math, struct, glob, sha
 import options, git
 from helpers import *
 
@@ -14,47 +14,15 @@ def next(it):
         return None
     
     
-optspec = """
-bup midx -o outfile.midx <idxnames...>
---
-o,output=  output midx file name
-"""
-o = options.Options('bup midx', optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if not extra:
-    log("bup midx: no input filenames given\n")
-    o.usage()
-if not opt.output:
-    log("bup midx: no output filename given\n")
-    o.usage()
-    
-inp = []
-total = 0
-for name in extra:
-    ix = git.PackIndex(name)
-    inp.append(ix)
-    total += len(ix)
-    
-log('total objects expected: %d\n' % total)
-pages = total/SHA_PER_PAGE
-log('pages: %d\n' % pages)
-bits = int(math.ceil(math.log(pages, 2)))
-log('table bits: %d\n' % bits)
-entries = 2**bits
-log('table entries: %d\n' % entries)
-log('table size: %d\n' % (entries*8))
-
-table = [0]*entries
-
-def merge(idxlist):
-    iters = [iter(i) for i in inp]
+def merge(idxlist, total, bits, table):
+    iters = [iter(i) for i in idxlist]
     iters = [[next(it), it] for it in iters]
     count = 0
     while iters:
         if (count % 10000) == 0:
-            log('\rMerging: %d/%d' % (count, total))
-        e = min(iters)  # FIXME: very slow for long lists
+            log('\rMerging: %.2f%% (%d/%d)'
+                % (count*100.0/total, count, total))
+        e = min(iters)  # FIXME: very slow for long lists; use smarter algo
         assert(e[0])
         yield e[0]
         count += 1
@@ -64,28 +32,97 @@ def merge(idxlist):
         iters = filter(lambda x: x[0], iters)
     log('\rMerging: done.                                    \n')
 
-f = open(opt.output, 'w+')
-f.write('MIDX\0\0\0\1')
-f.write(struct.pack('!I', bits))
-assert(f.tell() == 12)
-f.write('\0'*8*entries)
 
-for e in merge(inp):
-    f.write(e)
+def do_midx(outdir, outfilename, infilenames):
+    if not outfilename:
+        assert(outdir)
+        sum = sha.sha('\0'.join(infilenames)).hexdigest()
+        outfilename = '%s/midx-%s.midx' % (outdir, sum)
+    
+    inp = []
+    total = 0
+    for name in infilenames:
+        ix = git.PackIndex(name)
+        inp.append(ix)
+        total += len(ix)
+
+    if not total:
+        log('No new .idx files: nothing to do.\n')
+        return
+
+    log('Merging %d indexes (%d objects).\n' % (len(infilenames), total))
+    pages = total/SHA_PER_PAGE
+    bits = int(math.ceil(math.log(pages, 2)))
+    entries = 2**bits
+    log('table size: %d (%d bits)\n' % (entries*8, bits))
+    
+    table = [0]*entries
+
+    try:
+        os.unlink(outfilename)
+    except OSError:
+        pass
+    f = open(outfilename + '.tmp', 'w+')
+    f.write('MIDX\0\0\0\1')
+    f.write(struct.pack('!I', bits))
+    assert(f.tell() == 12)
+    f.write('\0'*8*entries)
+    
+    for e in merge(inp, total, bits, table):
+        f.write(e)
+        
+    f.write('\0'.join([os.path.basename(p) for p in infilenames]))
 
-f.write('\0'.join([os.path.basename(p) for p in extra]))
+    f.seek(12)
+    f.write(struct.pack('!%dQ' % entries, *table))
+    f.close()
 
-f.seek(12)
-f.write(struct.pack('!%dQ' % entries, *table))
-f.close()
+    # this is just for testing
+    if 0:
+        p = git.PackMidx(outfilename)
+        assert(len(p.idxnames) == len(infilenames))
+        print p.idxnames
+        assert(len(p) == total)
+        pi = iter(p)
+        for i in merge(inp, total, bits, table):
+            assert(i == pi.next())
+            assert(p.exists(i))
 
-# this is just for testing
-if 0:
-    p = git.PackMidx(opt.output)
-    assert(len(p.idxnames) == len(extra))
-    print p.idxnames
-    assert(len(p) == total)
-    pi = iter(p)
-    for i in merge(inp):
-        assert(i == pi.next())
-        assert(p.exists(i))
+    os.rename(outfilename + '.tmp', outfilename)
+    print outfilename
+
+optspec = """
+bup midx [options...] <idxnames...>
+--
+o,output=  output midx filename (default: auto-generated)
+a,auto     automatically create .midx from any unindexed .idx files
+f,force    automatically create .midx from *all* .idx files
+"""
+o = options.Options('bup midx', optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra and (opt.auto or opt.force):
+    log("bup midx: you can't use -f/-a and also provide filenames\n")
+    o.usage()
+
+git.check_repo_or_die()
+
+if extra:
+    do_midx(git.repo('objects/pack'), opt.output, extra)
+elif opt.auto or opt.force:
+    paths = [git.repo('objects/pack')]
+    paths += glob.glob(git.repo('index-cache/*/.'))
+    if opt.force:
+        for path in paths:
+            do_midx(path, opt.output, glob.glob('%s/*.idx' % path))
+    elif opt.auto:
+        for path in paths:
+            m = git.MultiPackIndex(path)
+            needed = {}
+            for pack in m.packs:  # only .idx files without a .midx are open
+                if pack.name.endswith('.idx'):
+                    needed[pack.name] = 1
+            do_midx(path, opt.output, needed.keys())
+else:
+    log("bup midx: you must use -f or -a or provide input filenames\n")
+    o.usage()

diff --git a/git.py b/git.py
index 67e14510dd6d3527df226960b4be8f5aa795f6c3..a29266bc729dfb568c52070ae801ec2f3749392c 100644 (file)
--- a/git.py
+++ b/git.py
@@ -234,6 +234,7 @@ class MultiPackIndex:
                     for sub in ix.idxnames:
                         if not d.get(os.path.join(self.dir, sub)):
                             self.packs.append(ix)
+                            d[ix.name] = 1
                             for name in ix.idxnames:
                                 d[os.path.join(self.dir, name)] = 1
                             break
@@ -241,7 +242,8 @@ class MultiPackIndex:
                 full = os.path.join(self.dir, f)
                 if f.endswith('.idx') and not d.get(full):
                     self.packs.append(PackIndex(full))
-        log('MultiPackIndex: using %d packs.\n' % len(self.packs))
+                    d[full] = 1
+        #log('MultiPackIndex: using %d packs.\n' % len(self.packs))
 
     def add(self, hash):
         self.also[hash] = 1