cmd-save: don't recurse into already-valid subdirs.

When iterating through the index, if we find out that a particular dir (like
/usr) has a known-valid sha1sum and isn't marked as changed, there's no need
to recurse into it at all.  This saves some pointless grinding through the
index when entire swaths of the tree are known to be already valid.

diff --git a/cmd-save.py b/cmd-save.py
index de1457b50ec07b8d10a91fc77d27d38052fb2271..81a663b1c33c4dd83e60fa467448c2642ec775e5 100755 (executable)
--- a/cmd-save.py
+++ b/cmd-save.py
@@ -54,11 +54,11 @@ def _push(part):
     parts.append(part)
     shalists.append([])
 
-def _pop():
+def _pop(force_tree):
     assert(len(parts) > 1)
     part = parts.pop()
     shalist = shalists.pop()
-    tree = w.new_tree(shalist)
+    tree = force_tree or w.new_tree(shalist)
     shalists[-1].append(('40000', part, tree))
     return tree
 
@@ -102,13 +102,19 @@ def progress_report(n):
 
 r = index.Reader(git.repo('bupindex'))
 
+def already_saved(ent):
+    return ent.is_valid() and w.exists(ent.sha) and ent.sha
+
+def wantrecurse(ent):
+    return not already_saved(ent)
+
 total = ftotal = 0
 if opt.progress:
-    for (transname,ent) in r.filter(extra):
+    for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse):
         if not (ftotal % 10024):
             progress('Reading index: %d\r' % ftotal)
         exists = (ent.flags & index.IX_EXISTS)
-        hashvalid = (ent.flags & index.IX_HASHVALID) and w.exists(ent.sha)
+        hashvalid = already_saved(ent)
         if exists and not hashvalid:
             total += ent.size
         ftotal += 1
@@ -117,10 +123,10 @@ if opt.progress:
 
 tstart = time.time()
 count = fcount = 0
-for (transname,ent) in r.filter(extra):
+for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse):
     (dir, file) = os.path.split(ent.name)
     exists = (ent.flags & index.IX_EXISTS)
-    hashvalid = (ent.flags & index.IX_HASHVALID) and w.exists(ent.sha)
+    hashvalid = already_saved(ent)
     if opt.verbose:
         if not exists:
             status = 'D'
@@ -144,15 +150,18 @@ for (transname,ent) in r.filter(extra):
     assert(dir.startswith('/'))
     dirp = dir.split('/')
     while parts > dirp:
-        _pop()
+        _pop(force_tree = None)
     if dir != '/':
         for part in dirp[len(parts):]:
             _push(part)
 
     if not file:
         # sub/parentdirectories already handled in the pop/push() part above.
-        ent.validate(040000, _pop())
-        ent.repack()
+        oldtree = already_saved(ent) # may be None
+        newtree = _pop(force_tree = oldtree)
+        if not oldtree:
+            ent.validate(040000, newtree)
+            ent.repack()
         count += ent.size
         continue  
 
@@ -198,12 +207,8 @@ if opt.progress:
     progress('Saving: %.2f%% (%d/%dk, %d/%d files), done.    \n'
              % (pct, count/1024, total/1024, fcount, ftotal))
 
-#log('parts out: %r\n' % parts)
-#log('stk out: %r\n' % shalists)
 while len(parts) > 1:
-    _pop()
-#log('parts out: %r\n' % parts)
-#log('stk out: %r\n' % shalists)
+    _pop(force_tree = None)
 assert(len(shalists) == 1)
 tree = w.new_tree(shalists[-1])
 if opt.tree:

diff --git a/index.py b/index.py
index 613fa6b565d0d1ca6396fef50c18813f4351afe2..225df3f8d02293db25ab8a291a553712447bdf86 100644 (file)
--- a/index.py
+++ b/index.py
@@ -114,8 +114,9 @@ class Entry:
         return (self.flags & IX_EXISTS) == 0
 
     def set_deleted(self):
-        self.flags &= ~(IX_EXISTS | IX_HASHVALID)
-        self.set_dirty()
+        if self.flags & IX_EXISTS:
+            self.flags &= ~(IX_EXISTS | IX_HASHVALID)
+            self.set_dirty()
 
     def set_dirty(self):
         pass # FIXME
@@ -170,7 +171,7 @@ class ExistingEntry(Entry):
             self.parent.invalidate()
             self.parent.repack()
 
-    def iter(self, name=None):
+    def iter(self, name=None, wantrecurse=None):
         dname = name
         if dname and not dname.endswith('/'):
             dname += '/'
@@ -188,8 +189,9 @@ class ExistingEntry(Entry):
             if (not dname
                  or child.name.startswith(dname)
                  or child.name.endswith('/') and dname.startswith(child.name)):
-                for e in child.iter(name=name):
-                    yield e
+                if not wantrecurse or wantrecurse(child):
+                    for e in child.iter(name=name, wantrecurse=wantrecurse):
+                        yield e
             if not name or child.name == name or child.name.startswith(dname):
                 yield child
             ofs = eon + 1 + ENTLEN
@@ -242,14 +244,14 @@ class Reader:
             yield ExistingEntry(None, basename, basename, self.m, eon+1)
             ofs = eon + 1 + ENTLEN
 
-    def iter(self, name=None):
+    def iter(self, name=None, wantrecurse=None):
         if len(self.m) > len(INDEX_HDR)+ENTLEN:
             dname = name
             if dname and not dname.endswith('/'):
                 dname += '/'
             root = ExistingEntry(None, '/', '/',
                                  self.m, len(self.m)-FOOTLEN-ENTLEN)
-            for sub in root.iter(name=name):
+            for sub in root.iter(name=name, wantrecurse=wantrecurse):
                 yield sub
             if not dname or dname == root.name:
                 yield root
@@ -270,9 +272,9 @@ class Reader:
             self.m = None
             self.writable = False
 
-    def filter(self, prefixes):
+    def filter(self, prefixes, wantrecurse=None):
         for (rp, path) in reduce_paths(prefixes):
-            for e in self.iter(rp):
+            for e in self.iter(rp, wantrecurse=wantrecurse):
                 assert(e.name.startswith(rp))
                 name = path + e.name[len(rp):]
                 yield (name, e)

diff --git a/t/test.sh b/t/test.sh
index d5a1917f3aa6d26ef2fd3449583988932219f1e8..40230a550b0b56a692dda9ac5c3698ec63684b1e 100755 (executable)
--- a/t/test.sh
+++ b/t/test.sh
@@ -93,8 +93,10 @@ WVPASSEQ "$(cd $D && bup index -m)" \
 "f
 a
 ./"
-WVPASS bup save -t $D
+tree1=$(bup save -t $D)
 WVPASSEQ "$(cd $D && bup index -m)" ""
+tree2=$(bup save -t $D)
+WVPASSEQ "$tree1" "$tree2"
 
 
 WVSTART "split"