cmd/save: when a file is chunked, mangle its name from * to *.bup

Files that are already named *.bup are renamed to *.bup.bupl, so that we can
just always drop either .bup or .bupl from a filename if it's there, and the
result will be the original filename.

Also updated lib/bup/vfs.py to demangle the names appropriately, and treat
git trees named *.bup as real chunked files (ie. by joining them back
together).

diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py
index edb712a1001435562f2dbaaf4fe1fdd0116d4f34..e840e557bba85e9bb87f5bc238cb3a750f37ff2c 100755 (executable)
--- a/cmd/save-cmd.py
+++ b/cmd/save-cmd.py
@@ -183,6 +183,7 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
             _push(part)
 
     if not file:
+        # no filename portion means this is a subdir.  But
         # sub/parentdirectories already handled in the pop/push() part above.
         oldtree = already_saved(ent) # may be None
         newtree = _pop(force_tree = oldtree)
@@ -201,7 +202,9 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     if hashvalid:
         mode = '%o' % ent.gitmode
         id = ent.sha
-        shalists[-1].append((mode, file, id))
+        shalists[-1].append((mode, 
+                             git.mangle_name(file, ent.mode, ent.gitmode),
+                             id))
     else:
         if stat.S_ISREG(ent.mode):
             try:
@@ -234,7 +237,9 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
         if id:
             ent.validate(int(mode, 8), id)
             ent.repack()
-            shalists[-1].append((mode, file, id))
+            shalists[-1].append((mode,
+                                 git.mangle_name(file, ent.mode, ent.gitmode),
+                                 id))
     if exists and wasmissing:
         count += oldsize
         subcount = 0

diff --git a/lib/bup/git.py b/lib/bup/git.py
index c7d4f728cc6684ab9c614d14b2ac9dd95e621dab..359dfa75de5cfbaa3c91517e7bc5cdedbec7cc7d 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -25,6 +25,25 @@ def repo(sub = ''):
     return os.path.join(repodir, sub)
 
 
+def mangle_name(name, mode, gitmode):
+    if stat.S_ISREG(mode) and not stat.S_ISREG(gitmode):
+        return name + '.bup'
+    elif name.endswith('.bup') or name[:-1].endswith('.bup'):
+        return name + '.bupl'
+    else:
+        return name
+
+
+(BUP_NORMAL, BUP_CHUNKED) = (0,1)
+def demangle_name(name):
+    if name.endswith('.bupl'):
+        return (name[:-5], BUP_NORMAL)
+    elif name.endswith('.bup'):
+        return (name[:-4], BUP_CHUNKED)
+    else:
+        return (name, BUP_NORMAL)
+
+
 def _encode_packobj(type, content):
     szout = ''
     sz = len(content)

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index 6a9857494343aa8c103976a18caa735b4b705fac..156486a0b960cc94243a97c87070bb5f2edc06ae 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -263,7 +263,7 @@ def parse_num(s):
         raise ValueError("invalid unit %r in number %r" % (unit, s))
     return int(num*mult)
 
-
+    
 # count the number of elements in an iterator (consumes the iterator)
 def count(l):
     return reduce(lambda x,y: x+1, l)

diff --git a/lib/bup/vfs.py b/lib/bup/vfs.py
index 80e3c1daa6d54778747209aedebc5f6832e01787..7964c9825f2bde56de7a64a10a2ea69cdd707eb8 100644 (file)
--- a/lib/bup/vfs.py
+++ b/lib/bup/vfs.py
@@ -149,11 +149,13 @@ class File(Node):
         return FileReader(self)
     
     def size(self):
-        # FIXME inefficient
+        # FIXME inefficient.  If a file is chunked, we could just check
+        # the offset + size of the very last chunk.
         return sum(len(blob) for blob in self._content())
     
     def readbytes(self, ofs, count):
-        # FIXME inefficient
+        # FIXME inefficient.  If a file is chunked, we could choose to
+        # read only the required chunks rather than joining the whole thing.
         buf = ''.join(self._content())
         return buf[ofs:ofs+count]
     
@@ -200,8 +202,12 @@ class Dir(Node):
             it = cp().get(self.hash.encode('hex') + ':')
             type = it.next()
         assert(type == 'tree')
-        for (mode,name,sha) in git._treeparse(''.join(it)):
+        for (mode,mangled_name,sha) in git._treeparse(''.join(it)):
             mode = int(mode, 8)
+            name = mangled_name
+            (name,bupmode) = git.demangle_name(mangled_name)
+            if bupmode == git.BUP_CHUNKED:
+                mode = 0100644
             if stat.S_ISDIR(mode):
                 self._subs[name] = Dir(self, name, mode, sha)
             elif stat.S_ISLNK(mode):

diff --git a/t/tgit.py b/t/tgit.py
index 2ff71ba385d0c20f8f0b88008d91f2b1757df0ae..83faadc5ea54e6d94c4778d382e3d9019eca58bf 100644 (file)
--- a/t/tgit.py
+++ b/t/tgit.py
@@ -4,6 +4,33 @@ from bup.helpers import *
 from wvtest import *
 
 
+@wvtest
+def testmangle():
+    afile  = 0100644
+    afile2 = 0100770
+    alink  = 0120000
+    adir   = 0040000
+    adir2  = 0040777
+    WVPASSEQ(git.mangle_name("a", adir2, adir), "a")
+    WVPASSEQ(git.mangle_name(".bup", adir2, adir), ".bup.bupl")
+    WVPASSEQ(git.mangle_name("a.bupa", adir2, adir), "a.bupa.bupl")
+    WVPASSEQ(git.mangle_name("b.bup", alink, alink), "b.bup.bupl")
+    WVPASSEQ(git.mangle_name("b.bu", alink, alink), "b.bu")
+    WVPASSEQ(git.mangle_name("f", afile, afile2), "f")
+    WVPASSEQ(git.mangle_name("f.bup", afile, afile2), "f.bup.bupl")
+    WVPASSEQ(git.mangle_name("f.bup", afile, adir), "f.bup.bup")
+    WVPASSEQ(git.mangle_name("f", afile, adir), "f.bup")
+
+    WVPASSEQ(git.demangle_name("f.bup"), ("f", git.BUP_CHUNKED))
+    WVPASSEQ(git.demangle_name("f.bupl"), ("f", git.BUP_NORMAL))
+    WVPASSEQ(git.demangle_name("f.bup.bupl"), ("f.bup", git.BUP_NORMAL))
+
+    # for safety, we ignore .bup? suffixes we don't recognize.  Future
+    # versions might implement a .bup[a-z] extension as something other
+    # than BUP_NORMAL.
+    WVPASSEQ(git.demangle_name("f.bupa"), ("f.bupa", git.BUP_NORMAL))
+
+
 @wvtest
 def testencode():
     s = 'hello world'
@@ -20,6 +47,7 @@ def testencode():
     WVPASSEQ(git._decode_packobj(packt), ('tree', s))
     WVPASSEQ(git._decode_packobj(packc), ('commit', s))
 
+
 @wvtest
 def testpacks():
     git.init_repo('pybuptest.tmp')