Speed up cmd-drecurse by 40%.

It's now 40% faster, ie. 1.769 seconds or so to go through my home
directory, instead of the previous 2.935.

Still sucks compared to the native C 'find' command, but that's probably
about as good as it's getting in pure python.

diff --git a/cmd-drecurse.py b/cmd-drecurse.py
index ffe25058a4f05ca55164753b8eaff1a48180d0ff..f09f0411b2e65adaecae58bbd62f7fd0a2d8ee9b 100755 (executable)
--- a/cmd-drecurse.py
+++ b/cmd-drecurse.py
@@ -5,8 +5,9 @@ from helpers import *
 optspec = """
 bup drecurse <path>
 --
-x,xdev   don't cross filesystem boundaries
+x,xdev,one-file-system   don't cross filesystem boundaries
 q,quiet  don't actually print filenames
+profile  run under the python profiler
 """
 o = options.Options('bup drecurse', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -15,6 +16,21 @@ if len(extra) != 1:
     log("drecurse: exactly one filename expected\n")
     o.usage()
 
-for (name,st) in drecurse.recursive_dirlist(extra, opt.xdev):
-    if not opt.quiet:
-        print name
+it = drecurse.recursive_dirlist(extra, opt.xdev)
+if opt.profile:
+    import cProfile
+    def do_it():
+        for i in it:
+            pass
+    cProfile.run('do_it()')
+else:
+    if opt.quiet:
+        for i in it:
+            pass
+    else:
+        for (name,st) in it:
+            print name
+
+if saved_errors:
+    log('WARNING: %d errors encountered.\n' % len(saved_errors))
+    sys.exit(1)

diff --git a/drecurse.py b/drecurse.py
index b1f9e16afbd76f53490d9ef83e838cf77516f0f3..b62e58a6884416ff446468793182c7ea3d81b484 100644 (file)
--- a/drecurse.py
+++ b/drecurse.py
@@ -1,4 +1,4 @@
-import stat
+import stat, heapq
 from helpers import *
 
 try:
@@ -7,6 +7,9 @@ except AttributeError:
     O_LARGEFILE = 0
 
 
+# the use of fchdir() and lstat() is for two reasons:
+#  - help out the kernel by not making it repeatedly look up the absolute path
+#  - avoid race conditions caused by doing listdir() on a changing symlink
 class OsFile:
     def __init__(self, path):
         self.fd = None
@@ -21,72 +24,71 @@ class OsFile:
     def fchdir(self):
         os.fchdir(self.fd)
 
+    def stat(self):
+        return os.fstat(self.fd)
 
-# the use of fchdir() and lstat() are for two reasons:
-#  - help out the kernel by not making it repeatedly look up the absolute path
-#  - avoid race conditions caused by doing listdir() on a changing symlink
-def dirlist(path):
+
+_IFMT = stat.S_IFMT(0xffffffff)  # avoid function call in inner loop
+def _dirlist():
     l = []
-    try:
-        OsFile(path).fchdir()
-    except OSError, e:
-        add_error(e)
-        return l
     for n in os.listdir('.'):
         try:
             st = os.lstat(n)
         except OSError, e:
-            add_error(Exception('in %s: %s' % (index.realpath(path), str(e))))
+            add_error(Exception('%s: %s' % (realpath(n), str(e))))
             continue
-        if stat.S_ISDIR(st.st_mode):
+        if (st.st_mode & _IFMT) == stat.S_IFDIR:
             n += '/'
-        l.append((os.path.join(path, n), st))
+        l.append((n,st))
     l.sort(reverse=True)
     return l
 
 
-def _recursive_dirlist(path, xdev):
-    olddir = OsFile('.')
-    for (path,pst) in dirlist(path):
-        if xdev != None and pst.st_dev != xdev:
-            log('Skipping %r: different filesystem.\n' % path)
-            continue
-        if stat.S_ISDIR(pst.st_mode):
-            for i in _recursive_dirlist(path, xdev=xdev):
-                yield i
-        yield (path,pst)
-    olddir.fchdir()
-
-
-def _matchlen(a,b):
-    bi = iter(b)
-    count = 0
-    for ai in a:
-        try:
-            if bi.next() == ai:
-                count += 1
-        except StopIteration:
-            break
-    return count
+def _recursive_dirlist(prepend, xdev):
+    for (name,pst) in _dirlist():
+        if name.endswith('/'):
+            if xdev != None and pst.st_dev != xdev:
+                log('Skipping %r: different filesystem.\n' % (prepend+name))
+                continue
+            try:
+                OsFile(name).fchdir()
+            except OSError, e:
+                add_error('%s: %s' % (prepend, e))
+            else:
+                for i in _recursive_dirlist(prepend=prepend+name, xdev=xdev):
+                    yield i
+                os.chdir('..')
+        yield (prepend + name, pst)
 
 
 def recursive_dirlist(paths, xdev):
-    assert(type(paths) != type(''))
-    last = ()
-    for path in paths:
-        ps = pathsplit(path)
-        while _matchlen(ps, last) < len(last):
-            yield (''.join(last), None)
-            last.pop()
-        pst = os.lstat(path)
-        if xdev:
-            xdev = pst.st_dev
-        else:
-            xdev = None
-        if stat.S_ISDIR(pst.st_mode):
-            for i in _recursive_dirlist(path, xdev=xdev):
-                yield i
-        yield (path,pst)
-        last = ps[:-1]
-
-
+    startdir = OsFile('.')
+    try:
+        assert(type(paths) != type(''))
+        for path in paths:
+            try:
+                rpath = os.path.realpath(path)
+                pfile = OsFile(rpath)
+            except OSError, e:
+                add_error(e)
+                continue
+            pst = pfile.stat()
+            if xdev:
+                xdev = pst.st_dev
+            else:
+                xdev = None
+            if stat.S_ISDIR(pst.st_mode):
+                pfile.fchdir()
+                prepend = os.path.join(path, '')
+                for i in _recursive_dirlist(prepend=prepend, xdev=xdev):
+                    yield i
+                startdir.fchdir()
+            else:
+                prepend = path
+            yield (prepend,pst)
+    except:
+        try:
+            startdir.fchdir()
+        except:
+            pass
+        raise

diff --git a/helpers.py b/helpers.py
index 7f54430ac9a298f1637a694b4201162c3b6afa35..a7a6cc6d249465c09f017556ea60c850c1891119 100644 (file)
--- a/helpers.py
+++ b/helpers.py
@@ -1,4 +1,4 @@
-import sys, os, pwd, subprocess, errno, socket, select, mmap
+import sys, os, pwd, subprocess, errno, socket, select, mmap, stat
 
 
 def log(s):
@@ -48,6 +48,23 @@ def pathsplit(p):
     return l
 
 
+# like os.path.realpath, but doesn't follow a symlink for the last element.
+# (ie. if 'p' itself is itself a symlink, this one won't follow it)
+def realpath(p):
+    try:
+        st = os.lstat(p)
+    except OSError:
+        st = None
+    if st and stat.S_ISLNK(st.st_mode):
+        (dir, name) = os.path.split(p)
+        dir = os.path.realpath(dir)
+        out = os.path.join(dir, name)
+    else:
+        out = os.path.realpath(p)
+    #log('realpathing:%r,%r\n' % (p, out))
+    return out
+
+
 _username = None
 def username():
     global _username

diff --git a/index.py b/index.py
index 9a6475fd18e2e8d92dc775c102bf4237f6f96f4e..3af039df1999f1bccf7db49e8aebdef51ff3f955 100644 (file)
--- a/index.py
+++ b/index.py
@@ -151,7 +151,7 @@ class Reader:
         self.close()
 
     def iter(self, name=None):
-        if len(self.m):
+        if len(self.m) > len(INDEX_HDR)+ENTLEN:
             dname = name
             if dname and not dname.endswith('/'):
                 dname += '/'
@@ -321,23 +321,6 @@ class Writer:
         return Reader(self.tmpname)
 
 
-# like os.path.realpath, but doesn't follow a symlink for the last element.
-# (ie. if 'p' itself is itself a symlink, this one won't follow it)
-def realpath(p):
-    try:
-        st = os.lstat(p)
-    except OSError:
-        st = None
-    if st and stat.S_ISLNK(st.st_mode):
-        (dir, name) = os.path.split(p)
-        dir = os.path.realpath(dir)
-        out = os.path.join(dir, name)
-    else:
-        out = os.path.realpath(p)
-    #log('realpathing:%r,%r\n' % (p, out))
-    return out
-
-
 def reduce_paths(paths):
     xpaths = []
     for p in paths: