_helpers.firstword(): a new function to extract the first 32 bits.

This is a pretty common operation in git.py and it speeds up cmd/memtest
results considerably: from 3.7 seconds to 3.0 seconds.

That gets us *almost* as fast as we were before the whole statistical
guessing thing, but we still enjoy the improved memory usage.

Signed-off-by: Avery Pennarun <apenwarr@gmail.com>

diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c
index 3ce5ecb28b64a147d0efb0f5dd301ae0f17cc3de..510ab3f15032e09cccc351b4dd0745d1b4d0daab 100644 (file)
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@ -3,6 +3,7 @@
 #include <assert.h>
 #include <stdint.h>
 #include <fcntl.h>
+#include <arpa/inet.h>
 
 static PyObject *selftest(PyObject *self, PyObject *args)
 {
@@ -59,6 +60,23 @@ static PyObject *bitmatch(PyObject *self, PyObject *args)
 }
 
 
+static PyObject *firstword(PyObject *self, PyObject *args)
+{
+    unsigned char *buf = NULL;
+    int len = 0;
+    uint32_t v;
+
+    if (!PyArg_ParseTuple(args, "t#", &buf, &len))
+       return NULL;
+    
+    if (len < 4)
+       return NULL;
+    
+    v = ntohl(*(uint32_t *)buf);
+    return Py_BuildValue("I", v);
+}
+
+
 // I would have made this a lower-level function that just fills in a buffer
 // with random values, and then written those values from python.  But that's
 // about 20% slower in my tests, and since we typically generate random
@@ -165,6 +183,8 @@ static PyMethodDef faster_methods[] = {
        "Split a list of strings based on a rolling checksum." },
     { "bitmatch", bitmatch, METH_VARARGS,
        "Count the number of matching prefix bits between two strings." },
+    { "firstword", firstword, METH_VARARGS,
+        "Return an int corresponding to the first 32 bits of buf." },
     { "write_random", write_random, METH_VARARGS,
        "Write random bytes to the given file descriptor" },
     { "open_noatime", open_noatime, METH_VARARGS,

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 1887cc0e14bac6b84254a6d2f6cb534747da9d56..5ec94ebfa3f3c1f85705fc72e47ac24e723e3446 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -5,6 +5,7 @@ interact with the Git data structures.
 import os, zlib, time, subprocess, struct, stat, re, tempfile
 import heapq
 from bup.helpers import *
+from bup import _helpers
 
 verbose = 0
 ignore_midx = 0
@@ -194,7 +195,7 @@ class PackIdx:
 def extract_bits(buf, nbits):
     """Take the first 'nbits' bits from 'buf' and return them as an integer."""
     mask = (1<<nbits) - 1
-    v = struct.unpack('!I', buf[0:4])[0]
+    v = _helpers.firstword(buf)
     v = (v >> (32-nbits)) & mask
     return v
 
@@ -218,7 +219,7 @@ class PackMidx:
             self.idxnames = []
         else:
             assert(str(self.map[0:8]) == 'MIDX\0\0\0\2')
-            self.bits = struct.unpack('!I', self.map[8:12])[0]
+            self.bits = _helpers.firstword(self.map[8:12])
             self.entries = 2**self.bits
             self.fanout = buffer(self.map, 12, self.entries*4)
             shaofs = 12 + self.entries*4
@@ -229,14 +230,11 @@ class PackMidx:
     def _fanget(self, i):
         start = i*4
         s = self.fanout[start:start+4]
-        return struct.unpack('!I', s)[0]
+        return _helpers.firstword(s)
 
     def _get(self, i):
         return str(self.shalist[i*20:(i+1)*20])
 
-    def _num(self, hash):
-        return struct.unpack('!I', hash[:4])[0]
-
     def exists(self, hash):
         """Return nonempty if the object exists in the index files."""
         global _total_searches, _total_steps
@@ -252,7 +250,7 @@ class PackMidx:
         end = self._fanget(el)
         endv = (el+1) << (32-self.bits)
         _total_steps += 1   # lookup table is a step
-        hashv = self._num(hash)
+        hashv = _helpers.firstword(hash)
         #print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv)
         while start < end:
             _total_steps += 1
@@ -263,10 +261,10 @@ class PackMidx:
             #print '    %08x' % self._num(v)
             if v < want:
                 start = mid+1
-                startv = self._num(v)
+                startv = _helpers.firstword(v)
             elif v > want:
                 end = mid
-                endv = self._num(v)
+                endv = _helpers.firstword(v)
             else: # got it!
                 return True
         return None