fun-with-a-profiler-20040504

fun with a profiler:  afs_RemoveVCB accounts for 20% of the total AFS
client CPU utilization, even under a moderate vcache turnover rate of
less than 10 new vcache's per second.

introduce a hash on the afs_cbr objects to speed this up.  if you are
seeing your AFS client being CPU-bound, e.g. on your web server, you
may want to try this.

diff --git a/src/afs/afs.h b/src/afs/afs.h
index cf95716ca2982323a2261d37c505beb008c5b16a..c3464ef6941e83f5fd36b3161b963e12e96ec9df 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -71,6 +71,7 @@ extern int afs_shuttingdown;
 #define        NFENTRIES       256     /* hash table size for disk volume table */
 #define        VCSIZE         1024     /* stat cache hash table size */
 #define        DCSIZE          512     /* disk cache hash table size */
+#define CBRSIZE                512     /* call back returns hash table size */
 #define        PIGGYSIZE       1350    /* max piggyback size */
 #define        MAXVOLS         128     /* max vols we can store */
 #define        MAXSYSNAME      128     /* max sysname (i.e. @sys) size */
@@ -191,7 +192,12 @@ struct vrequest {
  * callbacks.  Sent asynchronously when we run a little low on free dudes.
  */
 struct afs_cbr {
+    struct afs_cbr **pprev;
     struct afs_cbr *next;
+
+    struct afs_cbr **hash_pprev;
+    struct afs_cbr *hash_next;
+
     struct AFSFid fid;
 };
 

diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h
index adaf96f0273e64fe971546ca2accdcd63464447d..43aa28236425e40a7646390ff25c6c0393d99b7e 100644 (file)
--- a/src/afs/afs_prototypes.h
+++ b/src/afs/afs_prototypes.h
@@ -464,7 +464,7 @@ extern void afs_osi_Invisible(void);
 extern void afs_osi_RxkRegister(void);
 extern void afs_osi_MaskSignals(void);
 extern void afs_osi_UnmaskRxkSignals(void);
-extern void *afs_osi_Alloc(size_t x);
+extern void *afs_osi_Alloc_debug(size_t x, char *func, int line);
 #ifndef afs_osi_Alloc_NoSleep
 extern void *afs_osi_Alloc_NoSleep(size_t x);
 #endif
@@ -855,7 +855,7 @@ extern void afs_ProcessFS(register struct vcache *avc,
                          struct vrequest *areq);
 extern struct afs_cbr *afs_AllocCBR(void);
 extern int afs_FreeCBR(register struct afs_cbr *asp);
-extern int afs_RemoveVCB(register struct VenusFid *afid);
+extern void afs_RemoveVCB(register struct VenusFid *afid);
 extern void afs_FlushActiveVcaches(register afs_int32 doflocks);
 extern int afs_WriteVCache(register struct vcache *avc,
                           register struct AFSStoreStatus *astatus,

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index 18137aa396105f312b0711379f8c2a465055093a..e467beb0f7104d1f3be228a0ec8a9b9a2e00b4f6 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -69,12 +69,40 @@ struct afs_q VLRU;          /*vcache LRU */
 afs_int32 vcachegen = 0;
 unsigned int afs_paniconwarn = 0;
 struct vcache *afs_vhashT[VCSIZE];
+static struct afs_cbr *afs_cbrHashT[CBRSIZE];
 afs_int32 afs_bulkStatsLost;
 int afs_norefpanic = 0;
 
 /* Forward declarations */
 static afs_int32 afs_QueueVCB(struct vcache *avc);
 
+/*
+ * afs_HashCBRFid
+ *
+ * Generate an index into the hash table for a given Fid.
+ */
+static int
+afs_HashCBRFid(struct AFSFid *fid) {
+    return (fid->Volume + fid->Vnode + fid->Unique) % CBRSIZE;
+}
+
+/*
+ * afs_InsertHashCBR
+ *
+ * Insert a CBR entry into the hash table.
+ * Must be called with afs_xvcb held.
+ */
+static void
+afs_InsertHashCBR(struct afs_cbr *cbr) {
+    int slot = afs_HashCBRFid(&cbr->fid);
+
+    cbr->hash_next = afs_cbrHashT[slot];
+    if (afs_cbrHashT[slot])
+       afs_cbrHashT[slot]->hash_pprev = &cbr->hash_next;
+
+    cbr->hash_pprev = &afs_cbrHashT[slot];
+    afs_cbrHashT[slot] = cbr;
+}
 
 /*
  * afs_FlushVCache
@@ -295,7 +323,7 @@ afs_AllocCBR(void)
 /*
  * afs_FreeCBR
  *
- * Description: free a callback return structure.
+ * Description: free a callback return structure, removing it from all lists.
  *
  * Parameters:
  *     asp -- the address of the structure to free.
@@ -305,6 +333,14 @@ afs_AllocCBR(void)
 int
 afs_FreeCBR(register struct afs_cbr *asp)
 {
+    *(asp->pprev) = asp->next;
+    if (asp->next)
+       asp->next->pprev = asp->pprev;
+
+    *(asp->hash_pprev) = asp->hash_next;
+    if (asp->hash_next)
+       asp->hash_next->hash_pprev = asp->hash_pprev;
+
     asp->next = afs_cbrSpace;
     afs_cbrSpace = asp;
     return 0;
@@ -406,7 +442,8 @@ afs_FlushVCBs(afs_int32 lockit)
                 */
                tcbrp = tsp->cbrs;
                tfids[tcount++] = tcbrp->fid;
-               tsp->cbrs = tcbrp->next;
+
+               /* Freeing the CBR will unlink it from the server's CBR list */
                afs_FreeCBR(tcbrp);
            }                   /* while loop for this one server */
            if (safety2 > afs_cacheStats) {
@@ -447,8 +484,8 @@ afs_FlushVCBs(afs_int32 lockit)
 static afs_int32
 afs_QueueVCB(struct vcache *avc)
 {
-    register struct server *tsp;
-    register struct afs_cbr *tcbp;
+    struct server *tsp;
+    struct afs_cbr *tcbp;
 
     AFS_STATCNT(afs_QueueVCB);
     /* The callback is really just a struct server ptr. */
@@ -460,8 +497,15 @@ afs_QueueVCB(struct vcache *avc)
     MObtainWriteLock(&afs_xvcb, 274);
     tcbp = afs_AllocCBR();
     tcbp->fid = avc->fid.Fid;
+
     tcbp->next = tsp->cbrs;
+    if (tsp->cbrs)
+       tsp->cbrs->pprev = &tcbp->next;
+
     tsp->cbrs = tcbp;
+    tcbp->pprev = &tsp->cbrs;
+
+    afs_InsertHashCBR(tcbp);
 
     /* now release locks and return */
     MReleaseWriteLock(&afs_xvcb);
@@ -473,8 +517,7 @@ afs_QueueVCB(struct vcache *avc)
  * afs_RemoveVCB
  *
  * Description:
- *     Remove a queued callback by looking through all the servers
- *     to see if any have this callback queued.
+ *     Remove a queued callback for a given Fid.
  *
  * Parameters:
  *     afid: The fid we want cleansed of queued callbacks.
@@ -485,44 +528,30 @@ afs_QueueVCB(struct vcache *avc)
  *     entries locked.
  */
 
-int
+void
 afs_RemoveVCB(struct VenusFid *afid)
 {
-    register int i;
-    register struct server *tsp;
-    register struct afs_cbr *tcbrp;
-    struct afs_cbr **lcbrpp;
+    int slot;
+    struct afs_cbr *cbr, *ncbr;
 
     AFS_STATCNT(afs_RemoveVCB);
     MObtainWriteLock(&afs_xvcb, 275);
-    ObtainReadLock(&afs_xserver);
-    for (i = 0; i < NSERVERS; i++) {
-       for (tsp = afs_servers[i]; tsp; tsp = tsp->next) {
-           /* if cell is known, and is wrong, then skip this server */
-           if (tsp->cell && tsp->cell->cellNum != afid->Cell)
-               continue;
 
-           /*
-            * Otherwise, iterate through file IDs we're sending to the
-            * server.
-            */
-           lcbrpp = &tsp->cbrs;        /* first queued return callback */
-           for (tcbrp = *lcbrpp; tcbrp;
-                lcbrpp = &tcbrp->next, tcbrp = *lcbrpp) {
-               if (afid->Fid.Volume == tcbrp->fid.Volume
-                   && afid->Fid.Unique == tcbrp->fid.Unique
-                   && afid->Fid.Vnode == tcbrp->fid.Vnode) {
-                   *lcbrpp = tcbrp->next;      /* unthread from list */
-                   afs_FreeCBR(tcbrp);
-                   goto done;
-               }
-           }
+    slot = afs_HashCBRFid(&afid->Fid);
+    ncbr = afs_cbrHashT[slot];
+
+    while (ncbr) {
+       cbr = ncbr;
+       ncbr = cbr->hash_next;
+
+       if (afid->Fid.Volume == cbr->fid.Volume &&
+           afid->Fid.Vnode  == cbr->fid.Vnode  &&
+           afid->Fid.Unique == cbr->fid.Unique) {
+           afs_FreeCBR(cbr);
        }
     }
-  done:
-    ReleaseReadLock(&afs_xserver);
+
     MReleaseWriteLock(&afs_xvcb);
-    return 0;
 }
 
 #if defined(AFS_LINUX22_ENV) && !defined(AFS_LINUX26_ENV)