Windows: Add caching to cm_GetAddrsU

Cache the results of VL_GetAddrsU queries and reuse the results
for subsequent calls when possible.

Change-Id: I7e2b086ec311208a46439588bc820a1929d2b2b9
Reviewed-on: http://gerrit.openafs.org/10764
Tested-by: Jeffrey Altman <jaltman@your-file-system.com>
Reviewed-by: Derrick Brashear <shadow@your-file-system.com>
Reviewed-by: Jeffrey Altman <jaltman@your-file-system.com>

diff --git a/src/WINNT/afsd/cm.h b/src/WINNT/afsd/cm.h
index 706fe502df03e56b69e84d77b7bc634e20359fe4..6975dba105c48ab464b71504803619bfc960c777 100644 (file)
--- a/src/WINNT/afsd/cm.h
+++ b/src/WINNT/afsd/cm.h
@@ -83,6 +83,7 @@
 #define LOCK_HIERARCHY_ACL_GLOBAL              730
 #define LOCK_HIERARCHY_EACCES_GLOBAL           740
 #define LOCK_HIERARCHY_USER_GLOBAL             750
+#define LOCK_HIERARCHY_GETADDRS_GLOBAL         800
 #define LOCK_HIERARCHY_AFSDBSBMT_GLOBAL       1000
 #define LOCK_HIERARCHY_TOKEN_EVENT_GLOBAL     2000
 #define LOCK_HIERARCHY_SYSCFG_GLOBAL          3000

diff --git a/src/WINNT/afsd/cm_getaddrs.c b/src/WINNT/afsd/cm_getaddrs.c
index d87aa3edfc4a1b3c86f52d5ff04ae61fc9984874..29e9d1a6ccf63c4e5cb2b3a9486590b333908179 100644 (file)
--- a/src/WINNT/afsd/cm_getaddrs.c
+++ b/src/WINNT/afsd/cm_getaddrs.c
@@ -36,6 +36,159 @@
 #include "afsd.h"
 #include "cm_getaddrs.h"
 
+typedef struct _uuid2addrsEntry {
+    osi_queue_t q;
+    afsUUID     uuid;
+    afs_int32   unique;
+    afs_uint32  nentries;
+    bulkaddrs   addrs;
+    afs_int32   refCount;
+    afs_uint32  flags;
+} uuid2addrsEntry_t;
+
+#define CM_GETADDRS_FLAG_DELETE        1
+
+/* lock for hash table */
+osi_rwlock_t cm_getaddrsLock;
+
+/* hash table */
+#define CM_GETADDRS_HASHTABLESIZE      128
+static afs_uint32      cm_getaddrsHashTableSize = CM_GETADDRS_HASHTABLESIZE;
+
+#define CM_GETADDRS_HASH(uuidp) \
+  (opr_jhash((const afs_uint32 *)uuidp, 4, 0) & (cm_getaddrsHashTableSize - 1))
+
+static struct _uuid2addrsHashTableBucket {
+    uuid2addrsEntry_t * Firstp;
+    uuid2addrsEntry_t * Endp;
+} cm_getaddrsHashTable[CM_GETADDRS_HASHTABLESIZE];
+
+/*
+ * Return a cached entry that matches the requested
+ * uuid if the requested unique is less than or equal
+ * to the cached entry.  Otherwise, return NULL.
+ *
+ * Returned entries are returned with a reference.
+ */
+static uuid2addrsEntry_t *
+cm_getaddrsFind(afsUUID *uuid, afs_int32 srv_unique)
+{
+    uuid2addrsEntry_t * entry = NULL;
+    afs_uint32 hash = CM_GETADDRS_HASH(uuid);
+
+    lock_ObtainRead(&cm_getaddrsLock);
+
+    for ( entry = cm_getaddrsHashTable[hash].Firstp;
+         entry;
+         entry = (uuid2addrsEntry_t *)osi_QNext(&entry->q))
+    {
+       if (memcmp(uuid, &entry->uuid, sizeof(afsUUID)) == 0 &&
+           srv_unique <= entry->unique &&
+           !(entry->flags & CM_GETADDRS_FLAG_DELETE))
+           break;
+    }
+
+    if (entry)
+       entry->refCount++;
+    lock_ReleaseRead(&cm_getaddrsLock);
+
+    return entry;
+}
+
+static void
+cm_getaddrsPut(uuid2addrsEntry_t *entry)
+{
+    lock_ObtainRead(&cm_getaddrsLock);
+    entry->refCount--;
+    lock_ReleaseRead(&cm_getaddrsLock);
+}
+
+/*
+ * Add a bulkaddrs list to the getaddrs queue.
+ * Once this function is called the bulkaddrs structure
+ * no longer belongs to the caller and must not be
+ * accessed.  It may be freed if there is a race with
+ * another thread.
+ */
+
+static uuid2addrsEntry_t *
+cm_getaddrsAdd(afsUUID *uuid, afs_int32 srv_unique,
+              afs_uint32 nentries, bulkaddrs *addrsp)
+{
+    uuid2addrsEntry_t * entry, *next, *retentry = NULL;
+    afs_uint32 hash = CM_GETADDRS_HASH(uuid);
+    int insert = 1;
+
+    lock_ObtainWrite(&cm_getaddrsLock);
+    for ( entry = cm_getaddrsHashTable[hash].Firstp;
+         entry;
+         entry = next)
+    {
+       next = (uuid2addrsEntry_t *)osi_QNext(&entry->q);
+
+       if ((entry->flags & CM_GETADDRS_FLAG_DELETE) &&
+           entry->refCount == 0)
+       {
+           osi_QRemoveHT((osi_queue_t **) &cm_getaddrsHashTable[hash].Firstp,
+                         (osi_queue_t **) &cm_getaddrsHashTable[hash].Endp,
+                         &entry->q);
+           xdr_free((xdrproc_t) xdr_bulkaddrs, &entry->addrs);
+           continue;
+       }
+
+       if (memcmp(uuid, &entry->uuid, sizeof(afsUUID)) == 0)
+       {
+
+           if (srv_unique <= entry->unique) {
+               /*
+                * out of date or duplicate entry.
+                * discard the input.
+                */
+               xdr_free((xdrproc_t) xdr_bulkaddrs, addrsp);
+               insert = 0;
+               retentry = entry;
+               retentry->refCount++;
+               continue;
+           }
+
+           /*
+            * this entry is newer than the one we found,
+            * if it is unused then update it
+            */
+           if (entry->refCount == 0) {
+               xdr_free((xdrproc_t) xdr_bulkaddrs, &entry->addrs);
+
+               entry->unique = srv_unique;
+               entry->addrs = *addrsp;
+               entry->nentries = nentries;
+               insert = 0;
+               retentry = entry;
+               retentry->refCount++;
+               continue;
+           }
+       }
+    }
+
+    if (insert) {
+       entry = calloc(1, sizeof(uuid2addrsEntry_t));
+       if (entry) {
+           memcpy(&entry->uuid, uuid, sizeof(afsUUID));
+           entry->unique = srv_unique;
+           entry->addrs = *addrsp;
+           entry->nentries = nentries;
+
+           osi_QAddH((osi_queue_t **) &cm_getaddrsHashTable[hash].Firstp,
+                      (osi_queue_t **) &cm_getaddrsHashTable[hash].Endp,
+                      &entry->q);
+           retentry = entry;
+           retentry->refCount++;
+       }
+    }
+    lock_ReleaseWrite(&cm_getaddrsLock);
+
+    return retentry;
+}
+
 /*
  * cm_GetAddrsU takes as input a uuid and a unique value which
  * represent the set of addresses that are required.  These values
@@ -57,64 +210,97 @@ cm_GetAddrsU(cm_cell_t *cellp, cm_user_t *userp, cm_req_t *reqp,
             afs_int32 serverUnique[])
 {
     afs_uint32 code = 0;
-    cm_conn_t *connp;
-    struct rx_connection *rxconnp;
-    afs_uint32 * addrp, nentries;
-    afs_int32 unique;
-    bulkaddrs  addrs;
-    ListAddrByAttributes attrs;
-    afsUUID uuid;
-    int i;
-
-    memset(&uuid, 0, sizeof(uuid));
-    memset(&addrs, 0, sizeof(addrs));
-    memset(&attrs, 0, sizeof(attrs));
-
-    attrs.Mask = VLADDR_UUID;
-    attrs.uuid = *Uuid;
-
-    do {
-       code = cm_ConnByMServers(cellp->vlServersp, 0, userp, reqp, &connp);
-       if (code)
-           continue;
-       rxconnp = cm_GetRxConn(connp);
-       code = VL_GetAddrsU(rxconnp, &attrs, &uuid, &unique, &nentries,
-                           &addrs);
-       rx_PutConnection(rxconnp);
-    } while (cm_Analyze(connp, userp, reqp, NULL, cellp, 0, NULL, NULL,
-                       &cellp->vlServersp, NULL, code));
+    uuid2addrsEntry_t *entry;
+
+    entry = cm_getaddrsFind(Uuid, Unique);
+    if (entry == NULL)
+    {
+       cm_conn_t *connp;
+       struct rx_connection *rxconnp;
+       ListAddrByAttributes attrs;
+       afs_int32 unique;
+       afsUUID uuid;
+       afs_uint32 nentries;
+       bulkaddrs  addrs;
+
+       memset(&uuid, 0, sizeof(uuid));
+       memset(&addrs, 0, sizeof(addrs));
+       memset(&attrs, 0, sizeof(attrs));
 
-    code = cm_MapVLRPCError(code, reqp);
+       attrs.Mask = VLADDR_UUID;
+       attrs.uuid = *Uuid;
 
-    if (afsd_logp->enabled) {
-       char uuidstr[128];
-       afsUUID_to_string(Uuid, uuidstr, sizeof(uuidstr));
+       do {
+           code = cm_ConnByMServers(cellp->vlServersp, 0, userp, reqp, &connp);
+           if (code)
+               continue;
+           rxconnp = cm_GetRxConn(connp);
+           code = VL_GetAddrsU(rxconnp, &attrs, &uuid, &unique, &nentries,
+                                &addrs);
+           rx_PutConnection(rxconnp);
+       } while (cm_Analyze(connp, userp, reqp, NULL, cellp, 0, NULL, NULL,
+                            &cellp->vlServersp, NULL, code));
+
+       code = cm_MapVLRPCError(code, reqp);
+
+       if (afsd_logp->enabled) {
+           char uuidstr[128];
+           afsUUID_to_string(Uuid, uuidstr, sizeof(uuidstr));
+
+           if (code)
+               osi_Log2(afsd_logp,
+                         "CALL VL_GetAddrsU serverNumber %s FAILURE, code 0x%x",
+                         osi_LogSaveString(afsd_logp, uuidstr),
+                         code);
+           else
+               osi_Log1(afsd_logp, "CALL VL_GetAddrsU serverNumber %s SUCCESS",
+                         osi_LogSaveString(afsd_logp, uuidstr));
+       }
 
        if (code)
-           osi_Log2(afsd_logp,
-                    "CALL VL_GetAddrsU serverNumber %s FAILURE, code 0x%x",
-                    osi_LogSaveString(afsd_logp, uuidstr),
-                    code);
-       else
-           osi_Log1(afsd_logp, "CALL VL_GetAddrsU serverNumber %s SUCCESS",
-                    osi_LogSaveString(afsd_logp, uuidstr));
+           return CM_ERROR_RETRY;
+
+       if (nentries == 0) {
+           xdr_free((xdrproc_t) xdr_bulkaddrs, &addrs);
+           code = CM_ERROR_INVAL;
+       } else {
+           /* addrs will either be cached or freed by cm_getaddrsAdd() */
+           entry = cm_getaddrsAdd(&uuid, unique, nentries, &addrs);
+       }
     }
 
-    if (code)
-       return CM_ERROR_RETRY;
+    if (entry != NULL) {
+       afs_uint32 * addrp;
+       afs_uint32 n;
 
-    if (nentries == 0) {
-       code = CM_ERROR_INVAL;
-    } else {
-       addrp = addrs.bulkaddrs_val;
-       for (i = 0; i < nentries && (*index) < NMAXNSERVERS; (*index)++, i++) {
+       addrp = entry->addrs.bulkaddrs_val;
+       for (n = 0; n < entry->nentries && (*index) < NMAXNSERVERS;
+            (*index)++, n++) {
            serverFlags[*index] = Flags;
-           serverNumber[*index] = addrp[i];
-           serverUUID[*index] = uuid;
-           serverUnique[*index] = unique;
+           serverNumber[*index] = addrp[n];
+           serverUUID[*index] = entry->uuid;
+           serverUnique[*index] = entry->unique;
        }
+       cm_getaddrsPut(entry);
     }
-    xdr_free((xdrproc_t) xdr_bulkaddrs, &addrs);
 
     return code;
 }
+
+void
+cm_getaddrsInit(void)
+{
+    static osi_once_t once;
+
+    if (osi_Once(&once)) {
+       lock_InitializeRWLock(&cm_getaddrsLock, "cm_getaddrsLock",
+                             LOCK_HIERARCHY_GETADDRS_GLOBAL);
+       osi_EndOnce(&once);
+    }
+}
+
+void
+cm_getaddrsShutdown(void)
+{
+    lock_FinalizeRWLock(&cm_getaddrsLock);
+}

diff --git a/src/WINNT/afsd/cm_getaddrs.h b/src/WINNT/afsd/cm_getaddrs.h
index 0f8984cf0a9fc7628de91787f3e0959782f5cacf..7d954ab13eef1a63a54274e86a4f66b5b53d4640 100644 (file)
--- a/src/WINNT/afsd/cm_getaddrs.h
+++ b/src/WINNT/afsd/cm_getaddrs.h
@@ -31,12 +31,16 @@
 #ifndef _CM_GETADDRS_H_
 #define _CM_GETADDRS_H_
 
-#include <opr/uuid.h>
-
 extern afs_uint32
 cm_GetAddrsU(cm_cell_t *cellp, cm_user_t *userp, cm_req_t *reqp,
             afsUUID *Uuid, afs_int32 Unique, afs_int32 Flags, int *index,
             afs_int32 serverFlags[], afs_int32 serverNumber[], afsUUID serverUUID[],
             afs_int32 serverUnique[]);
 
+extern void
+cm_getaddrsInit(void);
+
+extern void
+cm_getaddrsShutdown(void);
+
 #endif /* _CM_GETADDRS_H_ */

diff --git a/src/WINNT/afsd/cm_volume.c b/src/WINNT/afsd/cm_volume.c
index cf95d00d90642b9754744f42ad1a3fcddd7e9467..b338d60c86e29cac976f62a6ae0fc1e1f7eb11d0 100644 (file)
--- a/src/WINNT/afsd/cm_volume.c
+++ b/src/WINNT/afsd/cm_volume.c
@@ -112,6 +112,8 @@ cm_ShutdownVolume(void)
         lock_FinalizeRWLock(&volp->rw);
     }
 
+    cm_getaddrsShutdown();
+
     return 0;
 }
 
@@ -154,7 +156,10 @@ void cm_InitVolume(int newFile, long maxVols)
                 _InterlockedAnd(&volp->flags, ~CM_VOLUMEFLAG_RO_SIZE_VALID);
             }
         }
-        osi_EndOnce(&once);
+
+       cm_getaddrsInit();
+
+       osi_EndOnce(&once);
     }
 }