Adds cm_RankUpServers() and cm_RankServer()

This adds the functions cm_RankUpServers() and cm_RankServer() to
the Windows cache manager. cm_RankUpServers() steps through the
list of servers, and calls cm_RankServer(), which in turn re-ranks
the servers that are currently up based on rx peer statistics as
exposed by rx_GetLocalPeers().

cm_RankUpServers() is called every 10 minutes by the cache manager
daemon, so as to allow re-ranking of the servers.

Also added is the struct server->adminRank data structure, to
allow for the modification of the rank that the admin has set,
without but basing this modification on the admin-set rank.

Change-Id: I118b885e179e4f84901dd9a3180ce821f194feb8
Reviewed-on: http://gerrit.openafs.org/317
Tested-by: Jeffrey Altman <jaltman@openafs.org>
Reviewed-by: Derrick Brashear <shadow@dementia.org>
Reviewed-by: Jeffrey Altman <jaltman@openafs.org>

diff --git a/src/WINNT/afsd/afsd_init.c b/src/WINNT/afsd/afsd_init.c
index 8760ccde5f1ee49316765c89d0b0883d2760f23e..faf74226849d8e487db93d0b244badcb95f57217 100644 (file)
--- a/src/WINNT/afsd/afsd_init.c
+++ b/src/WINNT/afsd/afsd_init.c
@@ -307,6 +307,7 @@ static void afsd_InitServerPreferences(void)
                 lock_ObtainMutex(&tsp->mx);
                 tsp->ipRank = (USHORT)dwRank;
                 tsp->flags |= CM_SERVERFLAG_PREF_SET;
+               tsp->adminRank = tsp->ipRank;
                 lock_ReleaseMutex(&tsp->mx);
 
                 /* set preferences for an existing vlserver */
@@ -319,6 +320,7 @@ static void afsd_InitServerPreferences(void)
                 lock_ObtainMutex(&tsp->mx);
                 tsp->ipRank = (USHORT)dwRank;
                 tsp->flags |= CM_SERVERFLAG_PREF_SET;
+               tsp->adminRank = tsp->ipRank;
                 lock_ReleaseMutex(&tsp->mx);
             }
         }
@@ -380,7 +382,8 @@ static void afsd_InitServerPreferences(void)
             {
                 lock_ObtainMutex(&tsp->mx);
                 tsp->ipRank = (USHORT)dwRank;
-                tsp->flags |= CM_SERVERFLAG_PREF_SET;
+               tsp->flags |= CM_SERVERFLAG_PREF_SET;
+               tsp->adminRank = tsp->ipRank;
                 lock_ReleaseMutex(&tsp->mx);
 
                 /* find volumes which might have RO copy 
@@ -396,6 +399,7 @@ static void afsd_InitServerPreferences(void)
                 lock_ObtainMutex(&tsp->mx);
                 tsp->ipRank = (USHORT)dwRank;
                 tsp->flags |= CM_SERVERFLAG_PREF_SET;
+               tsp->adminRank = tsp->ipRank;
                 lock_ReleaseMutex(&tsp->mx);
             }
         }

diff --git a/src/WINNT/afsd/cm_daemon.c b/src/WINNT/afsd/cm_daemon.c
index 8e1467730b40e266e2b59f4a20d8ea467fca81a6..e71fbbe7e1f4154df3827e42cc65ea55678930ca 100644 (file)
--- a/src/WINNT/afsd/cm_daemon.c
+++ b/src/WINNT/afsd/cm_daemon.c
@@ -35,6 +35,7 @@ long cm_daemonCheckLockInterval  = 60;
 long cm_daemonTokenCheckInterval = 180;
 long cm_daemonCheckOfflineVolInterval = 600;
 long cm_daemonPerformanceTuningInterval = 0;
+long cm_daemonRankServerInterval = 600;
 
 osi_rwlock_t cm_daemonLock;
 
@@ -336,6 +337,13 @@ cm_DaemonCheckInit(void)
     dummyLen = sizeof(DWORD);
     code = RegQueryValueEx(parmKey, "daemonPerformanceTuningInterval", NULL, NULL,
                            (BYTE *) &dummy, &dummyLen);
+    dummyLen = sizeof(DWORD);
+    code = RegQueryValueEx(parmKey, "daemonRankServerInterval", NULL, NULL,
+                           (BYTE *) &dummy, &dummyLen);
+    if (code == ERROR_SUCCESS && dummy)
+       cm_daemonRankServerInterval = dummy;
+    afsi_log("daemonRankServerInterval is %d", cm_daemonRankServerInterval);
+
     if (code == ERROR_SUCCESS)
        cm_daemonPerformanceTuningInterval = dummy;
     afsi_log("daemonPerformanceTuningInterval is %d", cm_daemonPerformanceTuningInterval);
@@ -359,6 +367,7 @@ void cm_Daemon(long parm)
     time_t lastTokenCacheCheck;
     time_t lastBusyVolCheck;
     time_t lastPerformanceCheck;
+    time_t lastServerRankCheck;
     char thostName[200];
     unsigned long code;
     struct hostent *thp;
@@ -409,6 +418,7 @@ void cm_Daemon(long parm)
     lastBusyVolCheck = now - cm_daemonCheckOfflineVolInterval/2 * (rand() % cm_daemonCheckOfflineVolInterval);
     if (cm_daemonPerformanceTuningInterval)
         lastPerformanceCheck = now - cm_daemonPerformanceTuningInterval/2 * (rand() % cm_daemonPerformanceTuningInterval);
+    lastServerRankCheck = now - cm_daemonRankServerInterval/2 * (rand() % cm_daemonRankServerInterval);
 
     while (daemon_ShutdownFlag == 0) {
         if (powerStateSuspended) {
@@ -442,11 +452,11 @@ void cm_Daemon(long parm)
            default:
                afsi_log("Unknown Windows Firewall Configuration error");
            }
-       } 
+       }
 
         /* find out what time it is */
         now = osi_Time();
-        
+
         /* Determine whether an address change took place that we need to respond to */
         if (bAddrChangeCheck)
             bAddrChangeCheck = 0;
@@ -502,6 +512,18 @@ void cm_Daemon(long parm)
            now = osi_Time();
         }
 
+       /* Rank all up servers */
+       if ((now > lastServerRankCheck + cm_daemonRankServerInterval) &&
+           daemon_ShutdownFlag == 0 &&
+           powerStateSuspended == 0) {
+           lastServerRankCheck = now;
+           osi_Log0(afsd_logp, "cm_Daemon RankServer");
+           cm_RankUpServers();
+           if(daemon_ShutdownFlag == 1 || powerStateSuspended)
+               break;
+           now = osi_Time();
+       }
+
         if (cm_daemonCheckVolCBInterval && 
             now > lastVolCBRenewalCheck + cm_daemonCheckVolCBInterval &&
             daemon_ShutdownFlag == 0 &&

diff --git a/src/WINNT/afsd/cm_ioctl.c b/src/WINNT/afsd/cm_ioctl.c
index b29c2ada01e82ae11c0e703eb7046e555c187555..e4cbb84c6ea09d673fb4e1e792a70cb3396f3546 100644 (file)
--- a/src/WINNT/afsd/cm_ioctl.c
+++ b/src/WINNT/afsd/cm_ioctl.c
@@ -1768,6 +1768,7 @@ cm_IoctlSetSPrefs(struct cm_ioctl *ioctlp, struct cm_user *userp)
             lock_ObtainMutex(&tsp->mx);
             tsp->ipRank = rank;
             tsp->flags |= CM_SERVERFLAG_PREF_SET;
+           tsp->adminRank = tsp->ipRank;
             lock_ReleaseMutex(&tsp->mx);
 
             switch (type) {
@@ -1791,6 +1792,7 @@ cm_IoctlSetSPrefs(struct cm_ioctl *ioctlp, struct cm_user *userp)
             lock_ObtainMutex(&tsp->mx);
             tsp->ipRank = rank;
             tsp->flags |= CM_SERVERFLAG_PREF_SET;
+           tsp->adminRank = tsp->ipRank;
             lock_ReleaseMutex(&tsp->mx);
             tsp->ipRank = rank;
         }

diff --git a/src/WINNT/afsd/cm_server.c b/src/WINNT/afsd/cm_server.c
index ed441cc20d318c3df1a8d8d09e5f1d1fdcee41a0..33ff2dc8b1977ed6e476f95b59247bc8aed8b40e 100644 (file)
--- a/src/WINNT/afsd/cm_server.c
+++ b/src/WINNT/afsd/cm_server.c
@@ -22,6 +22,7 @@
 #include <WINNT/afsreg.h>
 #include <osi.h>
 #include <rx/rx.h>
+#include <math.h>
 
 osi_rwlock_t cm_serverLock;
 osi_rwlock_t cm_syscfgLock;
@@ -46,6 +47,47 @@ cm_ForceNewConnectionsAllServers(void)
     lock_ReleaseRead(&cm_serverLock);
 }
 
+/*
+ * lock_ObtainMutex must be held prior to calling
+ * this function.
+ */
+afs_int32
+cm_RankServer(cm_server_t * tsp)
+{
+    afs_int32 code = 0; /* start with "success" */
+    struct rx_debugPeer tpeer;
+    afs_uint16 port;
+
+    switch(tsp->type) {
+       case CM_SERVER_VLDB:
+           port = htons(7003);
+           break;
+       case CM_SERVER_FILE:
+           port = htons(7000);
+           break;
+       default:
+           return -1;
+    }
+
+    code = rx_GetLocalPeers(tsp->addr.sin_addr.s_addr, port, &tpeer);
+
+    /*check if rx_GetLocalPeers succeeded and if there is data for tsp */
+    if(code == 0 && (tpeer.rtt == 0 && tpeer.rtt_dev == 0))
+       code = -1;
+
+    if(code == 0) {
+       if((tsp->flags & CM_SERVERFLAG_PREF_SET))
+           tsp->ipRank = tsp->adminRank + ((int)(623 * log(tpeer.rtt) / 10) *
+                                       10 + 5);
+       else /* rank has not been set by admin, derive rank from rtt */
+           tsp->ipRank = (int)(7200 * log(tpeer.rtt) / 5000) * 5000 + 5000;
+
+       tsp->ipRank += (rand() & 0x000f); /* randomize */
+    }
+
+    return code;
+}
+
 void 
 cm_PingServer(cm_server_t *tsp)
 {
@@ -200,6 +242,30 @@ cm_PingServer(cm_server_t *tsp)
     lock_ReleaseMutex(&tsp->mx);
 }
 
+void
+cm_RankUpServers()
+{
+    cm_server_t * tsp;
+
+    lock_ObtainRead(&cm_serverLock);
+    for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
+       cm_GetServerNoLock(tsp);
+       lock_ReleaseRead(&cm_serverLock);
+
+       lock_ObtainMutex(&tsp->mx);
+
+        /* if the server is not down, rank the server */
+        if(!(tsp->flags & CM_SERVERFLAG_DOWN))
+          cm_RankServer(tsp);
+
+       lock_ReleaseMutex(&tsp->mx);
+
+       lock_ObtainRead(&cm_serverLock);
+       cm_PutServerNoLock(tsp);
+    }
+    lock_ReleaseRead(&cm_serverLock);
+}
+
 static void cm_CheckServersSingular(afs_uint32 flags, cm_cell_t *cellp)
 {
     /* ping all file servers, up or down, with unauthenticated connection,

diff --git a/src/WINNT/afsd/cm_server.h b/src/WINNT/afsd/cm_server.h
index f4011237ed0bb0659e0993500a1eabf1014dec59..defb81d80ce1fec2f95d91bcc178eec308a87158 100644 (file)
--- a/src/WINNT/afsd/cm_server.h
+++ b/src/WINNT/afsd/cm_server.h
@@ -38,6 +38,7 @@ typedef struct cm_server {
     cm_server_vols_t *  vols;           /* by mx */
     time_t downTime;                    /* by mx */
     afsUUID uuid;                       /* by mx */
+    unsigned short adminRank;          /* only set if admin sets a rank */
 } cm_server_t;
 
 enum repstate {srv_not_busy, srv_busy, srv_offline, srv_deleted};
@@ -106,6 +107,10 @@ extern void cm_CheckServers(afs_uint32 flags, struct cm_cell *cellp);
 
 extern cm_server_t *cm_allServersp;
 
+extern afs_int32 cm_RankServer(cm_server_t * server);
+
+extern void cm_RankUpServers();
+
 extern void cm_SetServerPrefs(cm_server_t * serverp);
 
 extern void cm_InsertServerList(cm_serverRef_t** list,cm_serverRef_t* element);