From d22c05e6799649a1b5c06b0396e78b0520b3e4f6 Mon Sep 17 00:00:00 2001 From: Jacob Thebault-Spieker Date: Wed, 12 Aug 2009 01:32:06 -0400 Subject: [PATCH] Adds cm_RankUpServers() and cm_RankServer() This adds the functions cm_RankUpServers() and cm_RankServer() to the Windows cache manager. cm_RankUpServers() steps through the list of servers, and calls cm_RankServer(), which in turn re-ranks the servers that are currently up based on rx peer statistics as exposed by rx_GetLocalPeers(). cm_RankUpServers() is called every 10 minutes by the cache manager daemon, so as to allow re-ranking of the servers. Also added is the struct server->adminRank data structure, to allow for the modification of the rank that the admin has set, without but basing this modification on the admin-set rank. Change-Id: I118b885e179e4f84901dd9a3180ce821f194feb8 Reviewed-on: http://gerrit.openafs.org/317 Tested-by: Jeffrey Altman Reviewed-by: Derrick Brashear Reviewed-by: Jeffrey Altman --- src/WINNT/afsd/afsd_init.c | 6 +++- src/WINNT/afsd/cm_daemon.c | 26 +++++++++++++-- src/WINNT/afsd/cm_ioctl.c | 2 ++ src/WINNT/afsd/cm_server.c | 66 ++++++++++++++++++++++++++++++++++++++ src/WINNT/afsd/cm_server.h | 5 +++ 5 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/WINNT/afsd/afsd_init.c b/src/WINNT/afsd/afsd_init.c index 8760ccde5..faf742268 100644 --- a/src/WINNT/afsd/afsd_init.c +++ b/src/WINNT/afsd/afsd_init.c @@ -307,6 +307,7 @@ static void afsd_InitServerPreferences(void) lock_ObtainMutex(&tsp->mx); tsp->ipRank = (USHORT)dwRank; tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->adminRank = tsp->ipRank; lock_ReleaseMutex(&tsp->mx); /* set preferences for an existing vlserver */ @@ -319,6 +320,7 @@ static void afsd_InitServerPreferences(void) lock_ObtainMutex(&tsp->mx); tsp->ipRank = (USHORT)dwRank; tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->adminRank = tsp->ipRank; lock_ReleaseMutex(&tsp->mx); } } @@ -380,7 +382,8 @@ static void afsd_InitServerPreferences(void) { lock_ObtainMutex(&tsp->mx); tsp->ipRank = (USHORT)dwRank; - tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->adminRank = tsp->ipRank; lock_ReleaseMutex(&tsp->mx); /* find volumes which might have RO copy @@ -396,6 +399,7 @@ static void afsd_InitServerPreferences(void) lock_ObtainMutex(&tsp->mx); tsp->ipRank = (USHORT)dwRank; tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->adminRank = tsp->ipRank; lock_ReleaseMutex(&tsp->mx); } } diff --git a/src/WINNT/afsd/cm_daemon.c b/src/WINNT/afsd/cm_daemon.c index 8e1467730..e71fbbe7e 100644 --- a/src/WINNT/afsd/cm_daemon.c +++ b/src/WINNT/afsd/cm_daemon.c @@ -35,6 +35,7 @@ long cm_daemonCheckLockInterval = 60; long cm_daemonTokenCheckInterval = 180; long cm_daemonCheckOfflineVolInterval = 600; long cm_daemonPerformanceTuningInterval = 0; +long cm_daemonRankServerInterval = 600; osi_rwlock_t cm_daemonLock; @@ -336,6 +337,13 @@ cm_DaemonCheckInit(void) dummyLen = sizeof(DWORD); code = RegQueryValueEx(parmKey, "daemonPerformanceTuningInterval", NULL, NULL, (BYTE *) &dummy, &dummyLen); + dummyLen = sizeof(DWORD); + code = RegQueryValueEx(parmKey, "daemonRankServerInterval", NULL, NULL, + (BYTE *) &dummy, &dummyLen); + if (code == ERROR_SUCCESS && dummy) + cm_daemonRankServerInterval = dummy; + afsi_log("daemonRankServerInterval is %d", cm_daemonRankServerInterval); + if (code == ERROR_SUCCESS) cm_daemonPerformanceTuningInterval = dummy; afsi_log("daemonPerformanceTuningInterval is %d", cm_daemonPerformanceTuningInterval); @@ -359,6 +367,7 @@ void cm_Daemon(long parm) time_t lastTokenCacheCheck; time_t lastBusyVolCheck; time_t lastPerformanceCheck; + time_t lastServerRankCheck; char thostName[200]; unsigned long code; struct hostent *thp; @@ -409,6 +418,7 @@ void cm_Daemon(long parm) lastBusyVolCheck = now - cm_daemonCheckOfflineVolInterval/2 * (rand() % cm_daemonCheckOfflineVolInterval); if (cm_daemonPerformanceTuningInterval) lastPerformanceCheck = now - cm_daemonPerformanceTuningInterval/2 * (rand() % cm_daemonPerformanceTuningInterval); + lastServerRankCheck = now - cm_daemonRankServerInterval/2 * (rand() % cm_daemonRankServerInterval); while (daemon_ShutdownFlag == 0) { if (powerStateSuspended) { @@ -442,11 +452,11 @@ void cm_Daemon(long parm) default: afsi_log("Unknown Windows Firewall Configuration error"); } - } + } /* find out what time it is */ now = osi_Time(); - + /* Determine whether an address change took place that we need to respond to */ if (bAddrChangeCheck) bAddrChangeCheck = 0; @@ -502,6 +512,18 @@ void cm_Daemon(long parm) now = osi_Time(); } + /* Rank all up servers */ + if ((now > lastServerRankCheck + cm_daemonRankServerInterval) && + daemon_ShutdownFlag == 0 && + powerStateSuspended == 0) { + lastServerRankCheck = now; + osi_Log0(afsd_logp, "cm_Daemon RankServer"); + cm_RankUpServers(); + if(daemon_ShutdownFlag == 1 || powerStateSuspended) + break; + now = osi_Time(); + } + if (cm_daemonCheckVolCBInterval && now > lastVolCBRenewalCheck + cm_daemonCheckVolCBInterval && daemon_ShutdownFlag == 0 && diff --git a/src/WINNT/afsd/cm_ioctl.c b/src/WINNT/afsd/cm_ioctl.c index b29c2ada0..e4cbb84c6 100644 --- a/src/WINNT/afsd/cm_ioctl.c +++ b/src/WINNT/afsd/cm_ioctl.c @@ -1768,6 +1768,7 @@ cm_IoctlSetSPrefs(struct cm_ioctl *ioctlp, struct cm_user *userp) lock_ObtainMutex(&tsp->mx); tsp->ipRank = rank; tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->adminRank = tsp->ipRank; lock_ReleaseMutex(&tsp->mx); switch (type) { @@ -1791,6 +1792,7 @@ cm_IoctlSetSPrefs(struct cm_ioctl *ioctlp, struct cm_user *userp) lock_ObtainMutex(&tsp->mx); tsp->ipRank = rank; tsp->flags |= CM_SERVERFLAG_PREF_SET; + tsp->adminRank = tsp->ipRank; lock_ReleaseMutex(&tsp->mx); tsp->ipRank = rank; } diff --git a/src/WINNT/afsd/cm_server.c b/src/WINNT/afsd/cm_server.c index ed441cc20..33ff2dc8b 100644 --- a/src/WINNT/afsd/cm_server.c +++ b/src/WINNT/afsd/cm_server.c @@ -22,6 +22,7 @@ #include #include #include +#include osi_rwlock_t cm_serverLock; osi_rwlock_t cm_syscfgLock; @@ -46,6 +47,47 @@ cm_ForceNewConnectionsAllServers(void) lock_ReleaseRead(&cm_serverLock); } +/* + * lock_ObtainMutex must be held prior to calling + * this function. + */ +afs_int32 +cm_RankServer(cm_server_t * tsp) +{ + afs_int32 code = 0; /* start with "success" */ + struct rx_debugPeer tpeer; + afs_uint16 port; + + switch(tsp->type) { + case CM_SERVER_VLDB: + port = htons(7003); + break; + case CM_SERVER_FILE: + port = htons(7000); + break; + default: + return -1; + } + + code = rx_GetLocalPeers(tsp->addr.sin_addr.s_addr, port, &tpeer); + + /*check if rx_GetLocalPeers succeeded and if there is data for tsp */ + if(code == 0 && (tpeer.rtt == 0 && tpeer.rtt_dev == 0)) + code = -1; + + if(code == 0) { + if((tsp->flags & CM_SERVERFLAG_PREF_SET)) + tsp->ipRank = tsp->adminRank + ((int)(623 * log(tpeer.rtt) / 10) * + 10 + 5); + else /* rank has not been set by admin, derive rank from rtt */ + tsp->ipRank = (int)(7200 * log(tpeer.rtt) / 5000) * 5000 + 5000; + + tsp->ipRank += (rand() & 0x000f); /* randomize */ + } + + return code; +} + void cm_PingServer(cm_server_t *tsp) { @@ -200,6 +242,30 @@ cm_PingServer(cm_server_t *tsp) lock_ReleaseMutex(&tsp->mx); } +void +cm_RankUpServers() +{ + cm_server_t * tsp; + + lock_ObtainRead(&cm_serverLock); + for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) { + cm_GetServerNoLock(tsp); + lock_ReleaseRead(&cm_serverLock); + + lock_ObtainMutex(&tsp->mx); + + /* if the server is not down, rank the server */ + if(!(tsp->flags & CM_SERVERFLAG_DOWN)) + cm_RankServer(tsp); + + lock_ReleaseMutex(&tsp->mx); + + lock_ObtainRead(&cm_serverLock); + cm_PutServerNoLock(tsp); + } + lock_ReleaseRead(&cm_serverLock); +} + static void cm_CheckServersSingular(afs_uint32 flags, cm_cell_t *cellp) { /* ping all file servers, up or down, with unauthenticated connection, diff --git a/src/WINNT/afsd/cm_server.h b/src/WINNT/afsd/cm_server.h index f4011237e..defb81d80 100644 --- a/src/WINNT/afsd/cm_server.h +++ b/src/WINNT/afsd/cm_server.h @@ -38,6 +38,7 @@ typedef struct cm_server { cm_server_vols_t * vols; /* by mx */ time_t downTime; /* by mx */ afsUUID uuid; /* by mx */ + unsigned short adminRank; /* only set if admin sets a rank */ } cm_server_t; enum repstate {srv_not_busy, srv_busy, srv_offline, srv_deleted}; @@ -106,6 +107,10 @@ extern void cm_CheckServers(afs_uint32 flags, struct cm_cell *cellp); extern cm_server_t *cm_allServersp; +extern afs_int32 cm_RankServer(cm_server_t * server); + +extern void cm_RankUpServers(); + extern void cm_SetServerPrefs(cm_server_t * serverp); extern void cm_InsertServerList(cm_serverRef_t** list,cm_serverRef_t* element); -- 2.39.5