From: Marc Dionne Date: Sun, 23 Jan 2011 18:09:48 +0000 (-0500) Subject: ubik: Introduce new address lock X-Git-Tag: upstream/1.8.0_pre1^2~3899 X-Git-Url: https://git.michaelhowe.org/gitweb/?a=commitdiff_plain;h=5ceea5bbc6a3a69012d625d53149909490311e6c;p=packages%2Fo%2Fopenafs.git ubik: Introduce new address lock Introduce a new lock to protect: ubik_server->addr[] ubik_server->disk_rxcid ubik_server->vote_rxcid ubikSecClass ubikSecIndex Globals are put into a new addr_data structure along with the lock. Based on analysis and design work from Jeffrey Hutzelman. Change-Id: I33ef58a36e87a3925b310052f83bb3c6dd60d785 Reviewed-on: http://gerrit.openafs.org/4157 Tested-by: BuildBot Reviewed-by: Jeffrey Altman Reviewed-by: Derrick Brashear --- diff --git a/src/ubik/beacon.c b/src/ubik/beacon.c index b3e69bf93..ae482bb2b 100644 --- a/src/ubik/beacon.c +++ b/src/ubik/beacon.c @@ -50,6 +50,9 @@ static void * securityRock = NULL; afs_int32 ubikSecIndex; struct rx_securityClass *ubikSecClass; +/* Values protected by the address lock */ +struct addr_data addr_globals; + /* Values protected by the beacon lock */ struct beacon_data beacon_globals; @@ -172,22 +175,23 @@ ubeacon_InitServerList(afs_uint32 ame, afs_uint32 aservers[]) return code; } +/* Must be called with address lock held */ void ubeacon_InitSecurityClass(void) { int i; /* get the security index to use, if we can */ if (secLayerProc) { - i = (*secLayerProc) (securityRock, &ubikSecClass, &ubikSecIndex); + i = (*secLayerProc) (securityRock, &addr_globals.ubikSecClass, &addr_globals.ubikSecIndex); } else if (ubik_CRXSecurityProc) { - i = (*ubik_CRXSecurityProc) (ubik_CRXSecurityRock, &ubikSecClass, - &ubikSecIndex); + i = (*ubik_CRXSecurityProc) (ubik_CRXSecurityRock, &addr_globals.ubikSecClass, + &addr_globals.ubikSecIndex); } else i = 1; if (i) { /* don't have sec module yet */ - ubikSecIndex = 0; - ubikSecClass = rxnull_NewClientSecurityObject(); + addr_globals.ubikSecIndex = 0; + addr_globals.ubikSecClass = rxnull_NewClientSecurityObject(); } } @@ -198,11 +202,12 @@ ubeacon_ReinitServer(struct ubik_server *ts) struct rx_connection *disk_rxcid; struct rx_connection *vote_rxcid; struct rx_connection *tmp; + UBIK_ADDR_LOCK; ubeacon_InitSecurityClass(); disk_rxcid = rx_NewConnection(rx_HostOf(rx_PeerOf(ts->disk_rxcid)), ubik_callPortal, DISK_SERVICE_ID, - ubikSecClass, ubikSecIndex); + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); if (disk_rxcid) { tmp = ts->disk_rxcid; ts->disk_rxcid = disk_rxcid; @@ -211,12 +216,13 @@ ubeacon_ReinitServer(struct ubik_server *ts) vote_rxcid = rx_NewConnection(rx_HostOf(rx_PeerOf(ts->vote_rxcid)), ubik_callPortal, VOTE_SERVICE_ID, - ubikSecClass, ubikSecIndex); + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); if (vote_rxcid) { tmp = ts->vote_rxcid; ts->vote_rxcid = vote_rxcid; rx_PutConnection(tmp); } + UBIK_ADDR_UNLOCK; } } @@ -300,12 +306,12 @@ ubeacon_InitServerListCommon(afs_uint32 ame, struct afsconf_cell *info, ts->vote_rxcid = rx_NewConnection(info->hostAddr[i].sin_addr.s_addr, ubik_callPortal, VOTE_SERVICE_ID, - ubikSecClass, ubikSecIndex); + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); /* for disk reqs */ ts->disk_rxcid = rx_NewConnection(info->hostAddr[i].sin_addr.s_addr, ubik_callPortal, DISK_SERVICE_ID, - ubikSecClass, ubikSecIndex); + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); ts->up = 1; } } else { @@ -318,8 +324,10 @@ ubeacon_InitServerListCommon(afs_uint32 ame, struct afsconf_cell *info, ts->next = ubik_servers; ubik_servers = ts; ts->addr[0] = servAddr; /* primary address in net byte order */ - ts->vote_rxcid = rx_NewConnection(servAddr, ubik_callPortal, VOTE_SERVICE_ID, ubikSecClass, ubikSecIndex); /* for vote reqs */ - ts->disk_rxcid = rx_NewConnection(servAddr, ubik_callPortal, DISK_SERVICE_ID, ubikSecClass, ubikSecIndex); /* for disk reqs */ + ts->vote_rxcid = rx_NewConnection(servAddr, ubik_callPortal, VOTE_SERVICE_ID, + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); /* for vote reqs */ + ts->disk_rxcid = rx_NewConnection(servAddr, ubik_callPortal, DISK_SERVICE_ID, + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); /* for disk reqs */ ts->isClone = 0; /* don't know about clones */ ts->up = 1; if (ntohl((afs_uint32) servAddr) < (afs_uint32) magicHost) { @@ -425,12 +433,14 @@ ubeacon_Interact(void *dummy) * prepare to send them an r multi-call containing the beacon message */ i = 0; /* collect connections */ UBIK_BEACON_LOCK; + UBIK_ADDR_LOCK; for (ts = ubik_servers; ts; ts = ts->next) { if (ts->up && ts->addr[0] != ubik_host[0]) { servers[i] = ts; connections[i++] = ts->vote_rxcid; } } + UBIK_ADDR_UNLOCK; UBIK_BEACON_UNLOCK; servers[i] = (struct ubik_server *)0; /* end of list */ /* note that we assume in the vote module that we'll always get at least BIGTIME @@ -721,10 +731,12 @@ updateUbikNetworkAddress(afs_uint32 ubik_host[UBIK_MAX_INTERFACE_ADDR]) char buffer[32]; char hoststr[16]; + UBIK_ADDR_LOCK; for (count = 0, ts = ubik_servers; ts; count++, ts = ts->next) { conns[count] = ts->disk_rxcid; server[count] = ts; } + UBIK_ADDR_UNLOCK; /* inform all other servers only if there are more than one @@ -741,6 +753,7 @@ updateUbikNetworkAddress(afs_uint32 ubik_host[UBIK_MAX_INTERFACE_ADDR]) multi_DISK_UpdateInterfaceAddr(&inAddr, &outAddr); ts = server[multi_i]; /* reply received from this server */ if (!multi_error) { + UBIK_ADDR_LOCK; if (ts->addr[0] != htonl(outAddr.hostAddr[0])) { code = UBADHOST; strcpy(buffer, afs_inet_ntoa_r(ts->addr[0], hoststr)); @@ -751,15 +764,20 @@ updateUbikNetworkAddress(afs_uint32 ubik_host[UBIK_MAX_INTERFACE_ADDR]) for (j = 1; j < UBIK_MAX_INTERFACE_ADDR; j++) ts->addr[j] = htonl(outAddr.hostAddr[j]); } + UBIK_ADDR_UNLOCK; } else if (multi_error == RXGEN_OPCODE) { /* pre 3.5 remote server */ + UBIK_ADDR_LOCK; ubik_print ("ubik server %s does not support UpdateInterfaceAddr RPC\n", afs_inet_ntoa_r(ts->addr[0], hoststr)); + UBIK_ADDR_UNLOCK; } else if (multi_error == UBADHOST) { code = UBADHOST; /* remote CellServDB inconsistency */ ubik_print("Inconsistent Cell Info on server: "); + UBIK_ADDR_LOCK; for (j = 0; j < UBIK_MAX_INTERFACE_ADDR && ts->addr[j]; j++) ubik_print("%s ", afs_inet_ntoa_r(ts->addr[j], hoststr)); + UBIK_ADDR_UNLOCK; ubik_print("\n"); } else { UBIK_BEACON_LOCK; diff --git a/src/ubik/recovery.c b/src/ubik/recovery.c index 79147f3b4..b3a00712e 100644 --- a/src/ubik/recovery.c +++ b/src/ubik/recovery.c @@ -528,7 +528,9 @@ urecovery_Interact(void *dummy) UBIK_BEACON_UNLOCK; if (ts->isClone) continue; + UBIK_ADDR_LOCK; code = DISK_GetVersion(ts->disk_rxcid, &ts->version); + UBIK_ADDR_UNLOCK; if (code == 0) { /* perhaps this is the best version */ if (vcmp(ts->version, bestDBVersion) > 0) { @@ -571,10 +573,12 @@ urecovery_Interact(void *dummy) /* Rx code to do the Bulk fetch */ file = 0; offset = 0; + UBIK_ADDR_LOCK; rxcall = rx_NewCall(bestServer->disk_rxcid); ubik_print("Ubik: Synchronize database with server %s\n", afs_inet_ntoa_r(bestServer->addr[0], hoststr)); + UBIK_ADDR_UNLOCK; code = StartDISK_GetFile(rxcall, file); if (code) { @@ -761,7 +765,9 @@ urecovery_Interact(void *dummy) } for (ts = ubik_servers; ts; ts = ts->next) { + UBIK_ADDR_LOCK; inAddr.s_addr = ts->addr[0]; + UBIK_ADDR_UNLOCK; UBIK_BEACON_LOCK; if (!ts->up) { UBIK_BEACON_UNLOCK; @@ -781,7 +787,9 @@ urecovery_Interact(void *dummy) if (!code) { length = ubikstat.size; file = offset = 0; + UBIK_ADDR_LOCK; rxcall = rx_NewCall(ts->disk_rxcid); + UBIK_ADDR_UNLOCK; code = StartDISK_SendFile(rxcall, file, length, &ubik_dbase->version); @@ -848,14 +856,13 @@ DoProbe(struct ubik_server *server) afs_uint32 addr; char buffer[32]; char hoststr[16]; - extern afs_int32 ubikSecIndex; - extern struct rx_securityClass *ubikSecClass; + UBIK_ADDR_LOCK; for (i = 0; (addr = server->addr[i]) && (i < UBIK_MAX_INTERFACE_ADDR); i++) { conns[i] = rx_NewConnection(addr, ubik_callPortal, DISK_SERVICE_ID, - ubikSecClass, ubikSecIndex); + addr_globals.ubikSecClass, addr_globals.ubikSecIndex); /* user requirement to use only the primary interface */ if (ubikPrimaryAddrOnly) { @@ -863,6 +870,7 @@ DoProbe(struct ubik_server *server) break; } } + UBIK_ADDR_UNLOCK; osi_Assert(i); /* at least one interface address for this server */ #ifdef AFS_PTHREAD_ENV @@ -883,6 +891,7 @@ DoProbe(struct ubik_server *server) #endif if (success_i >= 0) { + UBIK_ADDR_LOCK; addr = server->addr[success_i]; /* successful interface addr */ if (server->disk_rxcid) /* destroy existing conn */ @@ -893,14 +902,15 @@ DoProbe(struct ubik_server *server) /* make new connections */ server->disk_rxcid = conns[success_i]; server->vote_rxcid = rx_NewConnection(addr, ubik_callPortal, - VOTE_SERVICE_ID, ubikSecClass, - ubikSecIndex); + VOTE_SERVICE_ID, addr_globals.ubikSecClass, + addr_globals.ubikSecIndex); connSuccess = conns[success_i]; strcpy(buffer, afs_inet_ntoa_r(server->addr[0], hoststr)); ubik_print("ubik:server %s is back up: will be contacted through %s\n", buffer, afs_inet_ntoa_r(addr, hoststr)); + UBIK_ADDR_UNLOCK; } /* Destroy all connections except the one on which we succeeded */ diff --git a/src/ubik/remote.c b/src/ubik/remote.c index 44b132f08..7dd560ce3 100644 --- a/src/ubik/remote.c +++ b/src/ubik/remote.c @@ -605,6 +605,7 @@ SDISK_UpdateInterfaceAddr(struct rx_call *rxcall, int i, j, found = 0, probableMatch = 0; char hoststr[16]; + UBIK_ADDR_LOCK; /* copy the output parameters */ for (i = 0; i < UBIK_MAX_INTERFACE_ADDR; i++) outAddr->hostAddr[i] = ntohl(ubik_host[i]); @@ -646,6 +647,7 @@ SDISK_UpdateInterfaceAddr(struct rx_call *rxcall, fflush(stdout); fflush(stderr); printServerInfo(); + UBIK_ADDR_UNLOCK; return UBADHOST; } @@ -658,6 +660,7 @@ SDISK_UpdateInterfaceAddr(struct rx_call *rxcall, ubik_print("%s ", afs_inet_ntoa_r(ts->addr[i], hoststr)); ubik_print("\n"); + UBIK_ADDR_UNLOCK; return 0; } diff --git a/src/ubik/ubik.c b/src/ubik/ubik.c index d3d0b9918..c73e78142 100644 --- a/src/ubik/ubik.c +++ b/src/ubik/ubik.c @@ -105,11 +105,15 @@ Quorum_StartIO(struct ubik_trans *atrans, struct ubik_server *as) { struct rx_connection *conn; + UBIK_ADDR_LOCK; conn = as->disk_rxcid; #ifdef AFS_PTHREAD_ENV rx_GetConnection(conn); + UBIK_ADDR_UNLOCK; DBRELE(atrans->dbase); +#else + UBIK_ADDR_UNLOCK; #endif /* AFS_PTHREAD_ENV */ return conn; @@ -407,6 +411,7 @@ ubik_ServerInitCommon(afs_uint32 myHost, short myPort, MUTEX_INIT(&tdb->versionLock, "version lock", MUTEX_DEFAULT, 0); MUTEX_INIT(&beacon_globals.beacon_lock, "beacon lock", MUTEX_DEFAULT, 0); MUTEX_INIT(&vote_globals.vote_lock, "vote lock", MUTEX_DEFAULT, 0); + MUTEX_INIT(&addr_globals.addr_lock, "address lock", MUTEX_DEFAULT, 0); #else Lock_Init(&tdb->versionLock); #endif @@ -1368,10 +1373,14 @@ ubikGetPrimaryInterfaceAddr(afs_uint32 addr) struct ubik_server *ts; int j; + UBIK_ADDR_LOCK; for (ts = ubik_servers; ts; ts = ts->next) for (j = 0; j < UBIK_MAX_INTERFACE_ADDR; j++) - if (ts->addr[j] == addr) + if (ts->addr[j] == addr) { + UBIK_ADDR_UNLOCK; return ts->addr[0]; /* net byte order */ + } + UBIK_ADDR_UNLOCK; return 0; /* if not in server database, return error */ } diff --git a/src/ubik/ubik.p.h b/src/ubik/ubik.p.h index 20aebc727..78518969b 100644 --- a/src/ubik/ubik.p.h +++ b/src/ubik/ubik.p.h @@ -393,6 +393,24 @@ struct vote_data { #define UBIK_VOTE_LOCK MUTEX_ENTER(&vote_globals.vote_lock) #define UBIK_VOTE_UNLOCK MUTEX_EXIT(&vote_globals.vote_lock) +/*! + * \brief Server address data. All values are protected by addr_lock + * + * This lock also protects: + * ubik_server: addr[], vote_rxcid, disk_rxcid + * + */ +struct addr_data { +#ifdef AFS_PTHREAD_ENV + pthread_mutex_t addr_lock; +#endif + afs_int32 ubikSecIndex; + struct rx_securityClass *ubikSecClass; +}; + +#define UBIK_ADDR_LOCK MUTEX_ENTER(&addr_globals.addr_lock) +#define UBIK_ADDR_UNLOCK MUTEX_EXIT(&addr_globals.addr_lock) + /* phys.c */ extern int uphys_stat(struct ubik_dbase *adbase, afs_int32 afid, struct ubik_stat *astat); @@ -477,6 +495,7 @@ extern int ubeacon_InitServerListByInfo(afs_uint32 ame, extern int ubeacon_InitServerList(afs_uint32 ame, afs_uint32 aservers[]); extern void *ubeacon_Interact(void *); extern struct beacon_data beacon_globals; +extern struct addr_data addr_globals; /*\}*/