From cd1f72649650404581cfcdcf3beeeaf2bb960bd6 Mon Sep 17 00:00:00 2001 From: Derrick Brashear Date: Thu, 12 Jan 2012 16:48:54 -0500 Subject: [PATCH] libafs: add replicated connection pool keep pool of connections to use for replicated volumes, so we can have a separate idle time setting Change-Id: I61ed62c652c924b33fde920fac766c4ca0043826 Reviewed-on: http://gerrit.openafs.org/6546 Tested-by: BuildBot Reviewed-by: Derrick Brashear --- src/afs/afs.h | 5 +++++ src/afs/afs_analyze.c | 2 +- src/afs/afs_call.c | 1 + src/afs/afs_conn.c | 44 +++++++++++++++++++++++++++++++--------- src/afs/afs_pioctl.c | 2 +- src/afs/afs_prototypes.h | 4 +++- src/afs/afs_server.c | 9 ++++---- src/afs/afs_vcache.c | 2 +- src/afs/afs_volume.c | 4 ++-- 9 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/afs/afs.h b/src/afs/afs.h index 4c4873b17..9a396a4d1 100644 --- a/src/afs/afs.h +++ b/src/afs/afs.h @@ -96,12 +96,14 @@ extern int afs_shuttingdown; #define AFS_RXDEADTIME 50 #define AFS_HARDDEADTIME 120 #define AFS_IDLEDEADTIME 1200 +#define AFS_IDLEDEADTIME_REP 180 /* more than fs's cb dead time */ #define AFS_BLKBITS 12 #define AFS_BLKSIZE (1 << AFS_BLKBITS) extern afs_int32 afs_rx_deadtime; extern afs_int32 afs_rx_harddead; extern afs_int32 afs_rx_idledead; +extern afs_int32 afs_rx_idledead_rep; struct sysname_info { char *name; @@ -380,6 +382,8 @@ struct unixuser { struct sa_conn_vector; typedef struct sa_conn_vector * p_sa_conn_vector; /* forward decl */ +#define CONN_REPLICATED 0x1 + struct afs_conn { int refCount; int activated; @@ -403,6 +407,7 @@ struct sa_conn_vector { struct srvAddr *srvr; /* server associated with this conn */ short refCount; /* reference count for allocation */ unsigned short port; /* port associated with this connection */ + int flags; /* next connection to return when all in cvec are fully utilized */ int select_index; diff --git a/src/afs/afs_analyze.c b/src/afs/afs_analyze.c index 6950c9650..2f6e4003e 100644 --- a/src/afs/afs_analyze.c +++ b/src/afs/afs_analyze.c @@ -115,7 +115,7 @@ VLDB_Same(struct VenusFid *afid, struct vrequest *areq) VSleep(2); /* Better safe than sorry. */ tconn = afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum, - &treq, SHARED_LOCK, &rxconn); + &treq, SHARED_LOCK, 0, &rxconn); if (tconn) { if ( tconn->parent->srvr->server->flags & SNO_LHOSTS) { type = 0; diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c index 34b065591..0e7feb2dd 100644 --- a/src/afs/afs_call.c +++ b/src/afs/afs_call.c @@ -74,6 +74,7 @@ char afs_cachebasedir[1024]; afs_int32 afs_rx_deadtime = AFS_RXDEADTIME; afs_int32 afs_rx_harddead = AFS_HARDDEADTIME; afs_int32 afs_rx_idledead = AFS_IDLEDEADTIME; +afs_int32 afs_rx_idledead_rep = AFS_IDLEDEADTIME_REP; static int afscall_set_rxpck_received = 0; diff --git a/src/afs/afs_conn.c b/src/afs/afs_conn.c index 957365b2d..0bfb5262e 100644 --- a/src/afs/afs_conn.c +++ b/src/afs/afs_conn.c @@ -259,6 +259,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, int notbusy; int i; struct srvAddr *sa1p; + afs_int32 replicated = -1; /* a single RO will increment to 0 */ *rxconn = NULL; @@ -295,6 +296,8 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, */ for (notbusy = not_busy; (!lowp && (notbusy <= end_not_busy)); notbusy++) { for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) { + if (tv->states & VRO) + replicated++; if (((areq->tokenError > 0)||(areq->idleError > 0)) && (areq->skipserver[i] == 1)) continue; @@ -316,12 +319,20 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, } } } + if ((replicated == -1) && (tv->states & VRO)) { + for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) { + if (tv->states & VRO) + replicated++; + } + } else + replicated = 0; + afs_PutVolume(tv, READ_LOCK); if (lowp) { tu = afs_GetUser(areq->uid, afid->Cell, SHARED_LOCK); tconn = afs_ConnBySA(lowp, fsport, afid->Cell, tu, 0 /*!force */ , - 1 /*create */ , locktype, rxconn); + 1 /*create */ , locktype, replicated, rxconn); afs_PutUser(tu, SHARED_LOCK); } @@ -339,6 +350,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, * @param tu Connect as this user. * @param force_if_down * @param create + * @param replicated * @param locktype Specifies type of lock to be used for this function. * * @return The new connection. @@ -346,7 +358,8 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, struct afs_conn * afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, struct unixuser *tu, int force_if_down, afs_int32 create, - afs_int32 locktype, struct rx_connection **rxconn) + afs_int32 locktype, afs_int32 replicated, + struct rx_connection **rxconn) { int glocked, foundvec; struct afs_conn *tc = NULL; @@ -354,6 +367,7 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, struct rx_securityClass *csec; /*Security class object */ int isec; /*Security index */ int service; + int isrep = (replicated > 0)?CONN_REPLICATED:0; *rxconn = NULL; @@ -361,7 +375,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, ObtainSharedLock(&afs_xconn, 15); foundvec = 0; for (tcv = sap->conns; tcv; tcv = tcv->next) { - if (tcv->user == tu && tcv->port == aport) { + if (tcv->user == tu && tcv->port == aport && + (isrep == (tcv->flags & CONN_REPLICATED))) { /* return most eligible conn */ if (!foundvec) foundvec = 1; @@ -398,6 +413,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, tcv->port = aport; tcv->srvr = sap; tcv->next = sap->conns; + if (isrep) + tcv->flags |= CONN_REPLICATED; sap->conns = tcv; /* all struct afs_conn ptrs come from here */ @@ -462,14 +479,17 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, } /* Setting idle dead time to non-zero activates RX_CALL_IDLE errors. */ - rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead); + if (isrep) + rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead_rep); + else + rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead); /* * Only do this for the base connection, not per-user. * Will need to be revisited if/when CB gets security. */ if ((isec == 0) && (service != 52) && !(tu->states & UTokensBad) && - (tu->viceId == UNDEFVID) + (tu->viceId == UNDEFVID) && (isrep == 0) #ifndef UKERNEL /* ukernel runs as just one uid anyway */ && (tu->uid == 0) #endif @@ -502,13 +522,14 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, * @param areq The request. * @param aforce Force connection? * @param locktype Type of lock to be used. + * @param replicated * * @return The established connection. */ struct afs_conn * afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, struct vrequest *areq, int aforce, afs_int32 locktype, - struct rx_connection **rxconn) + afs_int32 replicated, struct rx_connection **rxconn) { struct unixuser *tu; struct afs_conn *tc = NULL; @@ -534,7 +555,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, for (sa = aserver->addr; sa; sa = sa->next_sa) { tc = afs_ConnBySA(sa, aport, acell, tu, aforce, 0 /*don't create one */ , - locktype, rxconn); + locktype, replicated, rxconn); if (tc) break; } @@ -543,7 +564,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, for (sa = aserver->addr; sa; sa = sa->next_sa) { tc = afs_ConnBySA(sa, aport, acell, tu, aforce, 1 /*create one */ , - locktype, rxconn); + locktype, replicated, rxconn); if (tc) break; } @@ -564,13 +585,15 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, * @param acell The cell where all of this happens. * @param areq The request. * @param locktype Type of lock to be used. + * @param replicated * * @return The established connection or NULL. */ struct afs_conn * afs_ConnByMHosts(struct server *ahosts[], unsigned short aport, afs_int32 acell, struct vrequest *areq, - afs_int32 locktype, struct rx_connection **rxconn) + afs_int32 locktype, afs_int32 replicated, + struct rx_connection **rxconn) { afs_int32 i; struct afs_conn *tconn; @@ -583,7 +606,8 @@ afs_ConnByMHosts(struct server *ahosts[], unsigned short aport, for (i = 0; i < AFS_MAXCELLHOSTS; i++) { if ((ts = ahosts[i]) == NULL) break; - tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype, rxconn); + tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype, + replicated, rxconn); if (tconn) { return tconn; } diff --git a/src/afs/afs_pioctl.c b/src/afs/afs_pioctl.c index d1af18155..5cbbdb066 100644 --- a/src/afs/afs_pioctl.c +++ b/src/afs/afs_pioctl.c @@ -5181,7 +5181,7 @@ DECL_PIOCTL(PCallBackAddr) /* get a connection, even if host is down; bumps conn ref count */ tu = afs_GetUser(areq->uid, ts->cell->cellNum, SHARED_LOCK); tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu, - 1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn); + 1 /*force */ , 1 /*create */ , SHARED_LOCK, 0, &rxconn); afs_PutUser(tu, SHARED_LOCK); if (!tc) continue; diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h index 940df15bb..71ef0a6a9 100644 --- a/src/afs/afs_prototypes.h +++ b/src/afs/afs_prototypes.h @@ -174,17 +174,19 @@ extern struct afs_conn *afs_Conn(struct VenusFid *afid, extern struct afs_conn *afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, struct unixuser *tu, int force_if_down, afs_int32 create, - afs_int32 locktype, + afs_int32 locktype, afs_int32 replicated, struct rx_connection **rxconn); extern struct afs_conn *afs_ConnByMHosts(struct server *ahosts[], unsigned short aport, afs_int32 acell, struct vrequest *areq, afs_int32 locktype, + afs_int32 replicated, struct rx_connection **rxconn); extern struct afs_conn *afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, struct vrequest *areq, int aforce, afs_int32 locktype, + afs_int32 replicated, struct rx_connection **rxconn); extern void afs_PutConn(struct afs_conn *ac, struct rx_connection *rxconn, afs_int32 locktype); diff --git a/src/afs/afs_server.c b/src/afs/afs_server.c index d8c4bbfa8..44f0f4e26 100644 --- a/src/afs/afs_server.c +++ b/src/afs/afs_server.c @@ -291,7 +291,8 @@ CheckVLServer(struct srvAddr *sa, struct vrequest *areq) return; /* can't do much */ tc = afs_ConnByHost(aserver, aserver->cell->vlport, - aserver->cell->cellNum, areq, 1, SHARED_LOCK, &rxconn); + aserver->cell->cellNum, areq, 1, SHARED_LOCK, 0, + &rxconn); if (!tc) return; rx_SetConnDeadTime(rxconn, 3); @@ -710,7 +711,8 @@ afs_LoopServers(int adown, struct cell *acellp, int vlalso, /* get a connection, even if host is down; bumps conn ref count */ tu = afs_GetUser(treq.uid, ts->cell->cellNum, SHARED_LOCK); tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu, - 1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn); + 1 /*force */ , 1 /*create */ , SHARED_LOCK, 0, + &rxconn); afs_PutUser(tu, SHARED_LOCK); if (!tc) continue; @@ -1549,8 +1551,7 @@ afs_GetCapabilities(struct server *ts) if ( !tu ) return; tc = afs_ConnBySA(ts->addr, ts->cell->fsport, ts->cell->cellNum, tu, 0, 1, - SHARED_LOCK, - &rxconn); + SHARED_LOCK, 0, &rxconn); afs_PutUser(tu, SHARED_LOCK); if ( !tc ) return; diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c index 08d54f56c..7428c94a6 100644 --- a/src/afs/afs_vcache.c +++ b/src/afs/afs_vcache.c @@ -456,7 +456,7 @@ afs_FlushVCBs(afs_int32 lockit) for (safety3 = 0; safety3 < AFS_MAXHOSTS * 2; safety3++) { tc = afs_ConnByHost(tsp, tsp->cell->fsport, tsp->cell->cellNum, &treq, 0, - SHARED_LOCK, &rxconn); + SHARED_LOCK, 0, &rxconn); if (tc) { XSTATS_START_TIME (AFS_STATS_FS_RPCIDX_GIVEUPCALLBACKS); diff --git a/src/afs/afs_volume.c b/src/afs/afs_volume.c index e12d3e37f..2602c3e57 100644 --- a/src/afs/afs_volume.c +++ b/src/afs/afs_volume.c @@ -766,7 +766,7 @@ afs_NewVolumeByName(char *aname, afs_int32 acell, int agood, do { tconn = afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum, - &treq, SHARED_LOCK, &rxconn); + &treq, SHARED_LOCK, 0, &rxconn); if (tconn) { if (tconn->parent->srvr->server->flags & SNO_LHOSTS) { type = 0; @@ -1114,7 +1114,7 @@ LockAndInstallUVolumeEntry(struct volume *av, struct uvldbentry *ve, int acell, tconn = afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum, areq, SHARED_LOCK, - &rxconn); + 0, &rxconn); if (tconn) { RX_AFS_GUNLOCK(); code = -- 2.39.5