From 05fb2529771d8f2230d384de1974b41c5ac57175 Mon Sep 17 00:00:00 2001 From: Derrick Brashear Date: Thu, 12 Jan 2012 16:48:54 -0500 Subject: [PATCH] libafs: add replicated connection pool keep pool of connections to use for replicated volumes, so we can have a separate idle time setting (cherry picked from commit cd1f72649650404581cfcdcf3beeeaf2bb960bd6) Reviewed-on: http://gerrit.openafs.org/6546 Tested-by: BuildBot Reviewed-by: Derrick Brashear Change-Id: I056ba28d11313c9925df63869e0c55a1a4f132da Reviewed-on: http://gerrit.openafs.org/6610 Tested-by: BuildBot Reviewed-by: Derrick Brashear --- src/afs/afs.h | 6 +++++ src/afs/afs_analyze.c | 2 +- src/afs/afs_call.c | 1 + src/afs/afs_conn.c | 49 ++++++++++++++++++++++++++++++++-------- src/afs/afs_pioctl.c | 2 +- src/afs/afs_prototypes.h | 4 +++- src/afs/afs_server.c | 9 ++++---- src/afs/afs_vcache.c | 2 +- src/afs/afs_volume.c | 4 ++-- 9 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/afs/afs.h b/src/afs/afs.h index 5581f9b2b..cf9345d48 100644 --- a/src/afs/afs.h +++ b/src/afs/afs.h @@ -96,12 +96,14 @@ extern int afs_shuttingdown; #define AFS_RXDEADTIME 50 #define AFS_HARDDEADTIME 120 #define AFS_IDLEDEADTIME 1200 +#define AFS_IDLEDEADTIME_REP 180 /* more than fs's cb dead time */ #define AFS_BLKBITS 12 #define AFS_BLKSIZE (1 << AFS_BLKBITS) extern afs_int32 afs_rx_deadtime; extern afs_int32 afs_rx_harddead; extern afs_int32 afs_rx_idledead; +extern afs_int32 afs_rx_idledead_rep; struct sysname_info { char *name; @@ -360,6 +362,9 @@ struct unixuser { void *cellinfo; /* pointer to cell info (PAG manager only) */ }; + +#define CONN_REPLICATED 0x1 + struct afs_conn { /* Per-connection block. */ struct afs_conn *next; /* Next dude same server. */ @@ -369,6 +374,7 @@ struct afs_conn { short refCount; /* reference count for allocation */ unsigned short port; /* port associated with this connection */ char forceConnectFS; /* Should we try again with these tokens? */ + int flags; }; diff --git a/src/afs/afs_analyze.c b/src/afs/afs_analyze.c index 1d1b7c40b..6dba5c0de 100644 --- a/src/afs/afs_analyze.c +++ b/src/afs/afs_analyze.c @@ -117,7 +117,7 @@ VLDB_Same(struct VenusFid *afid, struct vrequest *areq) VSleep(2); /* Better safe than sorry. */ tconn = afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum, - &treq, SHARED_LOCK, &rxconn); + &treq, SHARED_LOCK, 0, &rxconn); if (tconn) { if (tconn->srvr->server->flags & SNO_LHOSTS) { type = 0; diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c index 8334db87e..83dfd37a7 100644 --- a/src/afs/afs_call.c +++ b/src/afs/afs_call.c @@ -75,6 +75,7 @@ char afs_cachebasedir[1024]; afs_int32 afs_rx_deadtime = AFS_RXDEADTIME; afs_int32 afs_rx_harddead = AFS_HARDDEADTIME; afs_int32 afs_rx_idledead = AFS_IDLEDEADTIME; +afs_int32 afs_rx_idledead_rep = AFS_IDLEDEADTIME_REP; static int afscall_set_rxpck_received = 0; diff --git a/src/afs/afs_conn.c b/src/afs/afs_conn.c index 3de6c18dd..434c7617d 100644 --- a/src/afs/afs_conn.c +++ b/src/afs/afs_conn.c @@ -125,6 +125,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, int notbusy; int i; struct srvAddr *sa1p; + afs_int32 replicated = -1; /* a single RO will increment to 0 */ *rxconn = NULL; @@ -161,6 +162,8 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, */ for (notbusy = not_busy; (!lowp && (notbusy <= end_not_busy)); notbusy++) { for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) { + if (tv->states & VRO) + replicated++; if (((areq->tokenError > 0)||(areq->idleError > 0)) && (areq->skipserver[i] == 1)) continue; @@ -182,12 +185,20 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, } } } + if ((replicated == -1) && (tv->states & VRO)) { + for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) { + if (tv->states & VRO) + replicated++; + } + } else + replicated = 0; + afs_PutVolume(tv, READ_LOCK); if (lowp) { tu = afs_GetUser(areq->uid, afid->Cell, SHARED_LOCK); tconn = afs_ConnBySA(lowp, fsport, afid->Cell, tu, 0 /*!force */ , - 1 /*create */ , locktype, rxconn); + 1 /*create */ , locktype, replicated, rxconn); afs_PutUser(tu, SHARED_LOCK); } @@ -205,6 +216,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, * @param tu Connect as this user. * @param force_if_down * @param create + * @param replicated * @param locktype Specifies type of lock to be used for this function. * * @return The new connection. @@ -212,12 +224,14 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq, struct afs_conn * afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, struct unixuser *tu, int force_if_down, afs_int32 create, - afs_int32 locktype, struct rx_connection **rxconn) + afs_int32 locktype, afs_int32 replicated, + struct rx_connection **rxconn) { struct afs_conn *tc = 0; struct rx_securityClass *csec; /*Security class object */ int isec; /*Security index */ int service; + int isrep = (replicated > 0)?CONN_REPLICATED:0; *rxconn = NULL; @@ -229,7 +243,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, ObtainSharedLock(&afs_xconn, 15); /* Get conn by port and user. */ for (tc = sap->conns; tc; tc = tc->next) { - if (tc->user == tu && tc->port == aport) { + if (tc->user == tu && tc->port == aport && + (isrep == (tc->flags & CONN_REPLICATED))) { break; } } @@ -263,6 +278,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, tc->refCount = 0; /* bumped below */ tc->forceConnectFS = 1; tc->id = (struct rx_connection *)0; + if (isrep) + tc->flags |= CONN_REPLICATED; tc->next = sap->conns; sap->conns = tc; afs_ActivateServer(sap); @@ -314,14 +331,22 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, } /* set to a RX_CALL_TIMEOUT error to allow MTU retry to trigger */ rx_SetServerConnIdleDeadErr(tc->id, RX_CALL_DEAD); - rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead); + /* Setting idle dead time to non-zero activates RX_CALL_IDLE errors. */ + if (isrep) + rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead_rep); + else + rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead); /* * Only do this for the base connection, not per-user. * Will need to be revisited if/when CB gets security. */ if ((isec == 0) && (service != 52) && !(tu->states & UTokensBad) && - (tu->vid == UNDEFVID) && (tu->uid == 0)) + (tu->vid == UNDEFVID) && (isrep == 0) +#ifndef UKERNEL /* ukernel runs as just one uid anyway */ + && (tu->uid == 0) +#endif + ) rx_SetConnSecondsUntilNatPing(tc->id, 20); tc->forceConnectFS = 0; /* apparently we're appropriately connected now */ @@ -350,13 +375,14 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, * @param areq The request. * @param aforce Force connection? * @param locktype Type of lock to be used. + * @param replicated * * @return The established connection. */ struct afs_conn * afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, struct vrequest *areq, int aforce, afs_int32 locktype, - struct rx_connection **rxconn) + afs_int32 replicated, struct rx_connection **rxconn) { struct unixuser *tu; struct afs_conn *tc = 0; @@ -382,7 +408,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, for (sa = aserver->addr; sa; sa = sa->next_sa) { tc = afs_ConnBySA(sa, aport, acell, tu, aforce, 0 /*don't create one */ , - locktype, rxconn); + locktype, replicated, rxconn); if (tc) break; } @@ -391,7 +417,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, for (sa = aserver->addr; sa; sa = sa->next_sa) { tc = afs_ConnBySA(sa, aport, acell, tu, aforce, 1 /*create one */ , - locktype, rxconn); + locktype, replicated, rxconn); if (tc) break; } @@ -412,13 +438,15 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, * @param acell The cell where all of this happens. * @param areq The request. * @param locktype Type of lock to be used. + * @param replicated * * @return The established connection or NULL. */ struct afs_conn * afs_ConnByMHosts(struct server *ahosts[], unsigned short aport, afs_int32 acell, struct vrequest *areq, - afs_int32 locktype, struct rx_connection **rxconn) + afs_int32 locktype, afs_int32 replicated, + struct rx_connection **rxconn) { afs_int32 i; struct afs_conn *tconn; @@ -431,7 +459,8 @@ afs_ConnByMHosts(struct server *ahosts[], unsigned short aport, for (i = 0; i < AFS_MAXCELLHOSTS; i++) { if ((ts = ahosts[i]) == NULL) break; - tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype, rxconn); + tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype, + replicated, rxconn); if (tconn) { return tconn; } diff --git a/src/afs/afs_pioctl.c b/src/afs/afs_pioctl.c index c3ff965d4..440d617a9 100644 --- a/src/afs/afs_pioctl.c +++ b/src/afs/afs_pioctl.c @@ -5111,7 +5111,7 @@ DECL_PIOCTL(PCallBackAddr) /* get a connection, even if host is down; bumps conn ref count */ tu = afs_GetUser(areq->uid, ts->cell->cellNum, SHARED_LOCK); tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu, - 1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn); + 1 /*force */ , 1 /*create */ , SHARED_LOCK, 0, &rxconn); afs_PutUser(tu, SHARED_LOCK); if (!tc) continue; diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h index 59b84117b..6e8a87bfd 100644 --- a/src/afs/afs_prototypes.h +++ b/src/afs/afs_prototypes.h @@ -181,17 +181,19 @@ extern struct afs_conn *afs_Conn(struct VenusFid *afid, extern struct afs_conn *afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell, struct unixuser *tu, int force_if_down, afs_int32 create, - afs_int32 locktype, + afs_int32 locktype, afs_int32 replicated, struct rx_connection **rxconn); extern struct afs_conn *afs_ConnByMHosts(struct server *ahosts[], unsigned short aport, afs_int32 acell, struct vrequest *areq, afs_int32 locktype, + afs_int32 replicated, struct rx_connection **rxconn); extern struct afs_conn *afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell, struct vrequest *areq, int aforce, afs_int32 locktype, + afs_int32 replicated, struct rx_connection **rxconn); extern void afs_PutConn(struct afs_conn *ac, struct rx_connection *rxconn, diff --git a/src/afs/afs_server.c b/src/afs/afs_server.c index 3901fe55b..769f5e178 100644 --- a/src/afs/afs_server.c +++ b/src/afs/afs_server.c @@ -299,7 +299,8 @@ CheckVLServer(struct srvAddr *sa, struct vrequest *areq) return; /* can't do much */ tc = afs_ConnByHost(aserver, aserver->cell->vlport, - aserver->cell->cellNum, areq, 1, SHARED_LOCK, &rxconn); + aserver->cell->cellNum, areq, 1, SHARED_LOCK, 0, + &rxconn); if (!tc) return; rx_SetConnDeadTime(rxconn, 3); @@ -624,7 +625,8 @@ afs_CheckServers(int adown, struct cell *acellp) /* get a connection, even if host is down; bumps conn ref count */ tu = afs_GetUser(treq.uid, ts->cell->cellNum, SHARED_LOCK); tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu, - 1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn); + 1 /*force */ , 1 /*create */ , SHARED_LOCK, 0, + &rxconn); afs_PutUser(tu, SHARED_LOCK); if (!tc) continue; @@ -1719,8 +1721,7 @@ afs_GetCapabilities(struct server *ts) if ( !tu ) return; tc = afs_ConnBySA(ts->addr, ts->cell->fsport, ts->cell->cellNum, tu, 0, 1, - SHARED_LOCK, - &rxconn); + SHARED_LOCK, 0, &rxconn); afs_PutUser(tu, SHARED_LOCK); if ( !tc ) return; diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c index 56df59b87..ec45a1a17 100644 --- a/src/afs/afs_vcache.c +++ b/src/afs/afs_vcache.c @@ -410,7 +410,7 @@ afs_FlushVCBs(afs_int32 lockit) for (safety3 = 0; safety3 < AFS_MAXHOSTS * 2; safety3++) { tc = afs_ConnByHost(tsp, tsp->cell->fsport, tsp->cell->cellNum, &treq, 0, - SHARED_LOCK, &rxconn); + SHARED_LOCK, 0, &rxconn); if (tc) { XSTATS_START_TIME (AFS_STATS_FS_RPCIDX_GIVEUPCALLBACKS); diff --git a/src/afs/afs_volume.c b/src/afs/afs_volume.c index 94a806e12..2b278c389 100644 --- a/src/afs/afs_volume.c +++ b/src/afs/afs_volume.c @@ -765,7 +765,7 @@ afs_NewVolumeByName(char *aname, afs_int32 acell, int agood, do { tconn = afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum, - &treq, SHARED_LOCK, &rxconn); + &treq, SHARED_LOCK, 0, &rxconn); if (tconn) { if (tconn->srvr->server->flags & SNO_LHOSTS) { type = 0; @@ -1113,7 +1113,7 @@ LockAndInstallUVolumeEntry(struct volume *av, struct uvldbentry *ve, int acell, tconn = afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum, areq, SHARED_LOCK, - &rxconn); + 0, &rxconn); if (tconn) { RX_AFS_GUNLOCK(); code = -- 2.39.5