libafs: add replicated connection pool

keep pool of connections to use for replicated volumes,
so we can have a separate idle time setting

(cherry picked from commit cd1f72649650404581cfcdcf3beeeaf2bb960bd6)
Reviewed-on: http://gerrit.openafs.org/6546
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementix.org>
Change-Id: I056ba28d11313c9925df63869e0c55a1a4f132da
Reviewed-on: http://gerrit.openafs.org/6610
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementix.org>

diff --git a/src/afs/afs.h b/src/afs/afs.h
index 5581f9b2b1fa88f40c681faa76c2ef03c6323f7e..cf9345d481bcec50da81fa75104b0593a38579fc 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -96,12 +96,14 @@ extern int afs_shuttingdown;
 #define        AFS_RXDEADTIME  50
 #define AFS_HARDDEADTIME       120
 #define        AFS_IDLEDEADTIME        1200
+#define AFS_IDLEDEADTIME_REP    180 /* more than fs's cb dead time */
 #define AFS_BLKBITS    12
 #define AFS_BLKSIZE    (1 << AFS_BLKBITS)
 
 extern afs_int32 afs_rx_deadtime;
 extern afs_int32 afs_rx_harddead;
 extern afs_int32 afs_rx_idledead;
+extern afs_int32 afs_rx_idledead_rep;
 
 struct sysname_info {
     char *name;
@@ -360,6 +362,9 @@ struct unixuser {
     void *cellinfo;             /* pointer to cell info (PAG manager only) */
 };
 
+
+#define CONN_REPLICATED 0x1
+
 struct afs_conn {
     /* Per-connection block. */
     struct afs_conn *next;             /* Next dude same server. */
@@ -369,6 +374,7 @@ struct afs_conn {
     short refCount;            /* reference count for allocation */
     unsigned short port;       /* port associated with this connection */
     char forceConnectFS;       /* Should we try again with these tokens? */
+    int flags;
 };
 
 

diff --git a/src/afs/afs_analyze.c b/src/afs/afs_analyze.c
index 1d1b7c40b3774265c2c3f61b5a21b71572400561..6dba5c0de989c20c2fceb8c58d32e2f1ccc4d700 100644 (file)
--- a/src/afs/afs_analyze.c
+++ b/src/afs/afs_analyze.c
@@ -117,7 +117,7 @@ VLDB_Same(struct VenusFid *afid, struct vrequest *areq)
        VSleep(2);              /* Better safe than sorry. */
        tconn =
            afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum,
-                            &treq, SHARED_LOCK, &rxconn);
+                            &treq, SHARED_LOCK, 0, &rxconn);
        if (tconn) {
            if (tconn->srvr->server->flags & SNO_LHOSTS) {
                type = 0;

diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 8334db87ee3accd507b28e1150ed2d6413218bec..83dfd37a73643b19b178e225274216274400dd08 100644 (file)
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -75,6 +75,7 @@ char afs_cachebasedir[1024];
 afs_int32 afs_rx_deadtime = AFS_RXDEADTIME;
 afs_int32 afs_rx_harddead = AFS_HARDDEADTIME;
 afs_int32 afs_rx_idledead = AFS_IDLEDEADTIME;
+afs_int32 afs_rx_idledead_rep = AFS_IDLEDEADTIME_REP;
 
 static int afscall_set_rxpck_received = 0;
 

diff --git a/src/afs/afs_conn.c b/src/afs/afs_conn.c
index 3de6c18dd1087f59c95ab0e4dd0894f6858fbd03..434c7617d9b1967d56014b5b5491d5712c3c1b74 100644 (file)
--- a/src/afs/afs_conn.c
+++ b/src/afs/afs_conn.c
@@ -125,6 +125,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
     int notbusy;
     int i;
     struct srvAddr *sa1p;
+    afs_int32 replicated = -1; /* a single RO will increment to 0 */
 
     *rxconn = NULL;
 
@@ -161,6 +162,8 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
      */
     for (notbusy = not_busy; (!lowp && (notbusy <= end_not_busy)); notbusy++) {
        for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) {
+           if (tv->states & VRO)
+               replicated++;
            if (((areq->tokenError > 0)||(areq->idleError > 0))
                && (areq->skipserver[i] == 1))
                continue;
@@ -182,12 +185,20 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
            }
        }
     }
+    if ((replicated == -1) && (tv->states & VRO)) {
+       for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) {
+           if (tv->states & VRO)
+               replicated++;
+       }
+    } else
+       replicated = 0;
+
     afs_PutVolume(tv, READ_LOCK);
 
     if (lowp) {
        tu = afs_GetUser(areq->uid, afid->Cell, SHARED_LOCK);
        tconn = afs_ConnBySA(lowp, fsport, afid->Cell, tu, 0 /*!force */ ,
-                            1 /*create */ , locktype, rxconn);
+                            1 /*create */ , locktype, replicated, rxconn);
 
        afs_PutUser(tu, SHARED_LOCK);
     }
@@ -205,6 +216,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
  * @param tu Connect as this user.
  * @param force_if_down
  * @param create
+ * @param replicated
  * @param locktype Specifies type of lock to be used for this function.
  *
  * @return The new connection.
@@ -212,12 +224,14 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
 struct afs_conn *
 afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
             struct unixuser *tu, int force_if_down, afs_int32 create,
-            afs_int32 locktype, struct rx_connection **rxconn)
+            afs_int32 locktype, afs_int32 replicated,
+            struct rx_connection **rxconn)
 {
     struct afs_conn *tc = 0;
     struct rx_securityClass *csec;     /*Security class object */
     int isec;                  /*Security index */
     int service;
+    int isrep = (replicated > 0)?CONN_REPLICATED:0;
 
     *rxconn = NULL;
 
@@ -229,7 +243,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
     ObtainSharedLock(&afs_xconn, 15);
     /* Get conn by port and user. */
     for (tc = sap->conns; tc; tc = tc->next) {
-       if (tc->user == tu && tc->port == aport) {
+       if (tc->user == tu && tc->port == aport &&
+           (isrep == (tc->flags & CONN_REPLICATED))) {
            break;
        }
     }
@@ -263,6 +278,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
        tc->refCount = 0;       /* bumped below */
        tc->forceConnectFS = 1;
        tc->id = (struct rx_connection *)0;
+       if (isrep)
+           tc->flags |= CONN_REPLICATED;
        tc->next = sap->conns;
        sap->conns = tc;
        afs_ActivateServer(sap);
@@ -314,14 +331,22 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
        }
        /* set to a RX_CALL_TIMEOUT error to allow MTU retry to trigger */
        rx_SetServerConnIdleDeadErr(tc->id, RX_CALL_DEAD);
-       rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead);
+        /* Setting idle dead time to non-zero activates RX_CALL_IDLE errors. */
+       if (isrep)        
+           rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead_rep);
+       else
+           rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead);
 
        /*
         * Only do this for the base connection, not per-user.
         * Will need to be revisited if/when CB gets security.
         */
        if ((isec == 0) && (service != 52) && !(tu->states & UTokensBad) &&
-           (tu->vid == UNDEFVID) && (tu->uid == 0))
+           (tu->vid == UNDEFVID) && (isrep == 0)
+#ifndef UKERNEL /* ukernel runs as just one uid anyway */
+           && (tu->uid == 0)
+#endif
+           )
            rx_SetConnSecondsUntilNatPing(tc->id, 20);
 
        tc->forceConnectFS = 0; /* apparently we're appropriately connected now */
@@ -350,13 +375,14 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
  * @param areq The request.
  * @param aforce Force connection?
  * @param locktype Type of lock to be used.
+ * @param replicated
  *
  * @return The established connection.
  */
 struct afs_conn *
 afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
               struct vrequest *areq, int aforce, afs_int32 locktype,
-              struct rx_connection **rxconn)
+              afs_int32 replicated, struct rx_connection **rxconn)
 {
     struct unixuser *tu;
     struct afs_conn *tc = 0;
@@ -382,7 +408,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
     for (sa = aserver->addr; sa; sa = sa->next_sa) {
        tc = afs_ConnBySA(sa, aport, acell, tu, aforce,
                          0 /*don't create one */ ,
-                         locktype, rxconn);
+                         locktype, replicated, rxconn);
        if (tc)
            break;
     }
@@ -391,7 +417,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
        for (sa = aserver->addr; sa; sa = sa->next_sa) {
            tc = afs_ConnBySA(sa, aport, acell, tu, aforce,
                              1 /*create one */ ,
-                             locktype, rxconn);
+                             locktype, replicated, rxconn);
            if (tc)
                break;
        }
@@ -412,13 +438,15 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
  * @param acell The cell where all of this happens.
  * @param areq The request.
  * @param locktype Type of lock to be used.
+ * @param replicated
  *
  * @return The established connection or NULL.
  */
 struct afs_conn *
 afs_ConnByMHosts(struct server *ahosts[], unsigned short aport,
                 afs_int32 acell, struct vrequest *areq,
-                afs_int32 locktype, struct rx_connection **rxconn)
+                afs_int32 locktype, afs_int32 replicated,
+                struct rx_connection **rxconn)
 {
     afs_int32 i;
     struct afs_conn *tconn;
@@ -431,7 +459,8 @@ afs_ConnByMHosts(struct server *ahosts[], unsigned short aport,
     for (i = 0; i < AFS_MAXCELLHOSTS; i++) {
        if ((ts = ahosts[i]) == NULL)
            break;
-       tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype, rxconn);
+       tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype,
+                              replicated, rxconn);
        if (tconn) {
            return tconn;
        }

diff --git a/src/afs/afs_pioctl.c b/src/afs/afs_pioctl.c
index c3ff965d439b7125902b5da3bda64f3618610366..440d617a9cbfad7c6d414e60a11fce0d254478ff 100644 (file)
--- a/src/afs/afs_pioctl.c
+++ b/src/afs/afs_pioctl.c
@@ -5111,7 +5111,7 @@ DECL_PIOCTL(PCallBackAddr)
        /* get a connection, even if host is down; bumps conn ref count */
        tu = afs_GetUser(areq->uid, ts->cell->cellNum, SHARED_LOCK);
        tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu,
-                         1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn);
+                         1 /*force */ , 1 /*create */ , SHARED_LOCK, 0, &rxconn);
        afs_PutUser(tu, SHARED_LOCK);
        if (!tc)
            continue;

diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h
index 59b84117bca84471a27685b83a50f4c33d6c3477..6e8a87bfda1ef24e96931484c3b17d2900118423 100644 (file)
--- a/src/afs/afs_prototypes.h
+++ b/src/afs/afs_prototypes.h
@@ -181,17 +181,19 @@ extern struct afs_conn *afs_Conn(struct VenusFid *afid,
 extern struct afs_conn *afs_ConnBySA(struct srvAddr *sap, unsigned short aport,
                                 afs_int32 acell, struct unixuser *tu,
                                 int force_if_down, afs_int32 create,
-                                afs_int32 locktype,
+                                afs_int32 locktype, afs_int32 replicated,
                                 struct rx_connection **rxconn);
 extern struct afs_conn *afs_ConnByMHosts(struct server *ahosts[],
                                     unsigned short aport, afs_int32 acell,
                                     struct vrequest *areq,
                                     afs_int32 locktype,
+                                    afs_int32 replicated,
                                     struct rx_connection **rxconn);
 extern struct afs_conn *afs_ConnByHost(struct server *aserver,
                                   unsigned short aport, afs_int32 acell,
                                   struct vrequest *areq, int aforce,
                                   afs_int32 locktype,
+                                  afs_int32 replicated,
                                   struct rx_connection **rxconn);
 extern void afs_PutConn(struct afs_conn *ac,
                         struct rx_connection *rxconn,

diff --git a/src/afs/afs_server.c b/src/afs/afs_server.c
index 3901fe55b60516c8bad9ea3c6b6bcac6a50dfc49..769f5e17861c1d58e7169a3f5dccb64d076db2f1 100644 (file)
--- a/src/afs/afs_server.c
+++ b/src/afs/afs_server.c
@@ -299,7 +299,8 @@ CheckVLServer(struct srvAddr *sa, struct vrequest *areq)
        return;                 /* can't do much */
 
     tc = afs_ConnByHost(aserver, aserver->cell->vlport,
-                       aserver->cell->cellNum, areq, 1, SHARED_LOCK, &rxconn);
+                       aserver->cell->cellNum, areq, 1, SHARED_LOCK, 0,
+                       &rxconn);
     if (!tc)
        return;
     rx_SetConnDeadTime(rxconn, 3);
@@ -624,7 +625,8 @@ afs_CheckServers(int adown, struct cell *acellp)
        /* get a connection, even if host is down; bumps conn ref count */
        tu = afs_GetUser(treq.uid, ts->cell->cellNum, SHARED_LOCK);
        tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu,
-                         1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn);
+                         1 /*force */ , 1 /*create */ , SHARED_LOCK, 0,
+                         &rxconn);
        afs_PutUser(tu, SHARED_LOCK);
        if (!tc)
            continue;
@@ -1719,8 +1721,7 @@ afs_GetCapabilities(struct server *ts)
     if ( !tu )
        return;
     tc = afs_ConnBySA(ts->addr, ts->cell->fsport, ts->cell->cellNum, tu, 0, 1,
-                     SHARED_LOCK,
-                     &rxconn);
+                     SHARED_LOCK, 0, &rxconn);
     afs_PutUser(tu, SHARED_LOCK);
     if ( !tc )
        return;

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index 56df59b87c218008e5b9b9aaf0eacdbdc72a730c..ec45a1a174e162f15f88c3d00ef9fa73b585827f 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -410,7 +410,7 @@ afs_FlushVCBs(afs_int32 lockit)
                    for (safety3 = 0; safety3 < AFS_MAXHOSTS * 2; safety3++) {
                        tc = afs_ConnByHost(tsp, tsp->cell->fsport,
                                            tsp->cell->cellNum, &treq, 0,
-                                           SHARED_LOCK, &rxconn);
+                                           SHARED_LOCK, 0, &rxconn);
                        if (tc) {
                            XSTATS_START_TIME
                                (AFS_STATS_FS_RPCIDX_GIVEUPCALLBACKS);

diff --git a/src/afs/afs_volume.c b/src/afs/afs_volume.c
index 94a806e12774579d0fb98892657e882820ca2a5e..2b278c389c77280650ee0e5f7ff0c0fa6987a26a 100644 (file)
--- a/src/afs/afs_volume.c
+++ b/src/afs/afs_volume.c
@@ -765,7 +765,7 @@ afs_NewVolumeByName(char *aname, afs_int32 acell, int agood,
     do {
        tconn =
            afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum,
-                            &treq, SHARED_LOCK, &rxconn);
+                            &treq, SHARED_LOCK, 0, &rxconn);
        if (tconn) {
            if (tconn->srvr->server->flags & SNO_LHOSTS) {
                type = 0;
@@ -1113,7 +1113,7 @@ LockAndInstallUVolumeEntry(struct volume *av, struct uvldbentry *ve, int acell,
                    tconn =
                        afs_ConnByMHosts(tcell->cellHosts, tcell->vlport,
                                         tcell->cellNum, areq, SHARED_LOCK,
-                                        &rxconn);
+                                        0, &rxconn);
                    if (tconn) {
                        RX_AFS_GUNLOCK();
                        code =