rx: Compute smoothed RTT per call, not per peer.

RX uses the TCP RTT smoothing algorithm as described in RFC2988.
However, the TCP algorithm is designed to accept samples from a
single connection, accepting a new sample once per RTT.
RFC2988 suggests that "when multiple samples are taken
per RTT the [ alogrithm ] may keep an inadequate RTT history."

In RX's implementation, we use a single instance of this alogrithm
per peer, and input all of the samples from all of the active calls
and connections into this same instance. This leads to us taking
a significantly (potentially many magnitudes) larger number of samples
per RTT, and rapidly losing the RTT history. With RX's implementation,
short lived network events may easily bias the RTT, and cause large
numbers of packets to time out.

This change fixes this by moving the RTT calculation onto a per call
basis. We still update the peer with our caclulated value, so that new
calls may be created with an RTT corresponding to the current value for
the connection, rather than having to start high and converge downwards.

(cherry picked from commit 39484c6e57cf993a713b4a989d1c0c227e6f496c)
Reviewed-on: http://gerrit.openafs.org/4861
Reviewed-by: Jeffrey Altman <jaltman@openafs.org>
Reviewed-by: Derrick Brashear <shadow@dementia.org>
Tested-by: Derrick Brashear <shadow@dementia.org>
Change-Id: I5b33323ceed7231bd70e43284d83dfe1db144188
Reviewed-on: http://gerrit.openafs.org/4916
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/rx/rx.c b/src/rx/rx.c
index 3a732f366c477d57e0ca18b3ac6acff72731b1d6..3e9c2bf9645d5ca9a72feb2b18361f8e1c5dac0e 100644 (file)
--- a/src/rx/rx.c
+++ b/src/rx/rx.c
@@ -127,7 +127,8 @@ int (*swapNameProgram) (PROCESS, const char *, char *) = 0;
 /* Local static routines */
 static void rxi_DestroyConnectionNoLock(struct rx_connection *conn);
 static void rxi_ComputeRoundTripTime(struct rx_packet *, struct rx_ackPacket *,
-                                    struct rx_peer *, struct clock *);
+                                    struct rx_call *, struct rx_peer *,
+                                    struct clock *);
 
 #ifdef RX_ENABLE_LOCKS
 static void rxi_SetAcksInTransmitQueue(struct rx_call *call);
@@ -639,6 +640,17 @@ rx_Init(u_int port)
     return rx_InitHost(htonl(INADDR_ANY), port);
 }
 
+/**
+ * Set an initial round trip timeout for a peer connection
+ *
+ * @param[in] secs The timeout to set in seconds
+ */
+
+void
+rx_rto_setPeerTimeoutSecs(struct rx_peer *peer, int secs) {
+    peer->rtt = secs * 8000;
+}
+
 /**
  * Sets the error generated when a busy call channel is detected.
  *
@@ -4279,7 +4291,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
 
        if (!(tp->flags & RX_PKTFLAG_ACKED)) {
            newAckCount++;
-           rxi_ComputeRoundTripTime(tp, ap, call->conn->peer, &now);
+           rxi_ComputeRoundTripTime(tp, ap, call, peer, &now);
        }
 
 #ifdef ADAPT_WINDOW
@@ -4351,8 +4363,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
            if (!(tp->flags & RX_PKTFLAG_ACKED)) {
                newAckCount++;
                tp->flags |= RX_PKTFLAG_ACKED;
-
-               rxi_ComputeRoundTripTime(tp, ap, call->conn->peer, &now);
+               rxi_ComputeRoundTripTime(tp, ap, call, peer, &now);
 #ifdef ADAPT_WINDOW
                rxi_ComputeRate(call->conn->peer, call, tp, np, ap->reason);
 #endif
@@ -4374,7 +4385,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
 
        if (!(tp->flags & RX_PKTFLAG_ACKED) && !clock_IsZero(&tp->retryTime)) {
             tp->retryTime = tp->timeSent;
-           clock_Add(&tp->retryTime, &peer->timeout);
+           clock_Add(&tp->retryTime, &call->rto);
            /* shift by eight because one quarter-sec ~ 256 milliseconds */
            clock_Addmsec(&(tp->retryTime), ((afs_uint32) tp->backoff) << 8);
        }
@@ -4391,7 +4402,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
 
     while (!queue_IsEnd(&call->tq, tp) && !clock_IsZero(&tp->retryTime)) {
        tp->retryTime = tp->timeSent;
-       clock_Add(&tp->retryTime, &peer->timeout);
+       clock_Add(&tp->retryTime, &call->rto);
        clock_Addmsec(&tp->retryTime, ((afs_uint32) tp->backoff) << 8);
        tp = queue_Next(tp, rx_packet);
     }
@@ -5227,6 +5238,11 @@ rxi_ResetCall(struct rx_call *call, int newcall)
     call->ssthresh = rx_maxSendWindow;
     call->nDgramPackets = peer->nDgramPackets;
     call->congestSeq = peer->congestSeq;
+    call->rtt = peer->rtt;
+    call->rtt_dev = peer->rtt_dev;
+    clock_Zero(&call->rto);
+    clock_Addmsec(&call->rto,
+                 MAX(((call->rtt >> 3) + call->rtt_dev), rx_minPeerTimeout) + 200);
     MUTEX_EXIT(&peer->peer_lock);
 
     flags = call->flags;
@@ -5627,7 +5643,7 @@ rxi_SendList(struct rx_call *call, struct rx_packet **list, int len,
     peer->nSent += len;
     if (resending)
        peer->reSends += len;
-    retryTime = peer->timeout;
+    retryTime = call->rto;
     MUTEX_EXIT(&peer->peer_lock);
 
     if (rx_stats_active) {
@@ -6177,8 +6193,8 @@ rxi_CheckCall(struct rx_call *call)
     }
 #endif
     /* RTT + 8*MDEV, rounded up to the next second. */
-    fudgeFactor = (((afs_uint32) conn->peer->rtt >> 3) +
-                   ((afs_uint32) conn->peer->rtt_dev << 1) + 1023) >> 10;
+    fudgeFactor = (((afs_uint32) call->rtt >> 3) +
+                   ((afs_uint32) call->rtt_dev << 1) + 1023) >> 10;
 
     deadTime = conn->secondsUntilDead + fudgeFactor;
     now = clock_Sec();
@@ -6686,6 +6702,7 @@ rxi_ChallengeOn(struct rx_connection *conn)
 static void
 rxi_ComputeRoundTripTime(struct rx_packet *p,
                         struct rx_ackPacket *ack,
+                        struct rx_call *call,
                         struct rx_peer *peer,
                         struct clock *now)
 {
@@ -6763,11 +6780,11 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
     /* better rtt calculation courtesy of UMich crew (dave,larry,peter,?) */
 
     /* Apply VanJacobson round-trip estimations */
-    if (peer->rtt) {
+    if (call->rtt) {
        int delta;
 
        /*
-        * srtt (peer->rtt) is in units of one-eighth-milliseconds.
+        * srtt (call->rtt) is in units of one-eighth-milliseconds.
         * srtt is stored as fixed point with 3 bits after the binary
         * point (i.e., scaled by 8). The following magic is
         * equivalent to the smoothing algorithm in rfc793 with an
@@ -6778,8 +6795,8 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
          * srtt' = srtt + (rtt - srtt)/8
         */
 
-       delta = _8THMSEC(&thisRtt) - peer->rtt;
-       peer->rtt += (delta >> 3);
+       delta = _8THMSEC(&thisRtt) - call->rtt;
+       call->rtt += (delta >> 3);
 
        /*
         * We accumulate a smoothed rtt variance (actually, a smoothed
@@ -6802,8 +6819,8 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
        if (delta < 0)
            delta = -delta;
 
-       delta -= (peer->rtt_dev << 1);
-       peer->rtt_dev += (delta >> 3);
+       delta -= (call->rtt_dev << 1);
+       call->rtt_dev += (delta >> 3);
     } else {
        /* I don't have a stored RTT so I start with this value.  Since I'm
         * probably just starting a call, and will be pushing more data down
@@ -6811,8 +6828,8 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
         * little, and I set deviance to half the rtt.  In practice,
         * deviance tends to approach something a little less than
         * half the smoothed rtt. */
-       peer->rtt = _8THMSEC(&thisRtt) + 8;
-       peer->rtt_dev = peer->rtt >> 2; /* rtt/2: they're scaled differently */
+       call->rtt = _8THMSEC(&thisRtt) + 8;
+       call->rtt_dev = call->rtt >> 2; /* rtt/2: they're scaled differently */
     }
     /* the smoothed RTT time is RTT + 4*MDEV
      *
@@ -6822,13 +6839,17 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
      * add on a fixed 200ms to account for that timer expiring.
      */
 
-    rtt_timeout = MAX(((peer->rtt >> 3) + peer->rtt_dev),
+    rtt_timeout = MAX(((call->rtt >> 3) + call->rtt_dev),
                      rx_minPeerTimeout) + 200;
-    clock_Zero(&(peer->timeout));
-    clock_Addmsec(&(peer->timeout), rtt_timeout);
+    clock_Zero(&call->rto);
+    clock_Addmsec(&call->rto, rtt_timeout);
+
+    /* Update the peer, so any new calls start with our values */
+    peer->rtt_dev = call->rtt_dev;
+    peer->rtt = call->rtt;
 
     dpf(("rxi_ComputeRoundTripTime(call=%d packet=%"AFS_PTR_FMT" rtt=%d ms, srtt=%d ms, rtt_dev=%d ms, timeout=%d.%06d sec)\n",
-          p->header.callNumber, p, MSEC(&thisRtt), peer->rtt >> 3, peer->rtt_dev >> 2, (peer->timeout.sec), (peer->timeout.usec)));
+          p->header.callNumber, p, MSEC(&thisRtt), call->rtt >> 3, call->rtt_dev >> 2, (call->rto.sec), (call->rto.usec)));
 }
 
 
@@ -7096,7 +7117,7 @@ rxi_ComputeRate(struct rx_peer *peer, struct rx_call *call,
     case RX_ACK_REQUESTED:
        xferSize =
            p->length + RX_HEADER_SIZE + call->conn->securityMaxTrailerSize;
-       xferMs = peer->rtt;
+       xferMs = call->rtt;
        break;
 
     case RX_ACK_PING_RESPONSE:
@@ -7407,9 +7428,8 @@ rx_PrintPeerStats(FILE * file, struct rx_peer *peer)
            (int)peer->burstWait.sec, (int)peer->burstWait.usec);
 
     fprintf(file,
-           "   Rtt %d, " "retry time %u.%06d, " "total sent %d, "
-           "resent %d\n", peer->rtt, (int)peer->timeout.sec,
-           (int)peer->timeout.usec, peer->nSent, peer->reSends);
+           "   Rtt %d, " "total sent %d, " "resent %d\n",
+           peer->rtt, peer->nSent, peer->reSends);
 
     fprintf(file,
            "   Packet size %d, " "max in packet skew %d, "
@@ -7787,8 +7807,8 @@ rx_GetServerPeers(osi_socket socket, afs_uint32 remoteAddr,
        peer->burstWait.usec = ntohl(peer->burstWait.usec);
        peer->rtt = ntohl(peer->rtt);
        peer->rtt_dev = ntohl(peer->rtt_dev);
-       peer->timeout.sec = ntohl(peer->timeout.sec);
-       peer->timeout.usec = ntohl(peer->timeout.usec);
+       peer->timeout.sec = 0;
+       peer->timeout.usec = 0;
        peer->nSent = ntohl(peer->nSent);
        peer->reSends = ntohl(peer->reSends);
        peer->inPacketSkew = ntohl(peer->inPacketSkew);
@@ -7846,8 +7866,8 @@ rx_GetLocalPeers(afs_uint32 peerHost, afs_uint16 peerPort,
                peerStats->burstWait.usec = tp->burstWait.usec;
                peerStats->rtt = tp->rtt;
                peerStats->rtt_dev = tp->rtt_dev;
-               peerStats->timeout.sec = tp->timeout.sec;
-               peerStats->timeout.usec = tp->timeout.usec;
+               peerStats->timeout.sec = 0;
+               peerStats->timeout.usec = 0;
                peerStats->nSent = tp->nSent;
                peerStats->reSends = tp->reSends;
                peerStats->inPacketSkew = tp->inPacketSkew;

diff --git a/src/rx/rx.h b/src/rx/rx.h
index e7dc9a932b6b3a3a8a852d27fa19e919ed95a903..17bc9aad992a610ab0d979ea8ea0e7579620397f 100644 (file)
--- a/src/rx/rx.h
+++ b/src/rx/rx.h
@@ -400,7 +400,6 @@ struct rx_peer {
     struct rx_queue congestionQueue;   /* Calls that are waiting for non-zero burst value */
     int rtt;                   /* Smoothed round trip time, measured in milliseconds/8 */
     int rtt_dev;               /* Smoothed rtt mean difference, in milliseconds/4 */
-    struct clock timeout;      /* Current retransmission delay */
     int nSent;                 /* Total number of distinct data packets sent, not including retransmissions */
     int reSends;               /* Total number of retransmissions for this peer, since this structure was created */
 
@@ -534,6 +533,9 @@ struct rx_call {
     u_short nSoftAcks;         /* The number of delayed soft acks */
     u_short nHardAcks;         /* The number of delayed hard acks */
     u_short congestSeq;                /* Peer's congestion sequence counter */
+    int rtt;
+    int rtt_dev;
+    struct clock rto;          /* The round trip timeout calculated for this call */
     struct rxevent *resendEvent;       /* If this is non-Null, there is a retransmission event pending */
     struct rxevent *timeoutEvent;      /* If this is non-Null, then there is an overall timeout for this call */
     struct rxevent *keepAliveEvent;    /* Scheduled periodically in active calls to keep call alive */

diff --git a/src/rx/rx_kcommon.c b/src/rx/rx_kcommon.c
index 94f8bb6d54490dd10a4a430d3a4558947d54db50..88157a8bc3cba630839dcd40215e0baf70228275 100644 (file)
--- a/src/rx/rx_kcommon.c
+++ b/src/rx/rx_kcommon.c
@@ -372,12 +372,10 @@ rxi_InitPeerParams(struct rx_peer *pp)
 
     i = rxi_Findcbi(pp->host);
     if (i == -1) {
-       pp->timeout.sec = 3;
-       /* pp->timeout.usec = 0; */
+       rx_rto_setPeerTimeoutSecs(pp, 3);
        pp->ifMTU = MIN(RX_REMOTE_PACKET_SIZE, rx_MyMaxSendSize);
     } else {
-       pp->timeout.sec = 2;
-       /* pp->timeout.usec = 0; */
+       rx_rto_setPeerTimeoutSecs(pp, 2);
        pp->ifMTU = MIN(RX_MAX_PACKET_SIZE, rx_MyMaxSendSize);
        mtu = ntohl(afs_cb_interface.mtu[i]);
        /* Diminish the packet size to one based on the MTU given by
@@ -398,13 +396,12 @@ rxi_InitPeerParams(struct rx_peer *pp)
 
     ifn = rxi_FindIfnet(pp->host, NULL);
     if (ifn) {
-       pp->timeout.sec = 2;
-       /* pp->timeout.usec = 0; */
+       rx_rto_setPeerTimeoutSecs(pp, 2);
        pp->ifMTU = MIN(RX_MAX_PACKET_SIZE, rx_MyMaxSendSize);
 #   ifdef IFF_POINTOPOINT
        if (rx_ifnet_flags(ifn) & IFF_POINTOPOINT) {
            /* wish we knew the bit rate and the chunk size, sigh. */
-           pp->timeout.sec = 4;
+           rx_rto_setPeerTimeoutSecs(pp, 4);
            pp->ifMTU = RX_PP_PACKET_SIZE;
        }
 #   endif /* IFF_POINTOPOINT */
@@ -416,8 +413,7 @@ rxi_InitPeerParams(struct rx_peer *pp)
                pp->ifMTU = rxmtu;
        }
     } else {                   /* couldn't find the interface, so assume the worst */
-       pp->timeout.sec = 3;
-       /* pp->timeout.usec = 0; */
+       rx_rto_setPeerTimeoutSecs(pp, 3);
        pp->ifMTU = MIN(RX_REMOTE_PACKET_SIZE, rx_MyMaxSendSize);
     }
 #  endif /* else AFS_USERSPACE_IP_ADDR */
@@ -427,12 +423,10 @@ rxi_InitPeerParams(struct rx_peer *pp)
     mtu = rxi_FindIfMTU(pp->host);
 
     if (mtu <= 0) {
-       pp->timeout.sec = 3;
-       /* pp->timeout.usec = 0; */
+       rx_rto_setPeerTimeoutSecs(pp, 3);
        pp->ifMTU = MIN(RX_REMOTE_PACKET_SIZE, rx_MyMaxSendSize);
     } else {
-       pp->timeout.sec = 2;
-       /* pp->timeout.usec = 0; */
+       rx_rto_setPeerTimeoutSecs(pp, 2);
        pp->ifMTU = MIN(RX_MAX_PACKET_SIZE, rx_MyMaxSendSize);
 
        /* Diminish the packet size to one based on the MTU given by
@@ -446,7 +440,7 @@ rxi_InitPeerParams(struct rx_peer *pp)
 # endif /* AFS_SUN5_ENV */
 #else /* ADAPT_MTU */
     pp->rateFlag = 2;          /* start timing after two full packets */
-    pp->timeout.sec = 2;
+    rx_rto_setPeerTimeoutSecs(pp, 2);
     pp->ifMTU = OLD_MAX_PACKET_SIZE;
 #endif /* else ADAPT_MTU */
     pp->ifMTU = rxi_AdjustIfMTU(pp->ifMTU);

diff --git a/src/rx/rx_packet.c b/src/rx/rx_packet.c
index f07114e68ccba8c2ed9220d5d8df2dbbf6244ca8..b2fabcc163711005e24fc0510ae3bee23efdf2b3 100644 (file)
--- a/src/rx/rx_packet.c
+++ b/src/rx/rx_packet.c
@@ -2002,8 +2002,6 @@ rxi_ReceiveDebugPacket(struct rx_packet *ap, osi_socket asocket,
                        tpeer.burstWait.usec = htonl(tp->burstWait.usec);
                        tpeer.rtt = htonl(tp->rtt);
                        tpeer.rtt_dev = htonl(tp->rtt_dev);
-                       tpeer.timeout.sec = htonl(tp->timeout.sec);
-                       tpeer.timeout.usec = htonl(tp->timeout.usec);
                        tpeer.nSent = htonl(tp->nSent);
                        tpeer.reSends = htonl(tp->reSends);
                        tpeer.inPacketSkew = htonl(tp->inPacketSkew);

diff --git a/src/rx/rx_prototypes.h b/src/rx/rx_prototypes.h
index 32428959fe393bc05b21a8b765518e2db90b792f..8925b4947219d8890dddc46c1bd85e930f0abb9e 100644 (file)
--- a/src/rx/rx_prototypes.h
+++ b/src/rx/rx_prototypes.h
@@ -20,6 +20,7 @@ extern int (*swapNameProgram) (PROCESS, const char *, char *);
 extern int (*rx_justReceived) (struct rx_packet *, struct sockaddr_in *);
 extern int (*rx_almostSent) (struct rx_packet *, struct sockaddr_in *);
 
+extern void rx_rto_setPeerTimeoutSecs(struct rx_peer *, int secs);
 
 extern void rx_SetEpoch(afs_uint32 epoch);
 extern int rx_Init(u_int port);

diff --git a/src/rx/rx_user.c b/src/rx/rx_user.c
index 2e44176d84f728e317ad8caecc742317a672ef4e..bd3965bafe50f129e1dd9f8bcb8f1e975290339f 100644 (file)
--- a/src/rx/rx_user.c
+++ b/src/rx/rx_user.c
@@ -685,8 +685,6 @@ rxi_InitPeerParams(struct rx_peer *pp)
     struct sockaddr_in addr;
 #endif
 
-
-
     LOCK_IF_INIT;
     if (!Inited) {
        UNLOCK_IF_INIT;
@@ -705,19 +703,20 @@ rxi_InitPeerParams(struct rx_peer *pp)
     ppaddr = ntohl(pp->host);
 
     pp->ifMTU = 0;
-    pp->timeout.sec = 2;
+    rx_rto_setPeerTimeoutSecs(pp, 2);
     pp->rateFlag = 2;          /* start timing after two full packets */
     /* I don't initialize these, because I presume they are bzero'd...
      * pp->burstSize pp->burst pp->burstWait.sec pp->burstWait.usec
-     * pp->timeout.usec */
+     */
 
     LOCK_IF;
     for (ix = 0; ix < rxi_numNetAddrs; ++ix) {
        if ((rxi_NetAddrs[ix] & myNetMasks[ix]) == (ppaddr & myNetMasks[ix])) {
 #ifdef IFF_POINTOPOINT
            if (myNetFlags[ix] & IFF_POINTOPOINT)
-               pp->timeout.sec = 4;
+               rx_rto_setPeerTimeoutSecs(pp, 4);
 #endif /* IFF_POINTOPOINT */
+
            rxmtu = myNetMTUs[ix] - RX_IPUDP_SIZE;
            if (rxmtu < RX_MIN_PACKET_SIZE)
                rxmtu = RX_MIN_PACKET_SIZE;
@@ -727,12 +726,12 @@ rxi_InitPeerParams(struct rx_peer *pp)
     }
     UNLOCK_IF;
     if (!pp->ifMTU) {          /* not local */
-       pp->timeout.sec = 3;
+       rx_rto_setPeerTimeoutSecs(pp, 3);
        pp->ifMTU = MIN(rx_MyMaxSendSize, RX_REMOTE_PACKET_SIZE);
     }
 #else /* ADAPT_MTU */
     pp->rateFlag = 2;          /* start timing after two full packets */
-    pp->timeout.sec = 2;
+    rx_rto_setPeerTimeoutSecs(pp, 2);
     pp->ifMTU = MIN(rx_MyMaxSendSize, OLD_MAX_PACKET_SIZE);
 #endif /* ADAPT_MTU */
 #if defined(ADAPT_PMTU) && defined(IP_MTU)

diff --git a/src/rx/test/testclient.c b/src/rx/test/testclient.c
index 16ad50bc663ed0361bbcd74b59f477969e7cc651..b4673ca4ebc0e46c18cb0a44f8835d365bf8993e 100644 (file)
--- a/src/rx/test/testclient.c
+++ b/src/rx/test/testclient.c
@@ -240,8 +240,7 @@ main(argc, argv)
     if (!clock_IsZero(&burstTime))
        conn->peer->burstWait = burstTime;
     if (!clock_IsZero(&retryTime))
-       conn->peer->timeout = retryTime;
-
+       conn->peer->rtt = _8THMSEC(&retryTime);
     if (sendFile)
        SendFile(sendFile, conn);
     else {