rx: Fix starting of transmit timers

The code used to start the transmit timer once for every set of packets
that it sends. However, these packets might be sent individually or in
clumps, with blocking for sendmsg, and on peer->lock, between each set
of packet sends. This has the effect of, even on a very stable network,
producing a high degree of variation in RTTs and timeouts. This is a
particular issue where the connection size is larger, as the number of
packets being sent individually under the one timer grows too.

Fix this by moving timer initialisation to SendList. This already takes
the peer lock, so obtain the timeout value here too. This means that
each jumbo gram, or individual packet (where jumbograms are disabled)
is sent with its own start time, and stabilises RTTs.

Change-Id: Ifc8242211cedad9b52a66a44bbdee400130ab40b
Reviewed-on: http://gerrit.openafs.org/3137
Reviewed-by: Jeffrey Altman <jaltman@openafs.org>
Tested-by: Derrick Brashear <shadow@dementia.org>
Reviewed-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/rx/rx.c b/src/rx/rx.c
index 28a7df1c62bdad23ecc2a5a2e0ebd664c7a1e1ec..a7d1c00392a5182b84c8f4a6cd3861990c1b6af3 100644 (file)
--- a/src/rx/rx.c
+++ b/src/rx/rx.c
@@ -5392,12 +5392,12 @@ struct xmitlist {
 /* Send all of the packets in the list in single datagram */
 static void
 rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
-            int istack, int moreFlag, struct clock *now,
-            struct clock *retryTime)
+            int istack, int moreFlag)
 {
     int i;
     int requestAck = 0;
     int lastPacket = 0;
+    struct clock now, retryTime;
     struct rx_connection *conn = call->conn;
     struct rx_peer *peer = conn->peer;
 
@@ -5405,6 +5405,7 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
     peer->nSent += xmit->len;
     if (xmit->resending)
        peer->reSends += xmit->len;
+    retryTime = peer->timeout;
     MUTEX_EXIT(&peer->peer_lock);
 
     if (rx_stats_active) {
@@ -5414,6 +5415,9 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
             rx_atomic_add(&rx_stats.dataPacketsSent, xmit->len);
     }
 
+    clock_GetTime(&now);
+    clock_Add(&retryTime, &now);
+
     if (xmit->list[xmit->len - 1]->header.flags & RX_LAST_PACKET) {
        lastPacket = 1;
     }
@@ -5423,7 +5427,7 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
     for (i = 0; i < xmit->len; i++) {
        struct rx_packet *packet = xmit->list[i];
 
-       packet->retryTime = *retryTime;
+       packet->retryTime = retryTime;
        if (packet->header.serial) {
            /* Exponentially backoff retry times */
            if (packet->backoff < MAXBACKOFF) {
@@ -5442,7 +5446,7 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
        }
 
        /* Record the time sent */
-       packet->timeSent = *now;
+       packet->timeSent = now;
 
        /* Ask for an ack on retransmitted packets,  on every other packet
         * if the peer doesn't support slow start. Ask for an ack on every
@@ -5451,7 +5455,7 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
            requestAck = 1;
        } else {
            /* improved RTO calculation- not Karn */
-           packet->firstSent = *now;
+           packet->firstSent = now;
            if (!lastPacket && (call->cwind <= (u_short) (conn->ackRate + 1)
                                || (!(call->flags & RX_CALL_SLOW_START_OK)
                                    && (packet->header.seq & 1)))) {
@@ -5464,10 +5468,6 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
        if (i < xmit->len - 1 || moreFlag) {
            packet->header.flags |= RX_MORE_PACKETS;
        }
-
-       /* Install the new retransmit time for the packet, and
-        * record the time sent */
-       packet->timeSent = *now;
     }
 
     if (requestAck) {
@@ -5512,7 +5512,7 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
 
 static void
 rxi_SendXmitList(struct rx_call *call, struct rx_packet **list, int len,
-                int istack, struct clock *now, struct clock *retryTime)
+                int istack)
 {
     int i;
     struct xmitlist working;
@@ -5536,7 +5536,7 @@ rxi_SendXmitList(struct rx_call *call, struct rx_packet **list, int len,
             * set into the 'last' one, and resets the working set */
 
            if (last.len > 0) {
-               rxi_SendList(call, &last, istack, 1, now, retryTime);
+               rxi_SendList(call, &last, istack, 1);
                /* If the call enters an error state stop sending, or if
                 * we entered congestion recovery mode, stop sending */
                if (call->error || (call->flags & RX_CALL_FAST_RECOVER_WAIT))
@@ -5562,7 +5562,7 @@ rxi_SendXmitList(struct rx_call *call, struct rx_packet **list, int len,
                || list[i]->header.serial
                || list[i]->length != RX_JUMBOBUFFERSIZE) {
                if (last.len > 0) {
-                   rxi_SendList(call, &last, istack, 1, now, retryTime);
+                   rxi_SendList(call, &last, istack, 1);
                    /* If the call enters an error state stop sending, or if
                     * we entered congestion recovery mode, stop sending */
                    if (call->error
@@ -5596,17 +5596,17 @@ rxi_SendXmitList(struct rx_call *call, struct rx_packet **list, int len,
            morePackets = 1;
        }
        if (last.len > 0) {
-           rxi_SendList(call, &last, istack, morePackets, now, retryTime);
+           rxi_SendList(call, &last, istack, morePackets);
            /* If the call enters an error state stop sending, or if
             * we entered congestion recovery mode, stop sending */
            if (call->error || (call->flags & RX_CALL_FAST_RECOVER_WAIT))
                return;
        }
        if (morePackets) {
-           rxi_SendList(call, &working, istack, 0, now, retryTime);
+           rxi_SendList(call, &working, istack, 0);
        }
     } else if (last.len > 0) {
-       rxi_SendList(call, &last, istack, 0, now, retryTime);
+       rxi_SendList(call, &last, istack, 0);
        /* Packets which are in 'working' are not sent by this call */
     }
 }
@@ -5713,19 +5713,10 @@ rxi_Start(struct rxevent *event,
     }
 
     if (queue_IsNotEmpty(&call->tq)) { /* If we have anything to send */
-       /* Get clock to compute the re-transmit time for any packets
-        * in this burst.  Note, if we back off, it's reasonable to
-        * back off all of the packets in the same manner, even if
-        * some of them have been retransmitted more times than more
-        * recent additions.
-        * Do a dance to avoid blocking after setting now. */
-       MUTEX_ENTER(&peer->peer_lock);
-        retryTime = peer->timeout;
-       MUTEX_EXIT(&peer->peer_lock);
 
-       clock_GetTime(&now);
-       clock_Add(&retryTime, &now);
+       clock_GetTime(&now);
        usenow = now;
+
        /* Send (or resend) any packets that need it, subject to
         * window restrictions and congestion burst control
         * restrictions.  Ask for an ack on the last packet sent in
@@ -5806,15 +5797,13 @@ rxi_Start(struct rxevent *event,
                    if (!clock_Lt(&now, &p->retryTime)) {
                        if (nXmitPackets == maxXmitPackets) {
                            rxi_SendXmitList(call, call->xmitList,
-                                            nXmitPackets, istack, &now, 
-                                            &retryTime);
+                                            nXmitPackets, istack);
                            goto restart;
                        }
-                        dpf(("call %d xmit packet %"AFS_PTR_FMT" now %u.%06u retryTime %u.%06u nextRetry %u.%06u\n",
+                        dpf(("call %d xmit packet %"AFS_PTR_FMT" now %u.%06u retryTime %u.%06u\n",
                               *(call->callNumber), p,
                               now.sec, now.usec,
-                              p->retryTime.sec, p->retryTime.usec,
-                              retryTime.sec, retryTime.usec));
+                              p->retryTime.sec, p->retryTime.usec));
                        call->xmitList[nXmitPackets++] = p;
                    }
                }
@@ -5823,7 +5812,7 @@ rxi_Start(struct rxevent *event,
                 * ready to send. Now we loop to send the packets */
                if (nXmitPackets > 0) {
                    rxi_SendXmitList(call, call->xmitList, nXmitPackets,
-                                    istack, &now, &retryTime);
+                                    istack);
                }
 
 #ifdef AFS_GLOBAL_RXLOCK_KERNEL