rx: Don't malloc the xmit list

Building the transmit list happens in a time critical section of
code. Using malloc to allocate the list which holds the packets to
be transmitted slows down this critical section. Instead, just
allocate the space as part of the call structure.

Locking of xmitList is somewhat tricksy, as the call->lock is
dropped over calls to sendmsg(). However, the xmitList is protected
by the TQ_BUSY call flag, which prevents multiple threads from
usign the transmit queue, and hence the xmitList, simultaneously.

Change-Id: Iff64979fa1caeed2ba57d915fecb7ce823f345cf
Reviewed-on: http://gerrit.openafs.org/2957
Reviewed-by: Jeffrey Altman <jaltman@openafs.org>
Tested-by: Jeffrey Altman <jaltman@openafs.org>

diff --git a/src/rx/rx.c b/src/rx/rx.c
index 71b1a9987f52e4c8bb06805f8c4ed3eb60a62433..9696e3df25a8e09a44f4ed48f9a9642c62d00cae 100644 (file)
--- a/src/rx/rx.c
+++ b/src/rx/rx.c
@@ -5610,7 +5610,6 @@ rxi_Start(struct rxevent *event,
     int haveEvent;
     int nXmitPackets;
     int maxXmitPackets;
-    struct rx_packet **xmitList;
     int resending = 0;
 
     /* If rxi_Start is being called as a result of a resend event,
@@ -5714,15 +5713,6 @@ rxi_Start(struct rxevent *event,
 #endif /* AFS_GLOBAL_RXLOCK_KERNEL */
                nXmitPackets = 0;
                maxXmitPackets = MIN(call->twind, call->cwind);
-               xmitList = (struct rx_packet **)
-#if defined(KERNEL) && !defined(UKERNEL) && defined(AFS_FBSD80_ENV)
-                   /* XXXX else we must drop any mtx we hold */
-                   afs_osi_Alloc_NoSleep(maxXmitPackets * sizeof(struct rx_packet *));
-#else
-               osi_Alloc(maxXmitPackets * sizeof(struct rx_packet *));
-#endif
-               if (xmitList == NULL)
-                   osi_Panic("rxi_Start, failed to allocate xmit list");
                for (queue_Scan(&call->tq, p, nxp, rx_packet)) {
                    if (call->flags & RX_CALL_FAST_RECOVER_WAIT) {
                        /* We shouldn't be sending packets if a thread is waiting
@@ -5776,11 +5766,9 @@ rxi_Start(struct rxevent *event,
                    /* Transmit the packet if it needs to be sent. */
                    if (!clock_Lt(&now, &p->retryTime)) {
                        if (nXmitPackets == maxXmitPackets) {
-                           rxi_SendXmitList(call, xmitList, nXmitPackets,
-                                            istack, &now, &retryTime,
-                                            resending);
-                           osi_Free(xmitList, maxXmitPackets *
-                                    sizeof(struct rx_packet *));
+                           rxi_SendXmitList(call, call->xmitList,
+                                            nXmitPackets, istack, &now, 
+                                            &retryTime, resending);
                            goto restart;
                        }
                         dpf(("call %d xmit packet %"AFS_PTR_FMT" now %u.%06u retryTime %u.%06u nextRetry %u.%06u\n",
@@ -5788,18 +5776,16 @@ rxi_Start(struct rxevent *event,
                               now.sec, now.usec,
                               p->retryTime.sec, p->retryTime.usec,
                               retryTime.sec, retryTime.usec));
-                       xmitList[nXmitPackets++] = p;
+                       call->xmitList[nXmitPackets++] = p;
                    }
                }
 
                /* xmitList now hold pointers to all of the packets that are
                 * ready to send. Now we loop to send the packets */
                if (nXmitPackets > 0) {
-                   rxi_SendXmitList(call, xmitList, nXmitPackets, istack,
-                                    &now, &retryTime, resending);
+                   rxi_SendXmitList(call, call->xmitList, nXmitPackets,
+                                    istack, &now, &retryTime, resending);
                }
-               osi_Free(xmitList,
-                        maxXmitPackets * sizeof(struct rx_packet *));
 
 #ifdef AFS_GLOBAL_RXLOCK_KERNEL
                /*

diff --git a/src/rx/rx.h b/src/rx/rx.h
index 49614aa0a06bce18d12fee6a728146e10cebf505..87049908451d3380201c68b83cbef223956ccb93 100644 (file)
--- a/src/rx/rx.h
+++ b/src/rx/rx.h
@@ -459,6 +459,9 @@ struct rx_peer {
 #define        RX_SERVER_CONNECTION    1
 #endif /* !KDUMP_RX_LOCK */
 
+/* Maximum number of acknowledgements in an acknowledge packet */
+#define        RX_MAXACKS          255
+
 /* Call structure:  only instantiated for active calls and dallying server calls.  The permanent call state (i.e. the call number as well as state shared with other calls associated with this connection) is maintained in the connection structure. */
 #ifdef KDUMP_RX_LOCK
 struct rx_call_rx_lock {
@@ -583,6 +586,8 @@ struct rx_call {
     afs_hyper_t bytesRcvd;     /* Number bytes received */
     u_short tqWaiters;
 
+    struct rx_packet *xmitList[RX_MAXACKS]; /* Can't xmit more than we ack */
+                                /* Protected by setting RX_CALL_TQ_BUSY */
 #ifdef ADAPT_WINDOW
     struct clock pingRequestTime;
 #endif
@@ -634,8 +639,6 @@ struct rx_call {
 #define RX_CALL_HAVE_LAST      32768   /* Last packet has been received */
 #define RX_CALL_NEED_START     0x10000 /* tells rxi_Start to start again */
 
-/* Maximum number of acknowledgements in an acknowledge packet */
-#define        RX_MAXACKS          255
 
 /* The structure of the data portion of an acknowledge packet: An acknowledge
  * packet is in network byte order at all times.  An acknowledgement is always