rx: Make nWaiting and nWaited atomic

Make the nWaiting and nWaited counters atomic, and get rid of the
mutex which used to protect them.

Change-Id: I0c4d8f1df1860baa2bb189ea77bf20ee9f2066f7
Reviewed-on: http://gerrit.openafs.org/2860
Reviewed-by: Jeffrey Altman <jaltman@openafs.org>
Tested-by: Jeffrey Altman <jaltman@openafs.org>

diff --git a/src/rx/rx.c b/src/rx/rx.c
index 81daa78ad4cc541a07063d03a9d35b3113aa2e31..66f88d937044ea60c3dd3f989bc10ca7132b0a51 100644 (file)
--- a/src/rx/rx.c
+++ b/src/rx/rx.c
@@ -70,6 +70,7 @@
 #include "rx_globals.h"
 #include "rx_trace.h"
 #include "rx_atomic.h"
+#include "rx_internal.h"
 #define        AFSOP_STOP_RXCALLBACK   210     /* Stop CALLBACK process */
 #define        AFSOP_STOP_AFS          211     /* Stop AFS process */
 #define        AFSOP_STOP_BKG          212     /* Stop BKG process */
@@ -108,6 +109,7 @@ extern afs_int32 afs_termState;
 # include "rx_atomic.h"
 # include "rx_globals.h"
 # include "rx_trace.h"
+# include "rx_internal.h"
 # include <afs/rxgen_consts.h>
 #endif /* KERNEL */
 
@@ -154,6 +156,9 @@ static unsigned int rxi_rpc_peer_stat_cnt;
 
 static unsigned int rxi_rpc_process_stat_cnt;
 
+rx_atomic_t rx_nWaiting = RX_ATOMIC_INIT(0);
+rx_atomic_t rx_nWaited = RX_ATOMIC_INIT(0);
+
 #if !defined(offsetof)
 #include <stddef.h>            /* for definition of offsetof() */
 #endif
@@ -171,7 +176,6 @@ afs_kmutex_t rx_atomic_mutex;
  */
 
 extern afs_kmutex_t rx_stats_mutex;
-extern afs_kmutex_t rx_waiting_mutex;
 extern afs_kmutex_t rx_quota_mutex;
 extern afs_kmutex_t rx_pthread_mutex;
 extern afs_kmutex_t rx_packets_mutex;
@@ -201,7 +205,6 @@ rxi_InitPthread(void)
 {
     MUTEX_INIT(&rx_clock_mutex, "clock", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_stats_mutex, "stats", MUTEX_DEFAULT, 0);
-    MUTEX_INIT(&rx_waiting_mutex, "waiting", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_atomic_mutex, "atomic", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_quota_mutex, "quota", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_pthread_mutex, "pthread", MUTEX_DEFAULT, 0);
@@ -478,7 +481,6 @@ rx_InitHost(u_int host, u_int port)
     rxdb_init();
 #endif /* RX_LOCKS_DB */
     MUTEX_INIT(&rx_stats_mutex, "rx_stats_mutex", MUTEX_DEFAULT, 0);
-    MUTEX_INIT(&rx_waiting_mutex, "rx_waiting_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_quota_mutex, "rx_quota_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_pthread_mutex, "rx_pthread_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_packets_mutex, "rx_packets_mutex", MUTEX_DEFAULT, 0);
@@ -1755,9 +1757,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
 
            if (call->flags & RX_CALL_WAIT_PROC) {
                call->flags &= ~RX_CALL_WAIT_PROC;
-                MUTEX_ENTER(&rx_waiting_mutex);
-                rx_nWaiting--;
-                MUTEX_EXIT(&rx_waiting_mutex);
+               rx_atomic_dec(&rx_nWaiting);
            }
 
            if (call->state != RX_STATE_PRECALL || call->error) {
@@ -1939,7 +1939,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
            rxi_minDeficit--;
        rxi_availProcs--;
         MUTEX_EXIT(&rx_quota_mutex);
-       rx_nWaiting--;
+       rx_atomic_dec(&rx_nWaiting);
        /* MUTEX_EXIT(&call->lock); */
     } else {
        /* If there are no eligible incoming calls, add this process
@@ -2912,7 +2912,8 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             * If the number of queued calls exceeds the overload
             * threshold then abort this call.
             */
-           if ((rx_BusyThreshold > 0) && (rx_nWaiting > rx_BusyThreshold)) {
+           if ((rx_BusyThreshold > 0) &&
+               (rx_atomic_read(&rx_nWaiting) > rx_BusyThreshold)) {
                struct rx_packet *tp;
 
                rxi_CallError(call, rx_BusyError);
@@ -2980,7 +2981,8 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             * If the number of queued calls exceeds the overload
             * threshold then abort this call.
             */
-           if ((rx_BusyThreshold > 0) && (rx_nWaiting > rx_BusyThreshold)) {
+           if ((rx_BusyThreshold > 0) &&
+               (rx_atomic_read(&rx_nWaiting) > rx_BusyThreshold)) {
                struct rx_packet *tp;
 
                rxi_CallError(call, rx_BusyError);
@@ -4456,10 +4458,8 @@ rxi_AttachServerProc(struct rx_call *call,
 
        if (!(call->flags & RX_CALL_WAIT_PROC)) {
            call->flags |= RX_CALL_WAIT_PROC;
-            MUTEX_ENTER(&rx_waiting_mutex);
-            rx_nWaiting++;
-            rx_nWaited++;
-            MUTEX_EXIT(&rx_waiting_mutex);
+           rx_atomic_inc(&rx_nWaiting);
+           rx_atomic_inc(&rx_nWaited);
            rxi_calltrace(RX_CALL_ARRIVAL, call);
            SET_CALL_QUEUE_LOCK(call, &rx_serverPool_lock);
            queue_Append(&rx_incomingCallQueue, call);
@@ -4487,9 +4487,7 @@ rxi_AttachServerProc(struct rx_call *call,
            if (queue_IsOnQueue(call)) {
                queue_Remove(call);
 
-                MUTEX_ENTER(&rx_waiting_mutex);
-                rx_nWaiting--;
-                MUTEX_EXIT(&rx_waiting_mutex);
+               rx_atomic_dec(&rx_nWaiting);
            }
        }
        call->state = RX_STATE_ACTIVE;
@@ -4958,10 +4956,7 @@ rxi_ResetCall(struct rx_call *call, int newcall)
        if (queue_IsOnQueue(call)) {
            queue_Remove(call);
            if (flags & RX_CALL_WAIT_PROC) {
-
-                MUTEX_ENTER(&rx_waiting_mutex);
-                rx_nWaiting--;
-                MUTEX_EXIT(&rx_waiting_mutex);
+               rx_atomic_dec(&rx_nWaiting);
            }
        }
        MUTEX_EXIT(call->call_queue_lock);
@@ -4971,7 +4966,7 @@ rxi_ResetCall(struct rx_call *call, int newcall)
     if (queue_IsOnQueue(call)) {
        queue_Remove(call);
        if (flags & RX_CALL_WAIT_PROC)
-           rx_nWaiting--;
+           rx_atomic_dec(&rx_nWaiting);
     }
 #endif /* RX_ENABLE_LOCKS */
 

diff --git a/src/rx/rx_globals.h b/src/rx/rx_globals.h
index 72f668351ceada457e3fd1f8b3e53e791ba63351..dbf7cc6b1850e9cc9772f69b926036aff7065f14 100644 (file)
--- a/src/rx/rx_globals.h
+++ b/src/rx/rx_globals.h
@@ -429,8 +429,6 @@ EXT int rx_TSFPQMaxProcs GLOBALSINIT(0); /* max number of threads expected */
 /* Number of free packets */
 EXT int rx_nFreePackets GLOBALSINIT(0);
 EXT int rxi_NeedMorePackets GLOBALSINIT(0);
-EXT int rx_nWaiting GLOBALSINIT(0);
-EXT int rx_nWaited GLOBALSINIT(0);
 EXT int rx_packetReclaims GLOBALSINIT(0);
 
 /* largest packet which we can safely receive, initialized to AFS 3.2 value

diff --git a/src/rx/rx_internal.h b/src/rx/rx_internal.h
new file mode 100644 (file)
index 0000000..91caee7
--- /dev/null
+++ b/src/rx/rx_internal.h
@@ -0,0 +1,11 @@
+/* Internal structures that are private to RX itself. These shouldn't be
+ * modified by library callers.
+ *
+ * Data structures that are visible to security layers, but not to
+ * customers of RX belong in rx_private.h, which is installed.
+ */
+
+
+/* Globals that we don't want the world to know about */
+extern rx_atomic_t rx_nWaiting;
+extern rx_atomic_t rx_nWaited;

diff --git a/src/rx/rx_packet.c b/src/rx/rx_packet.c
index f9411c5310ca418a5d02ad76ba6d422762a13743..83e50b324931faac0c84ecedae8b8be9c6165b97 100644 (file)
--- a/src/rx/rx_packet.c
+++ b/src/rx/rx_packet.c
@@ -22,6 +22,8 @@
 #include "rx/rx_clock.h"
 #include "rx/rx_queue.h"
 #include "rx/rx_packet.h"
+#include "rx/rx_atomic.h"
+#include "rx/rx_internal.h"
 #else /* defined(UKERNEL) */
 #ifdef RX_KERNEL_TRACE
 #include "../rx/rx_kcommon.h"
@@ -48,10 +50,12 @@
 #include "rx_kmutex.h"
 #include "rx/rx_clock.h"
 #include "rx/rx_queue.h"
+#include "rx_atomic.h"
 #ifdef AFS_SUN5_ENV
 #include <sys/sysmacros.h>
 #endif
 #include "rx/rx_packet.h"
+#include "rx_internal.h"
 #endif /* defined(UKERNEL) */
 #include "rx/rx_globals.h"
 #else /* KERNEL */
@@ -77,7 +81,9 @@
 #include <sys/sysmacros.h>
 #endif
 #include "rx_packet.h"
+#include "rx_atomic.h"
 #include "rx_globals.h"
+#include "rx_internal.h"
 #include <lwp.h>
 #include <assert.h>
 #include <string.h>
@@ -1821,8 +1827,8 @@ rxi_ReceiveDebugPacket(struct rx_packet *ap, osi_socket asocket,
            tstat.callsExecuted = htonl(rxi_nCalls);
            tstat.packetReclaims = htonl(rx_packetReclaims);
            tstat.usedFDs = CountFDs(64);
-           tstat.nWaiting = htonl(rx_nWaiting);
-           tstat.nWaited = htonl(rx_nWaited);
+           tstat.nWaiting = htonl(rx_atomic_read(&rx_nWaiting));
+           tstat.nWaited = htonl(rx_atomic_read(&rx_nWaited));
            queue_Count(&rx_idleServerQueue, np, nqe, rx_serverQueueEntry,
                        tstat.idleThreads);
            MUTEX_EXIT(&rx_serverPool_lock);

diff --git a/src/rx/rx_trace.c b/src/rx/rx_trace.c
index 973f795a1ef999854aa51a69f0e1a5e4d724529f..b4fbbd58c18e871098266cf79fee1a425a0fddd5 100644 (file)
--- a/src/rx/rx_trace.c
+++ b/src/rx/rx_trace.c
@@ -30,7 +30,9 @@ main(int argc, char **argv)
 #include <unistd.h>
 #endif
 #include "rx.h"
+#include "rx_atomic.h"
 #include "rx_globals.h"
+#include "rx_internal.h"
 #include "rx_trace.h"
 
 #ifdef RXTRACEON
@@ -81,7 +83,7 @@ rxi_calltrace(unsigned int event, struct rx_call *call)
     rxtinfo.now = now.sec * 1000 + now.usec / 1000;
     rxtinfo.cid = call->conn->cid;
     rxtinfo.call = *(call->callNumber);
-    rxtinfo.qlen = rx_nWaiting;
+    rxtinfo.qlen = rx_atomic_read(&rx_nWaiting);
     rxtinfo.servicetime = 0;
     rxtinfo.waittime = 0;