From 41633936ce0ac35c9b654c8666241d930f140bbd Mon Sep 17 00:00:00 2001 From: Simon Wilkinson Date: Wed, 29 Sep 2010 00:06:45 +0100 Subject: [PATCH] rx: Make nWaiting and nWaited atomic Make the nWaiting and nWaited counters atomic, and get rid of the mutex which used to protect them. Change-Id: I0c4d8f1df1860baa2bb189ea77bf20ee9f2066f7 Reviewed-on: http://gerrit.openafs.org/2860 Reviewed-by: Jeffrey Altman Tested-by: Jeffrey Altman --- src/rx/rx.c | 37 ++++++++++++++++--------------------- src/rx/rx_globals.h | 2 -- src/rx/rx_internal.h | 11 +++++++++++ src/rx/rx_packet.c | 10 ++++++++-- src/rx/rx_trace.c | 4 +++- 5 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 src/rx/rx_internal.h diff --git a/src/rx/rx.c b/src/rx/rx.c index 81daa78ad..66f88d937 100644 --- a/src/rx/rx.c +++ b/src/rx/rx.c @@ -70,6 +70,7 @@ #include "rx_globals.h" #include "rx_trace.h" #include "rx_atomic.h" +#include "rx_internal.h" #define AFSOP_STOP_RXCALLBACK 210 /* Stop CALLBACK process */ #define AFSOP_STOP_AFS 211 /* Stop AFS process */ #define AFSOP_STOP_BKG 212 /* Stop BKG process */ @@ -108,6 +109,7 @@ extern afs_int32 afs_termState; # include "rx_atomic.h" # include "rx_globals.h" # include "rx_trace.h" +# include "rx_internal.h" # include #endif /* KERNEL */ @@ -154,6 +156,9 @@ static unsigned int rxi_rpc_peer_stat_cnt; static unsigned int rxi_rpc_process_stat_cnt; +rx_atomic_t rx_nWaiting = RX_ATOMIC_INIT(0); +rx_atomic_t rx_nWaited = RX_ATOMIC_INIT(0); + #if !defined(offsetof) #include /* for definition of offsetof() */ #endif @@ -171,7 +176,6 @@ afs_kmutex_t rx_atomic_mutex; */ extern afs_kmutex_t rx_stats_mutex; -extern afs_kmutex_t rx_waiting_mutex; extern afs_kmutex_t rx_quota_mutex; extern afs_kmutex_t rx_pthread_mutex; extern afs_kmutex_t rx_packets_mutex; @@ -201,7 +205,6 @@ rxi_InitPthread(void) { MUTEX_INIT(&rx_clock_mutex, "clock", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_stats_mutex, "stats", MUTEX_DEFAULT, 0); - MUTEX_INIT(&rx_waiting_mutex, "waiting", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_atomic_mutex, "atomic", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_quota_mutex, "quota", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_pthread_mutex, "pthread", MUTEX_DEFAULT, 0); @@ -478,7 +481,6 @@ rx_InitHost(u_int host, u_int port) rxdb_init(); #endif /* RX_LOCKS_DB */ MUTEX_INIT(&rx_stats_mutex, "rx_stats_mutex", MUTEX_DEFAULT, 0); - MUTEX_INIT(&rx_waiting_mutex, "rx_waiting_mutex", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_quota_mutex, "rx_quota_mutex", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_pthread_mutex, "rx_pthread_mutex", MUTEX_DEFAULT, 0); MUTEX_INIT(&rx_packets_mutex, "rx_packets_mutex", MUTEX_DEFAULT, 0); @@ -1755,9 +1757,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp) if (call->flags & RX_CALL_WAIT_PROC) { call->flags &= ~RX_CALL_WAIT_PROC; - MUTEX_ENTER(&rx_waiting_mutex); - rx_nWaiting--; - MUTEX_EXIT(&rx_waiting_mutex); + rx_atomic_dec(&rx_nWaiting); } if (call->state != RX_STATE_PRECALL || call->error) { @@ -1939,7 +1939,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp) rxi_minDeficit--; rxi_availProcs--; MUTEX_EXIT(&rx_quota_mutex); - rx_nWaiting--; + rx_atomic_dec(&rx_nWaiting); /* MUTEX_EXIT(&call->lock); */ } else { /* If there are no eligible incoming calls, add this process @@ -2912,7 +2912,8 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket, * If the number of queued calls exceeds the overload * threshold then abort this call. */ - if ((rx_BusyThreshold > 0) && (rx_nWaiting > rx_BusyThreshold)) { + if ((rx_BusyThreshold > 0) && + (rx_atomic_read(&rx_nWaiting) > rx_BusyThreshold)) { struct rx_packet *tp; rxi_CallError(call, rx_BusyError); @@ -2980,7 +2981,8 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket, * If the number of queued calls exceeds the overload * threshold then abort this call. */ - if ((rx_BusyThreshold > 0) && (rx_nWaiting > rx_BusyThreshold)) { + if ((rx_BusyThreshold > 0) && + (rx_atomic_read(&rx_nWaiting) > rx_BusyThreshold)) { struct rx_packet *tp; rxi_CallError(call, rx_BusyError); @@ -4456,10 +4458,8 @@ rxi_AttachServerProc(struct rx_call *call, if (!(call->flags & RX_CALL_WAIT_PROC)) { call->flags |= RX_CALL_WAIT_PROC; - MUTEX_ENTER(&rx_waiting_mutex); - rx_nWaiting++; - rx_nWaited++; - MUTEX_EXIT(&rx_waiting_mutex); + rx_atomic_inc(&rx_nWaiting); + rx_atomic_inc(&rx_nWaited); rxi_calltrace(RX_CALL_ARRIVAL, call); SET_CALL_QUEUE_LOCK(call, &rx_serverPool_lock); queue_Append(&rx_incomingCallQueue, call); @@ -4487,9 +4487,7 @@ rxi_AttachServerProc(struct rx_call *call, if (queue_IsOnQueue(call)) { queue_Remove(call); - MUTEX_ENTER(&rx_waiting_mutex); - rx_nWaiting--; - MUTEX_EXIT(&rx_waiting_mutex); + rx_atomic_dec(&rx_nWaiting); } } call->state = RX_STATE_ACTIVE; @@ -4958,10 +4956,7 @@ rxi_ResetCall(struct rx_call *call, int newcall) if (queue_IsOnQueue(call)) { queue_Remove(call); if (flags & RX_CALL_WAIT_PROC) { - - MUTEX_ENTER(&rx_waiting_mutex); - rx_nWaiting--; - MUTEX_EXIT(&rx_waiting_mutex); + rx_atomic_dec(&rx_nWaiting); } } MUTEX_EXIT(call->call_queue_lock); @@ -4971,7 +4966,7 @@ rxi_ResetCall(struct rx_call *call, int newcall) if (queue_IsOnQueue(call)) { queue_Remove(call); if (flags & RX_CALL_WAIT_PROC) - rx_nWaiting--; + rx_atomic_dec(&rx_nWaiting); } #endif /* RX_ENABLE_LOCKS */ diff --git a/src/rx/rx_globals.h b/src/rx/rx_globals.h index 72f668351..dbf7cc6b1 100644 --- a/src/rx/rx_globals.h +++ b/src/rx/rx_globals.h @@ -429,8 +429,6 @@ EXT int rx_TSFPQMaxProcs GLOBALSINIT(0); /* max number of threads expected */ /* Number of free packets */ EXT int rx_nFreePackets GLOBALSINIT(0); EXT int rxi_NeedMorePackets GLOBALSINIT(0); -EXT int rx_nWaiting GLOBALSINIT(0); -EXT int rx_nWaited GLOBALSINIT(0); EXT int rx_packetReclaims GLOBALSINIT(0); /* largest packet which we can safely receive, initialized to AFS 3.2 value diff --git a/src/rx/rx_internal.h b/src/rx/rx_internal.h new file mode 100644 index 000000000..91caee7f3 --- /dev/null +++ b/src/rx/rx_internal.h @@ -0,0 +1,11 @@ +/* Internal structures that are private to RX itself. These shouldn't be + * modified by library callers. + * + * Data structures that are visible to security layers, but not to + * customers of RX belong in rx_private.h, which is installed. + */ + + +/* Globals that we don't want the world to know about */ +extern rx_atomic_t rx_nWaiting; +extern rx_atomic_t rx_nWaited; diff --git a/src/rx/rx_packet.c b/src/rx/rx_packet.c index f9411c531..83e50b324 100644 --- a/src/rx/rx_packet.c +++ b/src/rx/rx_packet.c @@ -22,6 +22,8 @@ #include "rx/rx_clock.h" #include "rx/rx_queue.h" #include "rx/rx_packet.h" +#include "rx/rx_atomic.h" +#include "rx/rx_internal.h" #else /* defined(UKERNEL) */ #ifdef RX_KERNEL_TRACE #include "../rx/rx_kcommon.h" @@ -48,10 +50,12 @@ #include "rx_kmutex.h" #include "rx/rx_clock.h" #include "rx/rx_queue.h" +#include "rx_atomic.h" #ifdef AFS_SUN5_ENV #include #endif #include "rx/rx_packet.h" +#include "rx_internal.h" #endif /* defined(UKERNEL) */ #include "rx/rx_globals.h" #else /* KERNEL */ @@ -77,7 +81,9 @@ #include #endif #include "rx_packet.h" +#include "rx_atomic.h" #include "rx_globals.h" +#include "rx_internal.h" #include #include #include @@ -1821,8 +1827,8 @@ rxi_ReceiveDebugPacket(struct rx_packet *ap, osi_socket asocket, tstat.callsExecuted = htonl(rxi_nCalls); tstat.packetReclaims = htonl(rx_packetReclaims); tstat.usedFDs = CountFDs(64); - tstat.nWaiting = htonl(rx_nWaiting); - tstat.nWaited = htonl(rx_nWaited); + tstat.nWaiting = htonl(rx_atomic_read(&rx_nWaiting)); + tstat.nWaited = htonl(rx_atomic_read(&rx_nWaited)); queue_Count(&rx_idleServerQueue, np, nqe, rx_serverQueueEntry, tstat.idleThreads); MUTEX_EXIT(&rx_serverPool_lock); diff --git a/src/rx/rx_trace.c b/src/rx/rx_trace.c index 973f795a1..b4fbbd58c 100644 --- a/src/rx/rx_trace.c +++ b/src/rx/rx_trace.c @@ -30,7 +30,9 @@ main(int argc, char **argv) #include #endif #include "rx.h" +#include "rx_atomic.h" #include "rx_globals.h" +#include "rx_internal.h" #include "rx_trace.h" #ifdef RXTRACEON @@ -81,7 +83,7 @@ rxi_calltrace(unsigned int event, struct rx_call *call) rxtinfo.now = now.sec * 1000 + now.usec / 1000; rxtinfo.cid = call->conn->cid; rxtinfo.call = *(call->callNumber); - rxtinfo.qlen = rx_nWaiting; + rxtinfo.qlen = rx_atomic_read(&rx_nWaiting); rxtinfo.servicetime = 0; rxtinfo.waittime = 0; -- 2.39.5