From d69b87724790ceb2db25b580cebfc4f18f99dbf9 Mon Sep 17 00:00:00 2001 From: Andrew Deason Date: Thu, 2 Sep 2010 11:25:27 -0500 Subject: [PATCH] vol: Add VInit cond var and remove busywaits In DAFS, FSYNC_sync was waiting for VInit to reach at least 2 by looping around pthread_yield(). For a server with a large number of volumes, it can take a while for volumes to preattach, and so we are effectively busy-waiting for preattach to finish. This can slow fileserver startup and peg the cpu. So instead, add a condition variable for when VInit changes, and wait on that. Also modify other checkers of VInit to use the cond var. (cherry picked from commit 31bb2e5eb5feb9c8e84d17b49fe70268127cdd43) Reviewed-on: http://gerrit.openafs.org/2648 Tested-by: Andrew Deason Tested-by: BuildBot Reviewed-by: Derrick Brashear Change-Id: I773ab01095bb0c101a34146ef9bf201946ebcf8b Reviewed-on: http://gerrit.openafs.org/3887 Tested-by: BuildBot Reviewed-by: Derrick Brashear --- src/vol/fssync-server.c | 11 ++++++++++- src/vol/volume.c | 31 ++++++++++++++++++++++++------- src/vol/volume.h | 1 + 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/vol/fssync-server.c b/src/vol/fssync-server.c index d9197894d..51e995258 100644 --- a/src/vol/fssync-server.c +++ b/src/vol/fssync-server.c @@ -262,6 +262,10 @@ FSYNC_sync(void * args) int min_vinit = 1; #endif /* AFS_DEMAND_ATTACH_FS */ + /* we must not be called before vol package initialization, since we use + * vol package mutexes and conds etc */ + osi_Assert(VInit); + SYNC_getAddr(&state->endpoint, &state->addr); SYNC_cleanupSock(state); @@ -278,17 +282,22 @@ FSYNC_sync(void * args) Log("Set thread id %d for FSYNC_sync\n", tid); #endif /* AFS_PTHREAD_ENV */ + VOL_LOCK; + while (VInit < min_vinit) { /* Let somebody else run until all volumes have been preattached * (DAFS), or we have started attaching volumes (non-DAFS). This * doesn't mean that all volumes have been attached. */ #ifdef AFS_PTHREAD_ENV - pthread_yield(); + VOL_CV_WAIT(&vol_vinit_cond); #else /* AFS_PTHREAD_ENV */ LWP_DispatchProcess(); #endif /* AFS_PTHREAD_ENV */ } + + VOL_UNLOCK; + state->fd = SYNC_getSock(&state->endpoint); code = SYNC_bindSock(state); osi_Assert(!code); diff --git a/src/vol/volume.c b/src/vol/volume.c index 421d5de6c..78aaa21b3 100644 --- a/src/vol/volume.c +++ b/src/vol/volume.c @@ -151,6 +151,7 @@ pthread_mutex_t vol_trans_mutex; pthread_cond_t vol_put_volume_cond; pthread_cond_t vol_sleep_cond; pthread_cond_t vol_init_attach_cond; +pthread_cond_t vol_vinit_cond; int vol_attach_threads = 1; #endif /* AFS_PTHREAD_ENV */ @@ -539,6 +540,20 @@ VOptDefaults(ProgramType pt, VolumePackageOptions *opts) } } +/** + * Set VInit to a certain value, and signal waiters. + * + * @param[in] value the value to set VInit to + * + * @pre VOL_LOCK held + */ +static void +VSetVInit_r(int value) +{ + VInit = value; + CV_BROADCAST(&vol_vinit_cond); +} + int VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts) { @@ -567,6 +582,7 @@ VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts) CV_INIT(&vol_put_volume_cond, "vol put", CV_DEFAULT, 0); CV_INIT(&vol_sleep_cond, "vol sleep", CV_DEFAULT, 0); CV_INIT(&vol_init_attach_cond, "vol init attach", CV_DEFAULT, 0); + CV_INIT(&vol_vinit_cond, "vol vinit", CV_DEFAULT, 0); #else /* AFS_PTHREAD_ENV */ IOMGR_Initialize(); #endif /* AFS_PTHREAD_ENV */ @@ -661,7 +677,7 @@ VInitAttachVolumes(ProgramType pt) } } VOL_LOCK; - VInit = 2; /* Initialized, and all volumes have been attached */ + VSetVInit_r(2); /* Initialized, and all volumes have been attached */ LWP_NoYieldSignal(VInitAttachVolumes); VOL_UNLOCK; return 0; @@ -743,7 +759,7 @@ VInitAttachVolumes(ProgramType pt) CV_DESTROY(¶ms.thread_done_cv); } VOL_LOCK; - VInit = 2; /* Initialized, and all volumes have been attached */ + VSetVInit_r(2); /* Initialized, and all volumes have been attached */ CV_BROADCAST(&vol_init_attach_cond); VOL_UNLOCK; return 0; @@ -867,7 +883,7 @@ VInitAttachVolumes(ProgramType pt) } VOL_LOCK; - VInit = 2; /* Initialized, and all volumes have been attached */ + VSetVInit_r(2); /* Initialized, and all volumes have been attached */ CV_BROADCAST(&vol_init_attach_cond); VOL_UNLOCK; @@ -5551,8 +5567,9 @@ VConnectFS_r(void) (programType != fileServer) && (programType != salvager)); rc = FSYNC_clientInit(); - if (rc) - VInit = 3; + if (rc) { + VSetVInit_r(3); + } return rc; } @@ -5578,7 +5595,7 @@ VDisconnectFS_r(void) osi_Assert((programType != fileServer) && (programType != salvager)); FSYNC_clientFinis(); - VInit = 2; + VSetVInit_r(2); } /** @@ -6215,7 +6232,7 @@ VSetDiskUsage_r(void) * initialization level indicates that all volumes are attached, * which implies that all partitions are initialized. */ #ifdef AFS_PTHREAD_ENV - sleep(10); + VOL_CV_WAIT(&vol_vinit_cond); #else /* AFS_PTHREAD_ENV */ IOMGR_Sleep(10); #endif /* AFS_PTHREAD_ENV */ diff --git a/src/vol/volume.h b/src/vol/volume.h index b6a54c9a4..da961a628 100644 --- a/src/vol/volume.h +++ b/src/vol/volume.h @@ -67,6 +67,7 @@ extern pthread_mutex_t vol_glock_mutex; extern pthread_mutex_t vol_trans_mutex; extern pthread_cond_t vol_put_volume_cond; extern pthread_cond_t vol_sleep_cond; +extern pthread_cond_t vol_vinit_cond; extern ih_init_params vol_io_params; extern int vol_attach_threads; #ifdef VOL_LOCK_DEBUG -- 2.39.5