From: Andrew Deason Date: Thu, 2 Sep 2010 16:25:27 +0000 (-0500) Subject: vol: Add VInit cond var and remove busywaits X-Git-Tag: upstream/1.8.0_pre1^2~4845 X-Git-Url: https://git.michaelhowe.org/gitweb/?a=commitdiff_plain;h=31bb2e5eb5feb9c8e84d17b49fe70268127cdd43;p=packages%2Fo%2Fopenafs.git vol: Add VInit cond var and remove busywaits In DAFS, FSYNC_sync was waiting for VInit to reach at least 2 by looping around pthread_yield(). For a server with a large number of volumes, it can take a while for volumes to preattach, and so we are effectively busy-waiting for preattach to finish. This can slow fileserver startup and peg the cpu. So instead, add a condition variable for when VInit changes, and wait on that. Also modify other checkers of VInit to use the cond var. Change-Id: Icc251fdf7d525d40a9db1938a9cb4d47a74dccba Reviewed-on: http://gerrit.openafs.org/2648 Tested-by: Andrew Deason Tested-by: BuildBot Reviewed-by: Derrick Brashear --- diff --git a/src/vol/fssync-server.c b/src/vol/fssync-server.c index c9d6b1c69..2b18048b8 100644 --- a/src/vol/fssync-server.c +++ b/src/vol/fssync-server.c @@ -266,6 +266,10 @@ FSYNC_sync(void * args) int min_vinit = 1; #endif /* AFS_DEMAND_ATTACH_FS */ + /* we must not be called before vol package initialization, since we use + * vol package mutexes and conds etc */ + assert(VInit); + SYNC_getAddr(&state->endpoint, &state->addr); SYNC_cleanupSock(state); @@ -282,17 +286,22 @@ FSYNC_sync(void * args) Log("Set thread id %d for FSYNC_sync\n", tid); #endif /* AFS_PTHREAD_ENV */ + VOL_LOCK; + while (VInit < min_vinit) { /* Let somebody else run until all volumes have been preattached * (DAFS), or we have started attaching volumes (non-DAFS). This * doesn't mean that all volumes have been attached. */ #ifdef AFS_PTHREAD_ENV - pthread_yield(); + VOL_CV_WAIT(&vol_vinit_cond); #else /* AFS_PTHREAD_ENV */ LWP_DispatchProcess(); #endif /* AFS_PTHREAD_ENV */ } + + VOL_UNLOCK; + state->fd = SYNC_getSock(&state->endpoint); code = SYNC_bindSock(state); assert(!code); diff --git a/src/vol/volume.c b/src/vol/volume.c index 6b7d95a1e..b9b0ab3e7 100644 --- a/src/vol/volume.c +++ b/src/vol/volume.c @@ -156,6 +156,7 @@ pthread_mutex_t vol_trans_mutex; pthread_cond_t vol_put_volume_cond; pthread_cond_t vol_sleep_cond; pthread_cond_t vol_init_attach_cond; +pthread_cond_t vol_vinit_cond; int vol_attach_threads = 1; #endif /* AFS_PTHREAD_ENV */ @@ -544,6 +545,22 @@ VOptDefaults(ProgramType pt, VolumePackageOptions *opts) } } +/** + * Set VInit to a certain value, and signal waiters. + * + * @param[in] value the value to set VInit to + * + * @pre VOL_LOCK held + */ +static void +VSetVInit_r(int value) +{ + VInit = value; +#ifdef AFS_PTHREAD_ENV + assert(pthread_cond_broadcast(&vol_vinit_cond) == 0); +#endif +} + int VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts) { @@ -572,6 +589,7 @@ VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts) assert(pthread_cond_init(&vol_put_volume_cond, NULL) == 0); assert(pthread_cond_init(&vol_sleep_cond, NULL) == 0); assert(pthread_cond_init(&vol_init_attach_cond, NULL) == 0); + assert(pthread_cond_init(&vol_vinit_cond, NULL) == 0); #else /* AFS_PTHREAD_ENV */ IOMGR_Initialize(); #endif /* AFS_PTHREAD_ENV */ @@ -666,7 +684,7 @@ VInitAttachVolumes(ProgramType pt) } } VOL_LOCK; - VInit = 2; /* Initialized, and all volumes have been attached */ + VSetVInit_r(2); /* Initialized, and all volumes have been attached */ LWP_NoYieldSignal(VInitAttachVolumes); VOL_UNLOCK; return 0; @@ -748,7 +766,7 @@ VInitAttachVolumes(ProgramType pt) assert(pthread_cond_destroy(¶ms.thread_done_cv) == 0); } VOL_LOCK; - VInit = 2; /* Initialized, and all volumes have been attached */ + VSetVInit_r(2); /* Initialized, and all volumes have been attached */ assert(pthread_cond_broadcast(&vol_init_attach_cond) == 0); VOL_UNLOCK; return 0; @@ -872,7 +890,7 @@ VInitAttachVolumes(ProgramType pt) } VOL_LOCK; - VInit = 2; /* Initialized, and all volumes have been attached */ + VSetVInit_r(2); /* Initialized, and all volumes have been attached */ assert(pthread_cond_broadcast(&vol_init_attach_cond) == 0); VOL_UNLOCK; @@ -5430,8 +5448,9 @@ VConnectFS_r(void) (programType != fileServer) && (programType != salvager)); rc = FSYNC_clientInit(); - if (rc) - VInit = 3; + if (rc) { + VSetVInit_r(3); + } return rc; } @@ -5457,7 +5476,7 @@ VDisconnectFS_r(void) assert((programType != fileServer) && (programType != salvager)); FSYNC_clientFinis(); - VInit = 2; + VSetVInit_r(2); } /** @@ -6094,7 +6113,7 @@ VSetDiskUsage_r(void) * initialization level indicates that all volumes are attached, * which implies that all partitions are initialized. */ #ifdef AFS_PTHREAD_ENV - sleep(10); + VOL_CV_WAIT(&vol_vinit_cond); #else /* AFS_PTHREAD_ENV */ IOMGR_Sleep(10); #endif /* AFS_PTHREAD_ENV */ diff --git a/src/vol/volume.h b/src/vol/volume.h index 8eabf8488..f46021200 100644 --- a/src/vol/volume.h +++ b/src/vol/volume.h @@ -68,6 +68,7 @@ extern pthread_mutex_t vol_glock_mutex; extern pthread_mutex_t vol_trans_mutex; extern pthread_cond_t vol_put_volume_cond; extern pthread_cond_t vol_sleep_cond; +extern pthread_cond_t vol_vinit_cond; extern ih_init_params vol_io_params; extern int vol_attach_threads; #ifdef VOL_LOCK_DEBUG