vol: Add VInit cond var and remove busywaits

In DAFS, FSYNC_sync was waiting for VInit to reach at least 2 by
looping around pthread_yield(). For a server with a large number of
volumes, it can take a while for volumes to preattach, and so we are
effectively busy-waiting for preattach to finish. This can slow
fileserver startup and peg the cpu.

So instead, add a condition variable for when VInit changes, and wait
on that. Also modify other checkers of VInit to use the cond var.

(cherry picked from commit 31bb2e5eb5feb9c8e84d17b49fe70268127cdd43)
Reviewed-on: http://gerrit.openafs.org/2648
Tested-by: Andrew Deason <adeason@sinenomine.net>
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementia.org>
Change-Id: I773ab01095bb0c101a34146ef9bf201946ebcf8b
Reviewed-on: http://gerrit.openafs.org/3887
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/vol/fssync-server.c b/src/vol/fssync-server.c
index d9197894d2abd35acee9cb77746c3b763a5dfa43..51e995258d71c0aa1772620839f3f5dfd90e770d 100644 (file)
--- a/src/vol/fssync-server.c
+++ b/src/vol/fssync-server.c
@@ -262,6 +262,10 @@ FSYNC_sync(void * args)
     int min_vinit = 1;
 #endif /* AFS_DEMAND_ATTACH_FS */
 
+    /* we must not be called before vol package initialization, since we use
+     * vol package mutexes and conds etc */
+    osi_Assert(VInit);
+
     SYNC_getAddr(&state->endpoint, &state->addr);
     SYNC_cleanupSock(state);
 
@@ -278,17 +282,22 @@ FSYNC_sync(void * args)
     Log("Set thread id %d for FSYNC_sync\n", tid);
 #endif /* AFS_PTHREAD_ENV */
 
+    VOL_LOCK;
+
     while (VInit < min_vinit) {
        /* Let somebody else run until all volumes have been preattached
         * (DAFS), or we have started attaching volumes (non-DAFS). This
         * doesn't mean that all volumes have been attached.
         */
 #ifdef AFS_PTHREAD_ENV
-       pthread_yield();
+       VOL_CV_WAIT(&vol_vinit_cond);
 #else /* AFS_PTHREAD_ENV */
        LWP_DispatchProcess();
 #endif /* AFS_PTHREAD_ENV */
     }
+
+    VOL_UNLOCK;
+
     state->fd = SYNC_getSock(&state->endpoint);
     code = SYNC_bindSock(state);
     osi_Assert(!code);

diff --git a/src/vol/volume.c b/src/vol/volume.c
index 421d5de6cafabb7555d97a906264e128e572d001..78aaa21b3c0d1bb714c1ab30675d50871e2499f7 100644 (file)
--- a/src/vol/volume.c
+++ b/src/vol/volume.c
@@ -151,6 +151,7 @@ pthread_mutex_t vol_trans_mutex;
 pthread_cond_t vol_put_volume_cond;
 pthread_cond_t vol_sleep_cond;
 pthread_cond_t vol_init_attach_cond;
+pthread_cond_t vol_vinit_cond;
 int vol_attach_threads = 1;
 #endif /* AFS_PTHREAD_ENV */
 
@@ -539,6 +540,20 @@ VOptDefaults(ProgramType pt, VolumePackageOptions *opts)
     }
 }
 
+/**
+ * Set VInit to a certain value, and signal waiters.
+ *
+ * @param[in] value  the value to set VInit to
+ *
+ * @pre VOL_LOCK held
+ */
+static void
+VSetVInit_r(int value)
+{
+    VInit = value;
+    CV_BROADCAST(&vol_vinit_cond);
+}
+
 int
 VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts)
 {
@@ -567,6 +582,7 @@ VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts)
     CV_INIT(&vol_put_volume_cond, "vol put", CV_DEFAULT, 0);
     CV_INIT(&vol_sleep_cond, "vol sleep", CV_DEFAULT, 0);
     CV_INIT(&vol_init_attach_cond, "vol init attach", CV_DEFAULT, 0);
+    CV_INIT(&vol_vinit_cond, "vol vinit", CV_DEFAULT, 0);
 #else /* AFS_PTHREAD_ENV */
     IOMGR_Initialize();
 #endif /* AFS_PTHREAD_ENV */
@@ -661,7 +677,7 @@ VInitAttachVolumes(ProgramType pt)
        }
     }
     VOL_LOCK;
-    VInit = 2;                 /* Initialized, and all volumes have been attached */
+    VSetVInit_r(2);                    /* Initialized, and all volumes have been attached */
     LWP_NoYieldSignal(VInitAttachVolumes);
     VOL_UNLOCK;
     return 0;
@@ -743,7 +759,7 @@ VInitAttachVolumes(ProgramType pt)
        CV_DESTROY(&params.thread_done_cv);
     }
     VOL_LOCK;
-    VInit = 2;                 /* Initialized, and all volumes have been attached */
+    VSetVInit_r(2);                    /* Initialized, and all volumes have been attached */
     CV_BROADCAST(&vol_init_attach_cond);
     VOL_UNLOCK;
     return 0;
@@ -867,7 +883,7 @@ VInitAttachVolumes(ProgramType pt)
     }
 
     VOL_LOCK;
-    VInit = 2;                 /* Initialized, and all volumes have been attached */
+    VSetVInit_r(2);                    /* Initialized, and all volumes have been attached */
     CV_BROADCAST(&vol_init_attach_cond);
     VOL_UNLOCK;
 
@@ -5551,8 +5567,9 @@ VConnectFS_r(void)
           (programType != fileServer) &&
           (programType != salvager));
     rc = FSYNC_clientInit();
-    if (rc)
-       VInit = 3;
+    if (rc) {
+       VSetVInit_r(3);
+    }
     return rc;
 }
 
@@ -5578,7 +5595,7 @@ VDisconnectFS_r(void)
     osi_Assert((programType != fileServer) &&
           (programType != salvager));
     FSYNC_clientFinis();
-    VInit = 2;
+    VSetVInit_r(2);
 }
 
 /**
@@ -6215,7 +6232,7 @@ VSetDiskUsage_r(void)
         * initialization level indicates that all volumes are attached,
         * which implies that all partitions are initialized. */
 #ifdef AFS_PTHREAD_ENV
-       sleep(10);
+       VOL_CV_WAIT(&vol_vinit_cond);
 #else /* AFS_PTHREAD_ENV */
        IOMGR_Sleep(10);
 #endif /* AFS_PTHREAD_ENV */

diff --git a/src/vol/volume.h b/src/vol/volume.h
index b6a54c9a497026b5eec079ab306bc5867091826f..da961a6284da53803a658609659d4172d3544e46 100644 (file)
--- a/src/vol/volume.h
+++ b/src/vol/volume.h
@@ -67,6 +67,7 @@ extern pthread_mutex_t vol_glock_mutex;
 extern pthread_mutex_t vol_trans_mutex;
 extern pthread_cond_t vol_put_volume_cond;
 extern pthread_cond_t vol_sleep_cond;
+extern pthread_cond_t vol_vinit_cond;
 extern ih_init_params vol_io_params;
 extern int vol_attach_threads;
 #ifdef VOL_LOCK_DEBUG