afs: do not allow two shutdown sequences in parallel

Often, ‘afsd -shutdown’ is called right after ‘umount’.
Both commands hold the glock before calling ‘afs_shutdown’.
However, one of the functions called by 'afs_shutdown', namely,
‘afs_FlushVCBs’, might drop the glock when the global
'afs_shuttingdown' is still equal to 0. As a result, a scenario
with two shutdown sequences proceeding in parallel is possible.

To fix the problem, the global ‘afs_shuttingdown’ is used as an
enumerated type to make sure that the second thread will not run
‘afs_shutdown’ while the first one is stuck inside ‘afs_FlushVCBs’.

Change-Id: Iffa89d82278b0df5fb90fc35608af66d8e8db29e
Reviewed-on: http://gerrit.openafs.org/12016
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Chas Williams <3chas3@gmail.com>
Reviewed-by: Michael Meffie <mmeffie@sinenomine.net>
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>

diff --git a/src/afs/AIX/osi_file.c b/src/afs/AIX/osi_file.c
index 9e70da4e7c6ec8cae5911b4d60a9a126eb3983f5..4bd33711e1a2c1059fe218968671cd0641b9ed8d 100644 (file)
--- a/src/afs/AIX/osi_file.c
+++ b/src/afs/AIX/osi_file.c
@@ -161,7 +161,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/DARWIN/osi_file.c b/src/afs/DARWIN/osi_file.c
index b61991010c107c511c4cb0d84e363840dd1958e8..a83dbf2e5787cd3bba877d571ec49a4c33d547bf 100644 (file)
--- a/src/afs/DARWIN/osi_file.c
+++ b/src/afs/DARWIN/osi_file.c
@@ -318,7 +318,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/DARWIN/osi_module.c b/src/afs/DARWIN/osi_module.c
index 90292532da4eb763152f8c829736ab7cb5ab33a9..046870a7f2c173e9610c840bd5b4b8753ffa565f 100644 (file)
--- a/src/afs/DARWIN/osi_module.c
+++ b/src/afs/DARWIN/osi_module.c
@@ -115,7 +115,7 @@ afs_modunload(struct kmod_info * kmod_info, void *data)
 {
     if (afs_globalVFS)
        return KERN_FAILURE;
-    if ((afs_initState != 0) || (afs_shuttingdown))
+    if ((afs_initState != 0) || (afs_shuttingdown != AFS_RUNNING))
        return KERN_FAILURE;
 #ifdef AFS_DARWIN80_ENV
     if (vfs_fsremove(afs_vfstable))

diff --git a/src/afs/FBSD/osi_file.c b/src/afs/FBSD/osi_file.c
index 096e4721ae9f9a4d4053b464d4c2ea302055f205..1cfc5c83c14fde26697fd8adddc19dd0c60598e3 100644 (file)
--- a/src/afs/FBSD/osi_file.c
+++ b/src/afs/FBSD/osi_file.c
@@ -171,7 +171,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/HPUX/osi_file.c b/src/afs/HPUX/osi_file.c
index 44ed27140a562ef8bc3d0b72b5a576773a35e1e7..a562fd46d01d02491a6a596741f660b579cf74c9 100644 (file)
--- a/src/afs/HPUX/osi_file.c
+++ b/src/afs/HPUX/osi_file.c
@@ -147,7 +147,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/HPUX/osi_vnodeops.c b/src/afs/HPUX/osi_vnodeops.c
index 839d76e2c7d47fd4b19edea601eadc7aa4bcf29c..bce2441da47e23b5fc7fdc53ca71b1a161a75ea0 100644 (file)
--- a/src/afs/HPUX/osi_vnodeops.c
+++ b/src/afs/HPUX/osi_vnodeops.c
@@ -297,7 +297,7 @@ afs_inactive(avc, acred)
     struct vnode *vp = AFSTOV(avc);
     ulong_t context;
     lock_t *sv_lock;
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
 
     /*

diff --git a/src/afs/IRIX/osi_file.c b/src/afs/IRIX/osi_file.c
index bdb458f14714f1bd934992100ce828a0de64715b..92a366cc880c4f529d1bcd8cf46e763f06577870 100644 (file)
--- a/src/afs/IRIX/osi_file.c
+++ b/src/afs/IRIX/osi_file.c
@@ -141,7 +141,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX/osi_file.c b/src/afs/LINUX/osi_file.c
index c0507152453f7deebb8ae7ff9b9c6ade4d3209dc..c125c1ca7b51fd71e1cf7dc6dc287f00edaf978a 100644 (file)
--- a/src/afs/LINUX/osi_file.c
+++ b/src/afs/LINUX/osi_file.c
@@ -218,7 +218,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;
@@ -258,7 +258,7 @@ afs_osi_Write(struct osi_file *afile, afs_int32 offset, void *aptr,
     AFS_STATCNT(osi_Write);
 
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("afs_osi_Write called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX/osi_vnodeops.c b/src/afs/LINUX/osi_vnodeops.c
index b2e6dd70747286edb06e1222d9dc49dccb6fad98..6b49b5e979c3da686b82ae1e9e7354f660a3dfb5 100644 (file)
--- a/src/afs/LINUX/osi_vnodeops.c
+++ b/src/afs/LINUX/osi_vnodeops.c
@@ -945,7 +945,7 @@ afs_linux_revalidate(struct dentry *dp)
     cred_t *credp;
     int code;
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     AFS_GLOCK();

diff --git a/src/afs/NBSD/osi_file.c b/src/afs/NBSD/osi_file.c
index dd7811a8dc02b982246e62d14dc21ad3149bf7bf..dfacb695d9d95fd7b589c698d58ed51ef59191a2 100644 (file)
--- a/src/afs/NBSD/osi_file.c
+++ b/src/afs/NBSD/osi_file.c
@@ -154,7 +154,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr, afs_int32 asize)
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/OBSD/osi_file.c b/src/afs/OBSD/osi_file.c
index c19d86b7b83da7119b8511686b3c7f6d940de545..792914142dd26463c275c1aa2921bdeb20c65411 100644 (file)
--- a/src/afs/OBSD/osi_file.c
+++ b/src/afs/OBSD/osi_file.c
@@ -143,7 +143,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr, afs_int32 asize)
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/SOLARIS/osi_file.c b/src/afs/SOLARIS/osi_file.c
index b4380f372df8796a330697c126648b1c4955a92b..a3387eced2c2991154918c3b7d4466357a6fd9b4 100644 (file)
--- a/src/afs/SOLARIS/osi_file.c
+++ b/src/afs/SOLARIS/osi_file.c
@@ -354,7 +354,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/SOLARIS/osi_vnodeops.c b/src/afs/SOLARIS/osi_vnodeops.c
index d19bcf65522bf31352f6b7d9e4abab9c5d9f97b3..69dd98595a5a32b6bd9986ee71bbc75a9b765bb8 100644 (file)
--- a/src/afs/SOLARIS/osi_vnodeops.c
+++ b/src/afs/SOLARIS/osi_vnodeops.c
@@ -1659,7 +1659,7 @@ int
 afs_inactive(struct vcache *avc, afs_ucred_t *acred)
 {
     struct vnode *vp = AFSTOV(avc);
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return 0;
 
     /*

diff --git a/src/afs/UKERNEL/osi_vnodeops.c b/src/afs/UKERNEL/osi_vnodeops.c
index 290d46d8696de2c8943b6c86867e870339a31a19..c78fe276b19e0f604b0fd4dc66fbfeaca2148845 100644 (file)
--- a/src/afs/UKERNEL/osi_vnodeops.c
+++ b/src/afs/UKERNEL/osi_vnodeops.c
@@ -33,7 +33,7 @@ afs_vrdwr(struct usr_vnode *avc, struct usr_uio *uio, int rw, int io,
 int
 afs_inactive(struct vcache *avc, afs_ucred_t *acred)
 {
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return 0;
 
     usr_assert(avc->vrefCount == 0);

diff --git a/src/afs/VNOPS/afs_vnop_attrs.c b/src/afs/VNOPS/afs_vnop_attrs.c
index 7e580c5a0525f71d4272707f32f656f3992866ac..cc1fd32193e8a67037562a8a2f9f7dbf0d9c38f8 100644 (file)
--- a/src/afs/VNOPS/afs_vnop_attrs.c
+++ b/src/afs/VNOPS/afs_vnop_attrs.c
@@ -241,7 +241,7 @@ afs_getattr(OSI_VC_DECL(avc), struct vattr *attrs, afs_ucred_t *acred)
 
     AFS_DISCON_LOCK();
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown != AFS_RUNNING) {
        AFS_DISCON_UNLOCK();
        return EIO;
     }

diff --git a/src/afs/VNOPS/afs_vnop_fid.c b/src/afs/VNOPS/afs_vnop_fid.c
index cdd147eb65b0c5c8e13d72df82197c9f5578f863..d10bcacb26a01cbe6574fc2823692344acc45aaf 100644 (file)
--- a/src/afs/VNOPS/afs_vnop_fid.c
+++ b/src/afs/VNOPS/afs_vnop_fid.c
@@ -83,7 +83,7 @@ afs_fid(OSI_VC_DECL(avc), struct fid **fidpp)
 
     AFS_STATCNT(afs_fid);
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     if (afs_NFSRootOnly && (avc == afs_globalVp))

diff --git a/src/afs/afs.h b/src/afs/afs.h
index a9877d536d6f88665bba1c0f08e34d471943bd06..f470947e1ebe02734a7148b1f7ecfcf58903d2b3 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -35,8 +35,12 @@
 /* Upper bound on number of iovecs out uio routines will deal with. */
 #define        AFS_MAXIOVCNT       16
 
-
-extern int afs_shuttingdown;
+enum afs_shutdown_state {
+    AFS_RUNNING = 0,
+    AFS_FLUSHING_CB = 1,
+    AFS_SHUTDOWN = 2,
+};
+extern enum afs_shutdown_state afs_shuttingdown;
 
 /*
  * Macros to uniquely identify the AFS vfs struct

diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 554da85eb792560d42ac8d5aa151ec2b798238f9..3ab1527b048898af168201865cb2e3f5c1359ef1 100644 (file)
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -1366,7 +1366,7 @@ afs_CheckInit(void)
     return code;
 }
 
-int afs_shuttingdown = 0;
+enum afs_shutdown_state afs_shuttingdown = AFS_RUNNING;
 void
 afs_shutdown(void)
 {
@@ -1381,15 +1381,15 @@ afs_shutdown(void)
       return;
     }
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
 
-    /* Give up all of our callbacks if we can. This must be done before setting
-     * afs_shuttingdown, since it calls afs_InitReq, which will fail if
-     * afs_shuttingdown is set. */
+    afs_shuttingdown = AFS_FLUSHING_CB;
+
+    /* Give up all of our callbacks if we can. */
     afs_FlushVCBs(2);
 
-    afs_shuttingdown = 1;
+    afs_shuttingdown = AFS_SHUTDOWN;
 
     if (afs_cold_shutdown)
        afs_warn("afs: COLD ");
@@ -1510,7 +1510,7 @@ afs_shutdown(void)
     memset(&afs_stats_cmfullperf, 0, sizeof(struct afs_stats_CMFullPerf));
     afs_warn(" ALL allocated tables... ");
 
-    afs_shuttingdown = 0;
+    afs_shuttingdown = AFS_RUNNING;
     afs_warn("done\n");
 
     return;                    /* Just kill daemons for now */

diff --git a/src/afs/afs_osi_pag.c b/src/afs/afs_osi_pag.c
index 309dcaa31e1f34799b6d4bcec327faf17fbb211f..9e868025144261b5ee27cdf7835dbbffc3893053 100644 (file)
--- a/src/afs/afs_osi_pag.c
+++ b/src/afs/afs_osi_pag.c
@@ -32,7 +32,7 @@
 
 
 /* Imported variables */
-extern int afs_shuttingdown;
+extern enum afs_shutdown_state afs_shuttingdown;
 
 /* Exported variables */
 afs_uint32 pag_epoch;
@@ -455,7 +455,7 @@ afs_InitReq(struct vrequest *av, afs_ucred_t *acred)
 
     AFS_STATCNT(afs_InitReq);
     memset(av, 0, sizeof(*av));
-    if (afs_shuttingdown)
+    if (afs_shuttingdown == AFS_SHUTDOWN)
        return EIO;
 
 #ifdef AFS_LINUX26_ENV
@@ -506,7 +506,7 @@ afs_CreateReq(struct vrequest **avpp, afs_ucred_t *acred)
     int code;
     struct vrequest *treq = NULL;
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown == AFS_SHUTDOWN) {
        return EIO;
     }
     if (!avpp || !acred) {

diff --git a/src/afs/afs_pag_call.c b/src/afs/afs_pag_call.c
index e38cf93cd846a945cf05895d36ac4dcfbe4bff28..d7a3113e7b0ef8cea15a0036c2f45ab9133d61ed 100644 (file)
--- a/src/afs/afs_pag_call.c
+++ b/src/afs/afs_pag_call.c
@@ -31,7 +31,7 @@
 
 afs_int32 afs_termState = 0;
 afs_int32 afs_gcpags = AFS_GCPAGS;
-int afs_shuttingdown = 0;
+enum afs_shutdown_state afs_shuttingdown = AFS_RUNNING;
 int afs_cold_shutdown = 0;
 int afs_resourceinit_flag = 0;
 afs_int32 afs_nfs_server_addr;
@@ -158,9 +158,9 @@ afspag_Init(afs_int32 nfs_server_addr)
 void
 afspag_Shutdown(void)
 {
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
-    afs_shuttingdown = 1;
+    afs_shuttingdown = AFS_SHUTDOWN;
     afs_termState = AFSOP_STOP_RXCALLBACK;
     rx_WakeupServerProcs();
     while (afs_termState == AFSOP_STOP_RXCALLBACK)

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index 87ec2fc6023cb309375a572678f9f6612395f8bf..c2b0d5a68627547125016de5cd24bd314e6acb60 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -261,7 +261,7 @@ afs_FlushVCache(struct vcache *avc, int *slept)
      * via which someone could try to use the vcache. It is okay to drop
      * afs_xvcache at this point (if *slept is set). */
 
-    if (!afs_shuttingdown)
+    if (afs_shuttingdown == AFS_RUNNING)
        afs_QueueVCB(avc, slept);
 
     /*

diff --git a/src/rx/SOLARIS/rx_knet.c b/src/rx/SOLARIS/rx_knet.c
index 101356b3963e17d96cd09f3898f63d57f7edc6ac..fe85e63b81509c58c01f52b90f5752d9e9bd1eb4 100644 (file)
--- a/src/rx/SOLARIS/rx_knet.c
+++ b/src/rx/SOLARIS/rx_knet.c
@@ -758,7 +758,7 @@ osi_NetIfPoller()
     if (li)
         (void) ldi_ident_release(li);
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown != AFS_RUNNING) {
        /* do not schedule to run again if we're shutting down */
        return;
     }