afs: do not allow two shutdown sequences in parallel

Often, ‘afsd -shutdown’ is called right after ‘umount’.
Both commands hold the glock before calling ‘afs_shutdown’.
However, one of the functions called by 'afs_shutdown', namely,
‘afs_FlushVCBs’, might drop the glock when the global
'afs_shuttingdown' is still equal to 0. As a result, a scenario
with two shutdown sequences proceeding in parallel is possible.

To fix the problem, the global ‘afs_shuttingdown’ is used as an
enumerated type to make sure that the second thread will not run
‘afs_shutdown’ while the first one is stuck inside ‘afs_FlushVCBs’.

Reviewed-on: http://gerrit.openafs.org/12016
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Chas Williams <3chas3@gmail.com>
Reviewed-by: Michael Meffie <mmeffie@sinenomine.net>
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>
(cherry picked from commit 70fd9bc6dcc79cb25e98cdcfd0f085c4bf4f310a)

Change-Id: I073d1914a7daa858a78305ff154074f2a51a9f5f
Reviewed-on: https://gerrit.openafs.org/12179
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Marcio Brito Barbosa <mbarbosa@sinenomine.net>
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>
Reviewed-by: Stephan Wiesand <stephan.wiesand@desy.de>

diff --git a/src/afs/AIX/osi_file.c b/src/afs/AIX/osi_file.c
index 7ecaaf2376f14a20b0ae6753d739608ed25e363b..ee9011346a883b77433724727b57f48eb9870268 100644 (file)
--- a/src/afs/AIX/osi_file.c
+++ b/src/afs/AIX/osi_file.c
@@ -161,7 +161,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/DARWIN/osi_file.c b/src/afs/DARWIN/osi_file.c
index bc271f02fca67fe55f41377d5b2fdf7e73b9c00b..3448738e6980a93bfe3604c3c2ea6e3ba3c49c27 100644 (file)
--- a/src/afs/DARWIN/osi_file.c
+++ b/src/afs/DARWIN/osi_file.c
@@ -318,7 +318,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/DARWIN/osi_module.c b/src/afs/DARWIN/osi_module.c
index 90292532da4eb763152f8c829736ab7cb5ab33a9..046870a7f2c173e9610c840bd5b4b8753ffa565f 100644 (file)
--- a/src/afs/DARWIN/osi_module.c
+++ b/src/afs/DARWIN/osi_module.c
@@ -115,7 +115,7 @@ afs_modunload(struct kmod_info * kmod_info, void *data)
 {
     if (afs_globalVFS)
        return KERN_FAILURE;
-    if ((afs_initState != 0) || (afs_shuttingdown))
+    if ((afs_initState != 0) || (afs_shuttingdown != AFS_RUNNING))
        return KERN_FAILURE;
 #ifdef AFS_DARWIN80_ENV
     if (vfs_fsremove(afs_vfstable))

diff --git a/src/afs/FBSD/osi_file.c b/src/afs/FBSD/osi_file.c
index 7c695d60ee73a6b0cb1c69e951f5c0b0698d3c03..6f7a03edeab7d041257c5c9e54e2b817f1503f3b 100644 (file)
--- a/src/afs/FBSD/osi_file.c
+++ b/src/afs/FBSD/osi_file.c
@@ -167,7 +167,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/HPUX/osi_file.c b/src/afs/HPUX/osi_file.c
index 59c2594af7c5b1b157095a4013022b4a95bda142..6584b581fbdef0b7236d0c36215d1e2575643d8d 100644 (file)
--- a/src/afs/HPUX/osi_file.c
+++ b/src/afs/HPUX/osi_file.c
@@ -147,7 +147,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/HPUX/osi_vnodeops.c b/src/afs/HPUX/osi_vnodeops.c
index efba180533b099e8396035ef923c2214ffc19d69..56da491d794121a0356d55ea9ce7117d9375a08e 100644 (file)
--- a/src/afs/HPUX/osi_vnodeops.c
+++ b/src/afs/HPUX/osi_vnodeops.c
@@ -297,7 +297,7 @@ afs_inactive(avc, acred)
     struct vnode *vp = AFSTOV(avc);
     ulong_t context;
     lock_t *sv_lock;
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
 
     /*

diff --git a/src/afs/IRIX/osi_file.c b/src/afs/IRIX/osi_file.c
index 86984937d254910c3d20cd1ee3a30a3d0d571e80..497c7dd0988783d27ea9d8cf347d6c965bb9d327 100644 (file)
--- a/src/afs/IRIX/osi_file.c
+++ b/src/afs/IRIX/osi_file.c
@@ -141,7 +141,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX/osi_file.c b/src/afs/LINUX/osi_file.c
index c1e52e3cbad796c5ba1b1fd2368699cda6fbb743..ea0ea5d233860170904e8f5436a64492bb472cf1 100644 (file)
--- a/src/afs/LINUX/osi_file.c
+++ b/src/afs/LINUX/osi_file.c
@@ -219,7 +219,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;
@@ -259,7 +259,7 @@ afs_osi_Write(struct osi_file *afile, afs_int32 offset, void *aptr,
     AFS_STATCNT(osi_Write);
 
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("afs_osi_Write called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX/osi_vnodeops.c b/src/afs/LINUX/osi_vnodeops.c
index 2a83fc0550fd899681774e018b43ab31fa9bd44c..09393f2e2788914a5bfb3f4859b02aa14f28ea0a 100644 (file)
--- a/src/afs/LINUX/osi_vnodeops.c
+++ b/src/afs/LINUX/osi_vnodeops.c
@@ -982,7 +982,7 @@ afs_linux_revalidate(struct dentry *dp)
     cred_t *credp;
     int code;
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     AFS_GLOCK();

diff --git a/src/afs/LINUX24/osi_file.c b/src/afs/LINUX24/osi_file.c
index 612db686d978872f5641826f7dd2cfa591b1a706..92a674e02a88abeede81b467be41a51469319076 100644 (file)
--- a/src/afs/LINUX24/osi_file.c
+++ b/src/afs/LINUX24/osi_file.c
@@ -200,7 +200,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;
@@ -240,7 +240,7 @@ afs_osi_Write(struct osi_file *afile, afs_int32 offset, void *aptr,
     AFS_STATCNT(osi_Write);
 
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("afs_osi_Write called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX24/osi_vnodeops.c b/src/afs/LINUX24/osi_vnodeops.c
index c64c427324f25ea9c46a4f14b236bcdbd1859b43..3ef8d9b05339e1c1f0d6fa0be3c4a883bd06ccba 100644 (file)
--- a/src/afs/LINUX24/osi_vnodeops.c
+++ b/src/afs/LINUX24/osi_vnodeops.c
@@ -890,7 +890,7 @@ afs_linux_revalidate(struct dentry *dp)
     cred_t *credp;
     int code;
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
 #ifdef AFS_LINUX24_ENV

diff --git a/src/afs/NBSD/osi_file.c b/src/afs/NBSD/osi_file.c
index 6d8746a1c88eee0040c7300e2e0e9c55b594e1e6..70bd262048eac38ba3db4a2ddc04c2f5082d5ead 100644 (file)
--- a/src/afs/NBSD/osi_file.c
+++ b/src/afs/NBSD/osi_file.c
@@ -139,7 +139,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr, afs_int32 asize)
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/OBSD/osi_file.c b/src/afs/OBSD/osi_file.c
index 72a43ef9fa335e1496ebe22c2f1cb8daebb8109c..578dde63ad21bfb1d777994ffbc33a5162cf91ca 100644 (file)
--- a/src/afs/OBSD/osi_file.c
+++ b/src/afs/OBSD/osi_file.c
@@ -143,7 +143,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr, afs_int32 asize)
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/SOLARIS/osi_file.c b/src/afs/SOLARIS/osi_file.c
index a0d38a8218ad24645fc4667e0a8383be1bb8a39c..264a812040270a415cdd4bbe2e67b6dcb33f01eb 100644 (file)
--- a/src/afs/SOLARIS/osi_file.c
+++ b/src/afs/SOLARIS/osi_file.c
@@ -372,7 +372,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/SOLARIS/osi_vnodeops.c b/src/afs/SOLARIS/osi_vnodeops.c
index 478d19625625e505dbe6d7ab7020520868f67fea..b31f1a08b116595b65d2416c2b0064d1e5cd345b 100644 (file)
--- a/src/afs/SOLARIS/osi_vnodeops.c
+++ b/src/afs/SOLARIS/osi_vnodeops.c
@@ -1816,7 +1816,7 @@ int
 afs_inactive(struct vcache *avc, afs_ucred_t *acred)
 {
     struct vnode *vp = AFSTOV(avc);
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return 0;
 
     /*

diff --git a/src/afs/UKERNEL/osi_vnodeops.c b/src/afs/UKERNEL/osi_vnodeops.c
index ecfa51a117796802167011bbd057c4cc2c5d4fc9..07642b62bc2c9d6629eb4caf6c983f91804127b0 100644 (file)
--- a/src/afs/UKERNEL/osi_vnodeops.c
+++ b/src/afs/UKERNEL/osi_vnodeops.c
@@ -33,7 +33,7 @@ afs_vrdwr(struct usr_vnode *avc, struct usr_uio *uio, int rw, int io,
 int
 afs_inactive(struct vcache *avc, afs_ucred_t *acred)
 {
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return 0;
 
     usr_assert(avc->vrefCount == 0);

diff --git a/src/afs/VNOPS/afs_vnop_attrs.c b/src/afs/VNOPS/afs_vnop_attrs.c
index 2633db3cb605c336d657b2f8ced76aa26ad24708..8208728a5f6e78e95b9c87165e6bbddd3b8941c3 100644 (file)
--- a/src/afs/VNOPS/afs_vnop_attrs.c
+++ b/src/afs/VNOPS/afs_vnop_attrs.c
@@ -245,7 +245,7 @@ afs_getattr(OSI_VC_DECL(avc), struct vattr *attrs, afs_ucred_t *acred)
     afs_BozonLock(&avc->pvnLock, avc);
 #endif
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     if (!(avc->f.states & CStatd)) {

diff --git a/src/afs/VNOPS/afs_vnop_fid.c b/src/afs/VNOPS/afs_vnop_fid.c
index 8dc51506aae3cacd3d4a6582f11ab57fe6b8dc0e..258f1d6dc1e74971ebc1efa48fc85740c83039d1 100644 (file)
--- a/src/afs/VNOPS/afs_vnop_fid.c
+++ b/src/afs/VNOPS/afs_vnop_fid.c
@@ -84,7 +84,7 @@ afs_fid(OSI_VC_DECL(avc), struct fid **fidpp)
 
     AFS_STATCNT(afs_fid);
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     if (afs_NFSRootOnly && (avc == afs_globalVp))

diff --git a/src/afs/afs.h b/src/afs/afs.h
index 0dbc11bf8f712717d452262aa4266a8a6954161b..f03609ca3c5695f5d90bc87217c9f0de60774af7 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -32,8 +32,12 @@
 /* Upper bound on number of iovecs out uio routines will deal with. */
 #define        AFS_MAXIOVCNT       16
 
-
-extern int afs_shuttingdown;
+enum afs_shutdown_state {
+    AFS_RUNNING = 0,
+    AFS_FLUSHING_CB = 1,
+    AFS_SHUTDOWN = 2,
+};
+extern enum afs_shutdown_state afs_shuttingdown;
 
 /*
  * Macros to uniquely identify the AFS vfs struct

diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 196bc849ce1b17fd76479bd5af7a9b52d78a74f3..ae589a766410bce9c947ddd9b531f01348b01d9d 100644 (file)
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -1365,7 +1365,7 @@ afs_CheckInit(void)
     return code;
 }
 
-int afs_shuttingdown = 0;
+enum afs_shutdown_state afs_shuttingdown = AFS_RUNNING;
 void
 afs_shutdown(void)
 {
@@ -1380,15 +1380,15 @@ afs_shutdown(void)
       return;
     }
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
 
-    /* Give up all of our callbacks if we can. This must be done before setting
-     * afs_shuttingdown, since it calls afs_InitReq, which will fail if
-     * afs_shuttingdown is set. */
+    afs_shuttingdown = AFS_FLUSHING_CB;
+
+    /* Give up all of our callbacks if we can. */
     afs_FlushVCBs(2);
 
-    afs_shuttingdown = 1;
+    afs_shuttingdown = AFS_SHUTDOWN;
 
     if (afs_cold_shutdown)
        afs_warn("afs: COLD ");
@@ -1509,7 +1509,7 @@ afs_shutdown(void)
     memset(&afs_stats_cmfullperf, 0, sizeof(struct afs_stats_CMFullPerf));
     afs_warn(" ALL allocated tables... ");
 
-    afs_shuttingdown = 0;
+    afs_shuttingdown = AFS_RUNNING;
     afs_warn("done\n");
 
     return;                    /* Just kill daemons for now */

diff --git a/src/afs/afs_osi_pag.c b/src/afs/afs_osi_pag.c
index afbb1cf0844a40a0c9cebfe63a43765a13da806a..34ca9b2f5c89957fe4a0a48aff718a1791933aad 100644 (file)
--- a/src/afs/afs_osi_pag.c
+++ b/src/afs/afs_osi_pag.c
@@ -32,7 +32,7 @@
 
 
 /* Imported variables */
-extern int afs_shuttingdown;
+extern enum afs_shutdown_state afs_shuttingdown;
 
 /* Exported variables */
 afs_uint32 pag_epoch;
@@ -456,7 +456,7 @@ afs_InitReq(struct vrequest *av, afs_ucred_t *acred)
 
     AFS_STATCNT(afs_InitReq);
     memset(av, 0, sizeof(*av));
-    if (afs_shuttingdown)
+    if (afs_shuttingdown == AFS_SHUTDOWN)
        return EIO;
 
 #ifdef AFS_LINUX26_ENV
@@ -507,7 +507,7 @@ afs_CreateReq(struct vrequest **avpp, afs_ucred_t *acred)
     int code;
     struct vrequest *treq = NULL;
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown == AFS_SHUTDOWN) {
        return EIO;
     }
     if (!avpp || !acred) {

diff --git a/src/afs/afs_pag_call.c b/src/afs/afs_pag_call.c
index 88c0d8f8d6f6c34e3d09b7559248f62e4fe6f716..a1f40b5809393512eefbf1e59f3121140294722c 100644 (file)
--- a/src/afs/afs_pag_call.c
+++ b/src/afs/afs_pag_call.c
@@ -31,7 +31,7 @@
 
 afs_int32 afs_termState = 0;
 afs_int32 afs_gcpags = AFS_GCPAGS;
-int afs_shuttingdown = 0;
+enum afs_shutdown_state afs_shuttingdown = AFS_RUNNING;
 int afs_cold_shutdown = 0;
 int afs_resourceinit_flag = 0;
 afs_int32 afs_nfs_server_addr;
@@ -163,9 +163,9 @@ afspag_Init(afs_int32 nfs_server_addr)
 void
 afspag_Shutdown(void)
 {
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
-    afs_shuttingdown = 1;
+    afs_shuttingdown = AFS_SHUTDOWN;
     afs_termState = AFSOP_STOP_RXCALLBACK;
     rx_WakeupServerProcs();
     while (afs_termState == AFSOP_STOP_RXCALLBACK)

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index f3b78803bd1b58553bc918b5b81f2f202ef6f9f3..d751a564ce8623da5edf80176b6f9d62a0c99ea2 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -234,7 +234,7 @@ afs_FlushVCache(struct vcache *avc, int *slept)
     else
        osi_dnlc_purgevp(avc);
 
-    if (!afs_shuttingdown)
+    if (afs_shuttingdown == AFS_RUNNING)
        afs_QueueVCB(avc, slept);
 
     /*

diff --git a/src/rx/SOLARIS/rx_knet.c b/src/rx/SOLARIS/rx_knet.c
index 19df875c5b0ec8b1aad42248f8a4d360b1efc10e..320a382725e3d5a7f520287537298f416f357fa2 100644 (file)
--- a/src/rx/SOLARIS/rx_knet.c
+++ b/src/rx/SOLARIS/rx_knet.c
@@ -765,7 +765,7 @@ osi_NetIfPoller()
     if (li)
         (void) ldi_ident_release(li);
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown != AFS_RUNNING) {
        /* do not schedule to run again if we're shutting down */
        return;
     }