afs: WriteThroughDSlots: Avoid write error panic

Currently, afs_WriteThroughDSlots panics if our call to
afs_WriteDCache fails. Since afs_WriteThroughDSlots is called every
minute by a background daemon, this means that if our cache fs becomes
inaccessible (by being forced read-only, or for any other reason), we
are virtually guaranteed to panic relatively quickly.

To try to avoid this at least for some cases, change
afs_WriteThroughDSlots to return an error to our caller when we
encounter such an error. For our background task, we can just ignore
the error and retry the writes on a future iteration. During shutdown,
we still panic if we encounter an error, to try to avoid silently
allowing a corrupt cache to be used on subsequent boots.

Reviewed-on: https://gerrit.openafs.org/13047
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>
Reviewed-by: Michael Meffie <mmeffie@sinenomine.net>
Tested-by: BuildBot <buildbot@rampaginggeek.com>
(cherry picked from commit 46d5695a383b2b993fdd598b770f4e3c0e1a41f3)

Change-Id: Iccbec550c00ac725c0033b075ac3cd0b4e1c78c2
Reviewed-on: https://gerrit.openafs.org/13073
Reviewed-by: Mark Vitale <mvitale@sinenomine.net>
Reviewed-by: Michael Meffie <mmeffie@sinenomine.net>
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>
Tested-by: BuildBot <buildbot@rampaginggeek.com>

diff --git a/src/afs/afs_daemons.c b/src/afs/afs_daemons.c
index b283f778cda7fef3efcac40007579e7a8666bfe7..3d42b3c8d62175956d1cf767847fa8309fa33c19 100644 (file)
--- a/src/afs/afs_daemons.c
+++ b/src/afs/afs_daemons.c
@@ -189,7 +189,7 @@ afs_Daemon(void)
        if (last1MinCheck + 60 < now) {
            /* things to do every minute */
            DFlush();           /* write out dir buffers */
-           afs_WriteThroughDSlots();   /* write through cacheinfo entries */
+           (void)afs_WriteThroughDSlots();     /* write through cacheinfo entries */
            ObtainWriteLock(&afs_xvcache, 736);
            afs_FlushReclaimedVcaches();
            ReleaseWriteLock(&afs_xvcache);

diff --git a/src/afs/afs_dcache.c b/src/afs/afs_dcache.c
index 6ca9c1fa1d7ac34342a7371e71ef1c007cc5eff0..f71f459c4857ca19a92d9db31d1218544fadc8dc 100644 (file)
--- a/src/afs/afs_dcache.c
+++ b/src/afs/afs_dcache.c
@@ -2663,11 +2663,12 @@ afs_GetDCache(struct vcache *avc, afs_size_t abyte,
  * Environment:
  *     The afs_xdcache is write-locked through this whole affair.
  */
-void
+int
 afs_WriteThroughDSlots(void)
 {
     struct dcache *tdc;
     afs_int32 i, touchedit = 0;
+    int code = 0;
 
     struct afs_q DirtyQ, *tq;
 
@@ -2703,7 +2704,7 @@ afs_WriteThroughDSlots(void)
 
 #define DQTODC(q)      ((struct dcache *)(((char *) (q)) - sizeof(struct afs_q)))
 
-    for (tq = DirtyQ.prev; tq != &DirtyQ; tq = QPrev(tq)) {
+    for (tq = DirtyQ.prev; tq != &DirtyQ && code == 0; tq = QPrev(tq)) {
        tdc = DQTODC(tq);
        if (tdc->dflags & DFEntryMod) {
            int wrLock;
@@ -2714,9 +2715,15 @@ afs_WriteThroughDSlots(void)
            if (wrLock && (tdc->dflags & DFEntryMod)) {
                tdc->dflags &= ~DFEntryMod;
                ObtainWriteLock(&afs_xdcache, 620);
-               osi_Assert(afs_WriteDCache(tdc, 1) == 0);
+               code = afs_WriteDCache(tdc, 1);
                ReleaseWriteLock(&afs_xdcache);
-               touchedit = 1;
+                if (code) {
+                    /* We didn't successfully write out the dslot; make sure we
+                     * try again later */
+                    tdc->dflags |= DFEntryMod;
+                } else {
+                    touchedit = 1;
+                }
            }
            if (wrLock)
                ReleaseWriteLock(&tdc->lock);
@@ -2725,6 +2732,10 @@ afs_WriteThroughDSlots(void)
        afs_PutDCache(tdc);
     }
 
+    if (code) {
+        return code;
+    }
+
     ObtainWriteLock(&afs_xdcache, 617);
     if (!touchedit && (cacheDiskType != AFS_FCACHE_TYPE_MEM)) {
        /* Touch the file to make sure that the mtime on the file is kept
@@ -2737,6 +2748,7 @@ afs_WriteThroughDSlots(void)
        afs_osi_Write(afs_cacheInodep, 0, &theader, sizeof(theader));
     }
     ReleaseWriteLock(&afs_xdcache);
+    return 0;
 }
 
 /*

diff --git a/src/afs/afs_init.c b/src/afs/afs_init.c
index 6262f928a34d367f0a89d0b20edd8accba228c86..476d5bdfc92afdee80be3529dbb279827df3fa8a 100644 (file)
--- a/src/afs/afs_init.c
+++ b/src/afs/afs_init.c
@@ -693,7 +693,7 @@ void
 shutdown_cache(void)
 {
     AFS_STATCNT(shutdown_cache);
-    afs_WriteThroughDSlots();
+    osi_Assert(afs_WriteThroughDSlots() == 0);
     if (1/*afs_cold_shutdown*/) {
        afs_cacheinit_flag = 0;
        shutdown_dcache();

diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h
index 3c503d682d4810a0cca17a435c6cbc3b93be2d9b..e51f62b9170725cb3f7ed93ae539d7520ba4058f 100644 (file)
--- a/src/afs/afs_prototypes.h
+++ b/src/afs/afs_prototypes.h
@@ -275,7 +275,7 @@ extern void afs_TryToSmush(struct vcache *avc,
                           afs_ucred_t *acred, int sync);
 extern void updateV2DC(int lockVc, struct vcache *v, struct dcache *d,
                       int src);
-extern void afs_WriteThroughDSlots(void);
+extern int afs_WriteThroughDSlots(void);
 extern struct dcache *afs_UFSGetDSlot(afs_int32 aslot, dslot_state type);
 extern int afs_WriteDCache(struct dcache *adc, int atime);
 extern int afs_wakeup(struct vcache *avc);