vos: Improve release recovery on timed-out trans

When a portion of 'vos release' takes long enough that a transaction
on one or more RO sites times out, it retries to release to those
sites. However, it does not take into account the situation where the
transaction on the original RO clone has also timed out, which it
usually will have if we took a long time to get to the ForwardMulti
stage for any reason.

Add a check to see if the clone RO transaction has timed out, and if
so, recreate the transaction on it. Check if the volume appears to
have changed at all, and if it has, bail out.

Reviewed-on: http://gerrit.openafs.org/2726
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementia.org>
(cherry picked from commit d8e382df39aa5a92950a1219002a85e5457d8a97)

Change-Id: I19541800fa7d1ce1c5922b9338b4b8370151fd93
Reviewed-on: http://gerrit.openafs.org/3781
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/volser/vsprocs.c b/src/volser/vsprocs.c
index 66dc4cd621bb423845773ab46a927bca2f7c7eda..662ccf188cc3eb89d8ba5b5a5268bb23f4f3bb08 100644 (file)
--- a/src/volser/vsprocs.c
+++ b/src/volser/vsprocs.c
@@ -3478,6 +3478,72 @@ SimulateForwardMultiple(struct rx_connection *fromconn, afs_int32 fromtid,
     return 0;
 }
 
+/**
+ * Check if a trans has timed out, and recreate it if necessary.
+ *
+ * @param[in] aconn  RX connection to the relevant server
+ * @param[inout] atid  Transaction ID to check; if we recreated the trans,
+ *                     contains the new trans ID on success
+ * @param[in] apart  Partition for the transaction
+ * @param[in] astat  The status of the original transaction
+ *
+ * @return operation status
+ *  @retval 0 existing transaction is still valid, or we managed to recreate
+ *            the trans successfully
+ *  @retval nonzero Fatal error; bail out
+ */
+static int
+CheckTrans(struct rx_connection *aconn, afs_int32 *atid, afs_int32 apart,
+           struct volser_status *astat)
+{
+    struct volser_status new_status;
+    afs_int32 code;
+
+    memset(&new_status, 0, sizeof(new_status));
+    code = AFSVolGetStatus(aconn, *atid, &new_status);
+    if (code) {
+       if (code == ENOENT) {
+           *atid = 0;
+           VPRINT1("Old transaction on cloned volume %lu timed out, "
+                   "restarting transaction\n", (long unsigned) astat->volID);
+           code = AFSVolTransCreate_retry(aconn, astat->volID, apart,
+                                          ITBusy, atid);
+           if (code) {
+               PrintError("Failed to recreate cloned RO volume transaction\n",
+                          code);
+               return 1;
+           }
+
+           memset(&new_status, 0, sizeof(new_status));
+           code = AFSVolGetStatus(aconn, *atid, &new_status);
+           if (code) {
+               PrintError("Failed to get status on recreated transaction\n",
+                          code);
+               return 1;
+           }
+
+           if (memcmp(&new_status, astat, sizeof(new_status)) != 0) {
+               PrintError("Recreated transaction on cloned RO volume, but "
+                          "the volume has changed!\n", 0);
+               return 1;
+           }
+       } else {
+           PrintError("Unable to get status of current cloned RO transaction\n",
+                      code);
+           return 1;
+       }
+    } else {
+       if (memcmp(&new_status, astat, sizeof(new_status)) != 0) {
+           /* sanity check */
+           PrintError("Internal error: current GetStatus does not match "
+                      "original GetStatus?\n", 0);
+           return 1;
+       }
+    }
+
+    return 0;
+}
+
 
 /* UV_ReleaseVolume()
  *    Release volume <afromvol> on <afromserver> <afrompart> to all
@@ -3531,6 +3597,7 @@ UV_ReleaseVolume(afs_uint32 afromvol, afs_uint32 afromserver,
     struct volser_status volstatus;
     char hoststr[16];
     afs_int32 origflags[NMAXNSERVERS];
+    struct volser_status orig_status;
 
     memset(remembertime, 0, sizeof(remembertime));
     memset(&results, 0, sizeof(results));
@@ -3836,6 +3903,10 @@ UV_ReleaseVolume(afs_uint32 afromvol, afs_uint32 afromserver,
     VPRINT1("Starting transaction on cloned volume %u...", cloneVolId);
     code =
        AFSVolTransCreate_retry(fromconn, cloneVolId, afrompart, ITBusy, &fromtid);
+    if (!code) {
+       memset(&orig_status, 0, sizeof(orig_status));
+       code = AFSVolGetStatus(fromconn, fromtid, &orig_status);
+    }
     if (!fullrelease && code)
        ONERROR(VOLSERNOVOL, afromvol,
                "Old clone is inaccessible. Try vos release -f %u.\n");
@@ -3932,6 +4003,12 @@ UV_ReleaseVolume(afs_uint32 afromvol, afs_uint32 afromserver,
        if (!volcount)
            continue;
 
+       code = CheckTrans(fromconn, &fromtid, afrompart, &orig_status);
+       if (code) {
+           code = ENOENT;
+           goto rfail;
+       }
+
        if (verbose) {
            fprintf(STDOUT, "Starting ForwardMulti from %lu to %u on %s",
                    (unsigned long)cloneVolId, entry.volumeId[ROVOL],