rx: Process ICMP unreachable errors

When a machine receives ICMP errors, we can detect them in
AFS_RXERRQ_ENV environments. Many of these errors indicate that a
machine is not reachable, so we are guaranteed to not get a response
from them. When we get such an error for a particular peer, mark all
relevant calls with an RX_CALL_DEAD error, since we know we won't get
a response from them. This allows some calls to dead/unreachable hosts
to fail much more quickly.

Do not immediately kill new calls, since obviously the host may have
come back up since then (or the routing/firewall/etc was fixed), but
only calls that were started before the current error was received.

Note that a call doesn't actually notice until the next rxi_CheckCall,
since directly killing each of the relevant calls would be rather
slow. So, we don't notice a dead peer immediately, though we notice
much more quickly than we used to.

Reorganize the error queue processing a little bit to make this easier
to do.

Change-Id: I403540e0677fe2d432901e4ecc19f7f385610b7f
Reviewed-on: http://gerrit.openafs.org/7929
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementix.org>

diff --git a/src/rx/LINUX/rx_knet.c b/src/rx/LINUX/rx_knet.c
index fdf042eda14ae3e078020faa5e95897dbc18eed9..5d2fc4a63fe334e3d446db401d10ada32a2a10c9 100644 (file)
--- a/src/rx/LINUX/rx_knet.c
+++ b/src/rx/LINUX/rx_knet.c
@@ -147,15 +147,7 @@ osi_HandleSocketError(osi_socket so)
 
     memcpy(&addr, offender, sizeof(addr));
 
-# ifdef AFS_ADAPT_PMTU
-    if (err->ee_origin == SO_EE_ORIGIN_ICMP &&
-       err->ee_type == ICMP_DEST_UNREACH &&
-       err->ee_code == ICMP_FRAG_NEEDED) {
-       rxi_SetPeerMtu(NULL, ntohl(addr.sin_addr.s_addr), ntohs(addr.sin_port),
-                      err->ee_info);
-    }
-# endif
-    /* other DEST_UNREACH's and TIME_EXCEEDED should be dealt with too */
+    rxi_ProcessNetError(err, addr.sin_addr.s_addr, addr.sin_port);
 
  out:
     if (controlmsgbuf) {

diff --git a/src/rx/rx.c b/src/rx/rx.c
index 17ab869e39eddf7645d0fa8ee83bc53431526067..b2226a32b088fb8d53ea8848761113f2869f51be 100644 (file)
--- a/src/rx/rx.c
+++ b/src/rx/rx.c
@@ -1626,6 +1626,13 @@ rx_NewCall(struct rx_connection *conn)
     else
        call->mode = RX_MODE_SENDING;
 
+#ifdef AFS_RXERRQ_ENV
+    /* remember how many network errors the peer has when we started, so if
+     * more errors are encountered after the call starts, we know the other endpoint won't be
+     * responding to us */
+    call->neterr_gen = rx_atomic_read(&conn->peer->neterrs);
+#endif
+
     /* remember start time for call in case we have hard dead time limit */
     call->queueTime = queueTime;
     clock_GetTime(&call->startTime);
@@ -2910,6 +2917,51 @@ rxi_SetPeerMtu(struct rx_peer *peer, afs_uint32 host, afs_uint32 port, int mtu)
     MUTEX_EXIT(&rx_peerHashTable_lock);
 }
 
+#ifdef AFS_RXERRQ_ENV
+static void
+rxi_SetPeerDead(afs_uint32 host, afs_uint16 port)
+{
+    int hashIndex = PEER_HASH(host, port);
+    struct rx_peer *peer;
+
+    MUTEX_ENTER(&rx_peerHashTable_lock);
+
+    for (peer = rx_peerHashTable[hashIndex]; peer; peer = peer->next) {
+       if (peer->host == host && peer->port == port) {
+           break;
+       }
+    }
+
+    if (peer) {
+       rx_atomic_inc(&peer->neterrs);
+    }
+
+    MUTEX_EXIT(&rx_peerHashTable_lock);
+}
+
+void
+rxi_ProcessNetError(struct sock_extended_err *err, afs_uint32 addr, afs_uint16 port)
+{
+# ifdef AFS_ADAPT_PMTU
+    if (err->ee_errno == EMSGSIZE && err->ee_info >= 68) {
+       rxi_SetPeerMtu(NULL, addr, port, err->ee_info - RX_IPUDP_SIZE);
+       return;
+    }
+# endif
+    if (err->ee_origin == SO_EE_ORIGIN_ICMP && err->ee_type == ICMP_DEST_UNREACH) {
+       switch (err->ee_code) {
+       case ICMP_NET_UNREACH:
+       case ICMP_HOST_UNREACH:
+       case ICMP_PORT_UNREACH:
+       case ICMP_NET_ANO:
+       case ICMP_HOST_ANO:
+           rxi_SetPeerDead(addr, port);
+           break;
+       }
+    }
+}
+#endif /* AFS_RXERRQ_ENV */
+
 /* Find the peer process represented by the supplied (host,port)
  * combination.  If there is no appropriate active peer structure, a
  * new one will be allocated and initialized
@@ -2933,6 +2985,9 @@ rxi_FindPeer(afs_uint32 host, u_short port,
            pp = rxi_AllocPeer();       /* This bzero's *pp */
            pp->host = host;    /* set here or in InitPeerParams is zero */
            pp->port = port;
+#ifdef AFS_RXERRQ_ENV
+           rx_atomic_set(&pp->neterrs, 0);
+#endif
            MUTEX_INIT(&pp->peer_lock, "peer_lock", MUTEX_DEFAULT, 0);
            queue_Init(&pp->rpcStats);
            pp->next = rx_peerHashTable[hashIndex];
@@ -3200,6 +3255,11 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
         */
 
        if (peer && (peer->refCount > 0)) {
+#ifdef AFS_RXERRQ_ENV
+           if (rx_atomic_read(&peer->neterrs)) {
+               rx_atomic_set(&peer->neterrs, 0);
+           }
+#endif
            MUTEX_ENTER(&peer->peer_lock);
            peer->bytesReceived += np->length;
            MUTEX_EXIT(&peer->peer_lock);
@@ -3254,6 +3314,12 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
         return np;
     }
 
+#ifdef AFS_RXERRQ_ENV
+    if (rx_atomic_read(&conn->peer->neterrs)) {
+       rx_atomic_set(&conn->peer->neterrs, 0);
+    }
+#endif
+
     /* If we're doing statistics, then account for the incoming packet */
     if (rx_stats_active) {
        MUTEX_ENTER(&conn->peer->peer_lock);
@@ -6179,6 +6245,23 @@ rxi_CheckCall(struct rx_call *call)
     int idle_timeout = 0;
     afs_int32  clock_diff = 0;
 
+#ifdef AFS_RXERRQ_ENV
+    int peererrs = rx_atomic_read(&call->conn->peer->neterrs);
+    if (call->neterr_gen < peererrs) {
+       /* we have received network errors since this call started; kill
+        * the call */
+       if (call->state == RX_STATE_ACTIVE) {
+           rxi_CallError(call, RX_CALL_DEAD);
+       }
+       return -1;
+    }
+    if (call->neterr_gen > peererrs) {
+       /* someone has reset the number of peer errors; set the call error gen
+        * so we can detect if more errors are encountered */
+       call->neterr_gen = peererrs;
+    }
+#endif
+
     now = clock_Sec();
 
     /* Large swings in the clock can have a significant impact on

diff --git a/src/rx/rx_call.h b/src/rx/rx_call.h
index e873eb4e1a51661ec87150abfad056748af77674..a1450f15cbd7f2506db7563b6a1c0f0d1151d77d 100644 (file)
--- a/src/rx/rx_call.h
+++ b/src/rx/rx_call.h
@@ -157,6 +157,9 @@ struct rx_call {
 #endif
     afs_uint32 call_id;
 #endif
+#ifdef AFS_RXERRQ_ENV
+    int neterr_gen;
+#endif
 };
 
 /* Only include this once, even when re-loading for kdump. */

diff --git a/src/rx/rx_internal.h b/src/rx/rx_internal.h
index e54d10c11707b365fc1aa6abdfa70ed411b2a1a8..83761d374777b38291c3c1fdc4809b4b09991637 100644 (file)
--- a/src/rx/rx_internal.h
+++ b/src/rx/rx_internal.h
@@ -5,6 +5,14 @@
  * customers of RX belong in rx_private.h, which is installed.
  */
 
+#ifdef AFS_RXERRQ_ENV
+# if defined(AFS_LINUX26_ENV) || defined(AFS_USR_LINUX26_ENV)
+#  include <linux/module.h>
+#  include <linux/types.h>
+#  include <linux/errqueue.h>
+#  include <linux/icmp.h>
+# endif
+#endif
 
 /* Globals that we don't want the world to know about */
 extern rx_atomic_t rx_nWaiting;
@@ -16,6 +24,10 @@ extern rx_atomic_t rx_nWaited;
 extern void rxi_PacketsUnWait(void);
 extern void rxi_SetPeerMtu(struct rx_peer *peer, afs_uint32 host,
                           afs_uint32 port, int mtu);
+#ifdef AFS_RXERRQ_ENV
+extern void rxi_ProcessNetError(struct sock_extended_err *err,
+                                afs_uint32 addr, afs_uint16 port);
+#endif
 extern struct rx_peer *rxi_FindPeer(afs_uint32 host, u_short port,
                                    struct rx_peer *origPeer, int create);
 extern struct rx_packet *rxi_ReceivePacket(struct rx_packet *np,

diff --git a/src/rx/rx_peer.h b/src/rx/rx_peer.h
index 318cbe8c7ccf719a555f98639555d7ee02263653..baf37b0f7aef4ca6ca78ed5d523d72ed4168e746 100644 (file)
--- a/src/rx/rx_peer.h
+++ b/src/rx/rx_peer.h
@@ -63,6 +63,9 @@ struct rx_peer {
     struct rx_queue rpcStats;  /* rpc statistic list */
     int lastReachTime;         /* Last time we verified reachability */
     afs_int32 maxPacketSize;    /* peer packetsize hint */
+#ifdef AFS_RXERRQ_ENV
+    rx_atomic_t neterrs;
+#endif
 };
 
 #endif

diff --git a/src/rx/rx_user.c b/src/rx/rx_user.c
index a0427e4570e75d12c6aeb1f7f516c0a4990c6d44..be74a64ec34278cb6a6d1483070064821c98a420 100644 (file)
--- a/src/rx/rx_user.c
+++ b/src/rx/rx_user.c
@@ -815,13 +815,7 @@ rxi_HandleSocketError(int socket)
     ret = 1;
     err = (struct sock_extended_err *) CMSG_DATA(cmsg);
 
-# ifdef AFS_ADAPT_PMTU
-    if (err->ee_errno == EMSGSIZE && err->ee_info >= 68) {
-       rxi_SetPeerMtu(NULL, addr.sin_addr.s_addr, addr.sin_port,
-                       err->ee_info - RX_IPUDP_SIZE);
-    }
-# endif
-    /* other DEST_UNREACH's and TIME_EXCEEDED should be dealt with too */
+    rxi_ProcessNetError(err, addr.sin_addr.s_addr, addr.sin_port);
 
 out:
     return ret;