From: Dale Ghent Date: Fri, 5 Oct 2007 03:54:08 +0000 (+0000) Subject: STABLE14-solaris10-network-updates-20071004 X-Git-Tag: openafs-stable-1_4_5-pre1~9 X-Git-Url: https://git.michaelhowe.org/gitweb/?a=commitdiff_plain;h=19ef6a96951e1edb995f741ea2242f2f4251f439;p=packages%2Fo%2Fopenafs.git STABLE14-solaris10-network-updates-20071004 FIXES 72759 This patch removes use of the non-Public ILL structures by OpenAFS to gather network interface information in order to make RX packet size and server locality decisions. (cherry picked from commit 4ac01ec1ed2ce588185c055acfa3d73095a5b444) --- diff --git a/src/afs/SOLARIS/osi_machdep.h b/src/afs/SOLARIS/osi_machdep.h index 744dbab0b..2974f3188 100644 --- a/src/afs/SOLARIS/osi_machdep.h +++ b/src/afs/SOLARIS/osi_machdep.h @@ -104,4 +104,24 @@ extern kmutex_t afs_global_lock; #define AfsLargeFileSize(pos, off) ( ((offset_t)(pos)+(offset_t)(off) > (offset_t)0x7fffffff)?1:0) #endif +#if defined(AFS_SUN510_ENV) +#include +#include +extern ddi_taskq_t *afs_taskq; +extern krwlock_t afsifinfo_lock; + +/* Global interface info struct */ +struct afs_ifinfo { + char ifname[LIFNAMSIZ]; + ipaddr_t ipaddr; + ipaddr_t netmask; + uint_t mtu; + uint64_t flags; + int metric; + ipaddr_t dstaddr; +}; + +extern struct afs_ifinfo afsifinfo[ADDRSPERSITE]; +#endif + #endif /* _OSI_MACHDEP_H_ */ diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c index 3911b8feb..f042c8778 100644 --- a/src/afs/afs_call.c +++ b/src/afs/afs_call.c @@ -29,7 +29,10 @@ RCSID #ifdef AFS_LINUX22_ENV #include "h/smp_lock.h" #endif - +#ifdef AFS_SUN510_ENV +#include "h/ksynch.h" +#include "h/sunddi.h" +#endif #if defined(AFS_SUN5_ENV) || defined(AFS_AIX_ENV) || defined(AFS_SGI_ENV) || defined(AFS_HPUX_ENV) #define AFS_MINBUFFERS 100 @@ -84,6 +87,11 @@ thread_t afs_global_owner; simple_lock_data afs_global_lock; #endif +#ifdef AFS_SUN510_ENV +ddi_taskq_t *afs_taskq; +krwlock_t afsifinfo_lock; +#endif + afs_int32 afs_initState = 0; afs_int32 afs_termState = 0; afs_int32 afs_setTime = 0; @@ -121,6 +129,16 @@ afs_InitSetup(int preallocs) if (afs_InitSetup_done) return EAGAIN; +#ifdef AFS_SUN510_ENV + /* Initialize a RW lock for the ifinfo global array */ + rw_init(&afsifinfo_lock, NULL, RW_DRIVER, NULL); + + /* Create a taskq */ + afs_taskq = ddi_taskq_create(NULL, "afs_taskq", 2, TASKQ_DEFAULTPRI, 0); + + osi_StartNetIfPoller(); +#endif + #ifndef AFS_NOSTATS /* * Set up all the AFS statistics variables. This should be done diff --git a/src/afs/afs_server.c b/src/afs/afs_server.c index 9c837b74b..6b0f15545 100644 --- a/src/afs/afs_server.c +++ b/src/afs/afs_server.c @@ -1273,23 +1273,74 @@ static int afs_SetServerPrefs(struct srvAddr *sa) { #else /* AFS_USERSPACE_IP_ADDR */ #if defined(AFS_SUN5_ENV) #ifdef AFS_SUN510_ENV - ill_walk_context_t ctx; + int i = 0; #else extern struct ill_s *ill_g_headp; long *addr = (long *)ill_g_headp; -#endif ill_t *ill; ipif_t *ipif; +#endif int subnet, subnetmask, net, netmask; if (sa) sa->sa_iprank = 0; #ifdef AFS_SUN510_ENV - for (ill = ILL_START_WALK_ALL(&ctx) ; ill ; ill = ill_next(&ctx, ill)) { + rw_enter(&afsifinfo_lock, RW_READER); + + for (i = 0; (afsifinfo[i].ipaddr != NULL) && (i < ADDRSPERSITE); i++) { + + if (IN_CLASSA(afsifinfo[i].ipaddr)) { + netmask = IN_CLASSA_NET; + } else if (IN_CLASSB(afsifinfo[i].ipaddr)) { + netmask = IN_CLASSB_NET; + } else if (IN_CLASSC(afsifinfo[i].ipaddr)) { + netmask = IN_CLASSC_NET; + } else { + netmask = 0; + } + net = afsifinfo[i].ipaddr & netmask; + +#ifdef notdef + if (!s) { + if (afsifinfo[i].ipaddr != 0x7f000001) { /* ignore loopback */ + *cnt += 1; + if (*cnt > 16) + return; + *addrp++ = afsifinfo[i].ipaddr; + } + } else +#endif /* notdef */ + { + /* XXXXXX Do the individual ip ranking below XXXXX */ + if ((sa->sa_ip & netmask) == net) { + if ((sa->sa_ip & subnetmask) == subnet) { + if (afsifinfo[i].ipaddr == sa->sa_ip) { /* ie, ME! */ + sa->sa_iprank = TOPR; + } else { + sa->sa_iprank = HI + afsifinfo[i].metric; /* case #2 */ + } + } else { + sa->sa_iprank = MED + afsifinfo[i].metric; /* case #3 */ + } + } else { + sa->sa_iprank = LO + afsifinfo[i].metric; /* case #4 */ + } + /* check for case #5 -- point-to-point link */ + if ((afsifinfo[i].flags & IFF_POINTOPOINT) + && (afsifinfo[i].dstaddr == sa->sa_ip)) { + + if (afsifinfo[i].metric >= (MAXDEFRANK - MED) / PPWEIGHT) + sa->sa_iprank = MAXDEFRANK; + else + sa->sa_iprank = MED + (PPWEIGHT << afsifinfo[i].metric); + } + } + } + + rw_exit(&afsifinfo_lock); #else for (ill = (struct ill_s *)*addr /*ill_g_headp */ ; ill; ill = ill->ill_next) { -#endif #ifdef AFS_SUN58_ENV /* Make sure this is an IPv4 ILL */ if (ill->ill_isv6) @@ -1349,6 +1400,7 @@ static int afs_SetServerPrefs(struct srvAddr *sa) { } } } +#endif /* AFS_SUN510_ENV */ #else #ifndef USEIFADDR struct ifnet *ifn = NULL; diff --git a/src/rx/SOLARIS/rx_knet.c b/src/rx/SOLARIS/rx_knet.c index 85eaa5f7c..c46f09b4e 100644 --- a/src/rx/SOLARIS/rx_knet.c +++ b/src/rx/SOLARIS/rx_knet.c @@ -36,6 +36,16 @@ RCSID #include "inet/ip.h" #include "inet/ip_if.h" #include "netinet/udp.h" +#ifdef AFS_SUN510_ENV +#include "h/ddi.h" +#include "h/ksynch.h" +#include "h/sunddi.h" +#include "h/sunldi.h" +#include "h/sockio.h" +#include "h/cmn_err.h" +#include "h/socket.h" +#include "netinet/in.h" +#endif /* * Function pointers for kernel socket routines @@ -68,14 +78,11 @@ rxi_GetIFInfo() { int i = 0; int different = 0; - +#ifndef AFS_SUN510_ENV ill_t *ill; ipif_t *ipif; - int rxmtu, maxmtu; -#ifdef AFS_SUN510_ENV - ill_walk_context_t ctx; #endif - + int rxmtu, maxmtu; int mtus[ADDRSPERSITE]; afs_uint32 addrs[ADDRSPERSITE]; afs_uint32 ifinaddr; @@ -84,10 +91,58 @@ rxi_GetIFInfo() memset(addrs, 0, sizeof(addrs)); #ifdef AFS_SUN510_ENV - for (ill = ILL_START_WALK_ALL(&ctx) ; ill ; ill = ill_next(&ctx, ill)) { + (void) rw_enter(&afsifinfo_lock, RW_READER); + + for (i = 0; (afsifinfo[i].ipaddr != NULL) && (i < ADDRSPERSITE); i++) { + + /* Ignore addresses which are down.. */ + if (!(afsifinfo[i].flags & IFF_UP)) + continue; + + /* Compute the Rx interface MTU */ + rxmtu = (afsifinfo[i].mtu - RX_IPUDP_SIZE); + + ifinaddr = afsifinfo[i].ipaddr; + if (myNetAddrs[i] != ifinaddr) + different++; + + /* Copy interface MTU and address; adjust maxmtu */ + mtus[i] = rxmtu; + rxmtu = rxi_AdjustIfMTU(rxmtu); + maxmtu = rxmtu * rxi_nRecvFrags + + ((rxi_nRecvFrags - 1) * UDP_HDR_SIZE); + maxmtu = rxi_AdjustMaxMTU(rxmtu, maxmtu); + addrs[i] = ifinaddr; + + if (ifinaddr != 0x7f000001 && maxmtu > rx_maxReceiveSize) { + rx_maxReceiveSize = MIN(RX_MAX_PACKET_SIZE, maxmtu); + rx_maxReceiveSize = + MIN(rx_maxReceiveSize, rx_maxReceiveSizeUser); + } + + } + + (void) rw_exit(&afsifinfo_lock); + + rx_maxJumboRecvSize = + RX_HEADER_SIZE + rxi_nDgramPackets * RX_JUMBOBUFFERSIZE + + (rxi_nDgramPackets - 1) * RX_JUMBOHEADERSIZE; + rx_maxJumboRecvSize = MAX(rx_maxJumboRecvSize, rx_maxReceiveSize); + + if (different) { + int j; + + for (j = 0; j < i; j++) { + myNetMTUs[j] = mtus[j]; + myNetAddrs[j] = addrs[j]; + } + } + + return different; +} + #else for (ill = ill_g_head; ill; ill = ill->ill_next) { -#endif #ifdef AFS_SUN58_ENV /* Make sure this is an IPv4 ILL */ if (ill->ill_isv6) @@ -144,17 +199,19 @@ rxi_GetIFInfo() return different; } +#endif int rxi_FindIfMTU(afs_uint32 addr) { - ill_t *ill; - ipif_t *ipif; afs_uint32 myAddr, netMask; int match_value = 0; int mtu = -1; #ifdef AFS_SUN510_ENV - ill_walk_context_t ctx; + int i = 0; +#else + ill_t *ill; + ipif_t *ipif; #endif if (numMyNetAddrs == 0) @@ -171,10 +228,46 @@ rxi_FindIfMTU(afs_uint32 addr) netMask = 0; #ifdef AFS_SUN510_ENV - for (ill = ILL_START_WALK_ALL(&ctx) ; ill ; ill = ill_next(&ctx, ill)) { + (void) rw_enter(&afsifinfo_lock, RW_READER); + + for (i = 0; (afsifinfo[i].ipaddr != NULL) && (i < ADDRSPERSITE); i++) { + afs_uint32 thisAddr, subnetMask; + int thisMtu; + + /* Ignore addresses which are down.. */ + if ((afsifinfo[i].flags & IFF_UP) == 0) + continue; + + thisAddr = afsifinfo[i].ipaddr; + subnetMask = afsifinfo[i].netmask; + thisMtu = afsifinfo[i].mtu; + + if ((myAddr & netMask) == (thisAddr & netMask)) { + if ((myAddr & subnetMask) == (thisAddr & subnetMask)) { + if (myAddr == thisAddr) { + match_value = 4; + mtu = thisMtu; + } + + if (match_value < 3) { + match_value = 3; + mtu = thisMtu; + } + } + + if (match_value < 2) { + match_value = 2; + mtu = thisMtu; + } + } + } + + (void) rw_exit(&afsifinfo_lock); + + return mtu; +} #else for (ill = ill_g_head; ill; ill = ill->ill_next) { -#endif #ifdef AFS_SUN58_ENV /* Make sure this is an IPv4 ILL */ if (ill->ill_isv6) @@ -213,6 +306,7 @@ rxi_FindIfMTU(afs_uint32 addr) return mtu; } +#endif /* rxi_NewSocket, rxi_FreeSocket and osi_NetSend are from the now defunct * afs_osinet.c. @@ -462,6 +556,174 @@ osi_NetReceive(osi_socket so, struct sockaddr_in *addr, struct iovec *dvec, return error; } +#if defined(AFS_SUN510_ENV) +/* How often afs collects interface info. Tunable via /etc/system: */ +/* set afs:afs_if_poll_interval = integer (value is in seconds) */ +static int afs_if_poll_interval = 30; + +/* Global array which holds the interface info for consumers */ +struct afs_ifinfo afsifinfo[ADDRSPERSITE]; + +void +osi_StartNetIfPoller() +{ + (void) ddi_taskq_dispatch(afs_taskq, (void(*) (void*)) osi_NetIfPoller, + NULL, DDI_SLEEP); +} + +void +osi_NetIfPoller() +{ + cred_t *cr; + ldi_ident_t li; + ldi_handle_t lh; + struct lifnum lifn; + struct lifconf lifc; + struct lifreq lifr; + struct lifreq *lifrp; + struct sockaddr_in *sin4_local; + struct sockaddr_in *sin4_dst; + major_t udpmajor; + caddr_t lifcbuf; + int i, count, error, rv; + int ifcount; + int metric; + int index; + uint_t mtu; + uint64_t flags; + + /* Get our permissions */ + cr = CRED(); + + /* Initialize and open /dev/udp for receiving ioctls */ + udpmajor = ddi_name_to_major(UDP_MOD_NAME); + + error = ldi_ident_from_major(udpmajor, &li); + if (error) + cmn_err(CE_PANIC, "osi_NetIfPoller: ldi_ident_from_major failed: %d", + error); + + error = ldi_open_by_name(UDP_DEV_NAME, FREAD, cr, &lh, li); + if (error) + cmn_err(CE_PANIC, + "osi_NetIfPoller: ldi_open_by_name failed: %d", error); + + + /* First, how many interfaces do we have? */ + (void) bzero((void *)&lifn, sizeof(struct lifnum)); + lifn.lifn_family = AF_INET; + + error = ldi_ioctl(lh, SIOCGLIFNUM, (intptr_t)&lifn, + FKIOCTL, cr, &rv); + if (error) + cmn_err(CE_PANIC, + "osi_NetIfPoller: ldi_ioctl: SIOCGLIFNUM failed: %d", error); + + ifcount = lifn.lifn_count; + + /* Set up some stuff for storing the results of SIOCGLIFCONF */ + (void) bzero((void *)&lifc, sizeof(struct lifconf)); + + lifcbuf = kmem_zalloc(ifcount * sizeof(struct lifreq), KM_SLEEP); + + lifc.lifc_family = AF_INET; + lifc.lifc_flags = IFF_UP; + lifc.lifc_len = ifcount * sizeof(struct lifreq); + lifc.lifc_buf = lifcbuf; + + /* Get info on each of our available interfaces. */ + error = ldi_ioctl(lh, SIOCGLIFCONF, (intptr_t)&lifc, + FKIOCTL, cr, &rv); + if (error) + cmn_err(CE_PANIC, + "osi_NetIfPoller: ldi_ioctl: SIOCGLIFCONF failed: %d", error); + + lifrp = lifc.lifc_req; + + count = 0; + + /* Loop through our interfaces and pick out the info we want */ + for (i = lifc.lifc_len / sizeof(struct lifreq); + i > 0; i--, lifrp++) { + + if (count >= ADDRSPERSITE) + break; + + (void) bzero((void *)&lifr, sizeof(struct lifreq)); + + (void) strncpy(lifr.lifr_name, lifrp->lifr_name, + sizeof(lifr.lifr_name)); + + /* Get this interface's Flags */ + error = ldi_ioctl(lh, SIOCGLIFFLAGS, (intptr_t)&lifr, + FKIOCTL, cr, &rv); + if (error) + cmn_err(CE_PANIC, + "osi_NetIfPoller: ldi_ioctl: SIOCGLIFFLAGS failed: %d", + error); + + /* Ignore plumbed but down interfaces. */ + if ((lifr.lifr_flags & IFF_UP) == 0) + continue; + + flags = lifr.lifr_flags; + + /* Get this interface's MTU */ + error = ldi_ioctl(lh, SIOCGLIFMTU, (intptr_t)&lifr, + FKIOCTL, cr, &rv); + + if (error) { + mtu = 1125; + } else { + mtu = lifr.lifr_metric; + } + + /* Get this interface's Metric */ + error = ldi_ioctl(lh, SIOCGLIFMETRIC, (intptr_t)&lifr, + FKIOCTL, cr, &rv); + + if (error) { + metric = 0; + } else { + metric = lifr.lifr_metric; + } + + sin4_local = (struct sockaddr_in *) &lifrp->lifr_addr; + sin4_dst = (struct sockaddr_in *) &lifrp->lifr_dstaddr; + + /* Acquire global array write lock */ + (void) rw_enter(&afsifinfo_lock, RW_WRITER); + + /* Copy our collected data into the global array */ + (void) strncpy(afsifinfo[count].ifname, lifrp->lifr_name, + sizeof(afsifinfo[count].ifname)); + afsifinfo[count].ipaddr = ntohl(sin4_local->sin_addr.s_addr); + afsifinfo[count].mtu = mtu; + afsifinfo[count].netmask = lifrp->lifr_addrlen; + afsifinfo[count].flags = flags; + afsifinfo[count].metric = metric; + afsifinfo[count].dstaddr = ntohl(sin4_dst->sin_addr.s_addr); + + /* Release global array write lock */ + (void) rw_exit(&afsifinfo_lock); + + count++; + + } /* Bottom of loop: for each interface ... */ + + kmem_free(lifcbuf, ifcount * sizeof(struct lifreq)); + + /* End of thread. Time to clean up */ + (void) ldi_close(lh, FREAD, cr); + (void) ldi_ident_release(li); + + /* Schedule this to run again after afs_if_poll_interval seconds */ + (void) timeout((void(*) (void *)) osi_StartNetIfPoller, NULL, + drv_usectohz((clock_t)afs_if_poll_interval * MICROSEC)); + +} +#endif /* AFS_SUN510_ENV */ + void shutdown_rxkernel(void) { diff --git a/src/rx/rx_prototypes.h b/src/rx/rx_prototypes.h index 4de165c00..4f75b833d 100644 --- a/src/rx/rx_prototypes.h +++ b/src/rx/rx_prototypes.h @@ -399,6 +399,10 @@ extern int osi_NetSend(osi_socket asocket, struct sockaddr_in *addr, #endif extern int osi_NetReceive(osi_socket so, struct sockaddr_in *addr, struct iovec *dvec, int nvecs, int *lengthp); +#if defined(AFS_SUN510_ENV) +extern void osi_StartNetIfPoller(void); +extern void osi_NetIfPoller(void); +#endif extern void osi_StopListener(void); extern int rxi_FindIfMTU(afs_uint32 addr); #ifndef RXK_LISTENER_ENV