From c95ace2c5963076deaf5f5c3a9856978f19ea0c4 Mon Sep 17 00:00:00 2001 From: "matt@linuxbox.com" Date: Mon, 3 Aug 2009 12:36:25 -0400 Subject: [PATCH] viced (non atomic) refcount/more-threads Replace host-hold bitmap with reference counting, phase 1, not atomic. Increase viced MAX_THREADS. NOTE: This code has been tested on Centos 5.3 x86_64, on VMWare, 2 physical, 2 logical CPUs. LICENSE BSD Change-Id: Id62dcc786619dfd99e1a7b69a196bbaa0b45158c Change-Id: I8ee2ec645402f02fdba0920815409b56172d393b Reviewed-on: http://gerrit.openafs.org/268 Reviewed-by: Derrick Brashear Tested-by: Derrick Brashear --- src/viced/afsfileprocs.c | 44 +++--- src/viced/callback.c | 66 ++++----- src/viced/host.c | 279 ++++++++++++++------------------------- src/viced/host.h | 64 ++++----- src/viced/viced.h | 5 +- 5 files changed, 188 insertions(+), 270 deletions(-) diff --git a/src/viced/afsfileprocs.c b/src/viced/afsfileprocs.c index 31bdc85bb..5041b4f6a 100644 --- a/src/viced/afsfileprocs.c +++ b/src/viced/afsfileprocs.c @@ -420,7 +420,6 @@ CallPostamble(register struct rx_connection *aconn, afs_int32 ret, struct host *thost; struct client *tclient; int translate = 0; - int held; H_LOCK; tclient = h_FindClient_r(aconn); @@ -430,23 +429,34 @@ CallPostamble(register struct rx_connection *aconn, afs_int32 ret, if (thost->hostFlags & HERRORTRANS) translate = 1; h_ReleaseClient_r(tclient); - held = h_Held_r(thost); - if (held) - h_Release_r(thost); - if (ahost && ahost != thost) { - char hoststr[16], hoststr2[16]; - ViceLog(0, ("CallPostamble: ahost %s:%d (%x) != thost %s:%d (%x)\n", - afs_inet_ntoa_r(ahost->host, hoststr), ntohs(ahost->port), - ahost, - afs_inet_ntoa_r(thost->host, hoststr2), ntohs(thost->port), - thost)); - h_Release_r(ahost); - } else if (!ahost) { - char hoststr[16]; - ViceLog(0, ("CallPostamble: null ahost for thost %s:%d (%x)\n", - afs_inet_ntoa_r(thost->host, hoststr), ntohs(thost->port), - thost)); + + /* return the reference taken in local h_FindClient_r--h_ReleaseClient_r + * does not decrement refcount on client->host */ + h_Release_r(thost); + + if (ahost) { + if (ahost != thost) { + /* host/client recycle */ + char hoststr[16], hoststr2[16]; + ViceLog(0, ("CallPostamble: ahost %s:%d (%x) != thost " + "%s:%d (%x)\n", + afs_inet_ntoa_r(ahost->host, hoststr), + ntohs(ahost->port), + ahost, + afs_inet_ntoa_r(thost->host, hoststr2), + ntohs(thost->port), + thost)); + } + /* return the reference taken in CallPreamble */ + h_Release_r(ahost); + } else { + char hoststr[16]; + ViceLog(0, ("CallPostamble: null ahost for thost %s:%d (%x)\n", + afs_inet_ntoa_r(thost->host, hoststr), + ntohs(thost->port), + thost)); } + busyout: H_UNLOCK; return (translate ? sys_error_to_et(ret) : ret); diff --git a/src/viced/callback.c b/src/viced/callback.c index 47c94ddf1..0403221bc 100644 --- a/src/viced/callback.c +++ b/src/viced/callback.c @@ -1474,28 +1474,32 @@ CleanupTimedOutCallBacks_r(void) static struct host *lih_host; static int lih_host_held; +/* Value of host->refCount that allows us to reliably infer that + * host may be held by some other thread */ +#define OTHER_MUSTHOLD_LIH 2 + /* This version does not allow 'host' to be selected unless its ActiveCall * is newer than 'hostp' which is the host with the oldest ActiveCall from * the last pass (if it is provided). We filter out any hosts that are * are held by other threads. */ static int -lih0_r(register struct host *host, register int held, void *rock) +lih0_r(register struct host *host, register int flags, void *rock) { struct host *hostp = (struct host *) rock; if (host->cblist && (hostp && host != hostp) - && (!held && !h_OtherHolds_r(host)) + && (host->refCount < OTHER_MUSTHOLD_LIH) && (!lih_host || host->ActiveCall < lih_host->ActiveCall) && (!hostp || host->ActiveCall > hostp->ActiveCall)) { - if (lih_host != NULL && lih_host_held) { - h_Release_r(lih_host); - } - lih_host = host; - lih_host_held = !held; - held = 1; + if (lih_host != NULL && lih_host_held) { + h_Release_r(lih_host); /* release prev host */ + } + lih_host = host; + lih_host_held = !flags; /* on i==1, this === (lih_host_held = 1) */ + flags = 1; /* now flags is 1, but at next(i), it will be 0 again */ } - return held; + return flags; } /* This version does not allow 'host' to be selected unless its ActiveCall @@ -1504,7 +1508,7 @@ lih0_r(register struct host *host, register int held, void *rock) * prevent held hosts from being selected. */ static int -lih1_r(register struct host *host, register int held, void *rock) +lih1_r(register struct host *host, register int flags, void *rock) { struct host *hostp = (struct host *) rock; @@ -1512,14 +1516,14 @@ lih1_r(register struct host *host, register int held, void *rock) && (hostp && host != hostp) && (!lih_host || host->ActiveCall < lih_host->ActiveCall) && (!hostp || host->ActiveCall > hostp->ActiveCall)) { - if (lih_host != NULL && lih_host_held) { - h_Release_r(lih_host); - } - lih_host = host; - lih_host_held = !held; - held = 1; + if (lih_host != NULL && lih_host_held) { + h_Release_r(lih_host); /* really? */ + } + lih_host = host; + lih_host_held = !flags; /* see note above */ + flags = 1; /* see note above */ } - return held; + return flags; } /* This could be upgraded to get more space each time */ @@ -1560,12 +1564,12 @@ GetSomeSpace_r(struct host *hostp, int locked) int lih_host_held2=lih_host_held; cbstuff.GSS4++; if ((hp != hostp) && !ClearHostCallbacks_r(hp, 0 /* not locked or held */ )) { - if (lih_host_held2) - h_Release_r(hp); + if (lih_host_held2) + h_Release_r(hp); return 0; } if (lih_host_held2) - h_Release_r(hp); + h_Release_r(hp); hp1 = hp; hp2 = hostList; } else { @@ -1603,25 +1607,23 @@ static int ClearHostCallbacks_r(struct host *hp, int locked) { int code; - int held = 0; char hoststr[16]; struct rx_connection *cb_conn = NULL; ViceLog(5, ("GSS: Delete longest inactive host %x (%s:%d)\n", hp, afs_inet_ntoa_r(hp->host, hoststr), ntohs(hp->port))); - if (!(held = h_Held_r(hp))) - h_Hold_r(hp); + + h_Hold_r(hp); /** Try a non-blocking lock. If the lock is already held return * after releasing hold on hp */ if (!locked) { - if (h_NBLock_r(hp)) { - if (!held) - h_Release_r(hp); - return 1; - } + if (h_NBLock_r(hp)) { + h_Release_r(hp); + return 1; + } } if (hp->Console & 2) { /* @@ -1664,11 +1666,9 @@ ClearHostCallbacks_r(struct host *hp, int locked) hp->hostFlags |= RESETDONE; } } - if (!locked) { - h_Unlock_r(hp); - } - if (!held) - h_Release_r(hp); + if (!locked) + h_Unlock_r(hp); + h_Release_r(hp); return 0; } diff --git a/src/viced/host.c b/src/viced/host.c index ed0b5a385..4c2de6568 100644 --- a/src/viced/host.c +++ b/src/viced/host.c @@ -94,7 +94,7 @@ struct CEBlock { /* block of CESPERBLOCK file entries */ struct client entry[CESPERBLOCK]; }; -static void h_TossStuff_r(register struct host *host); +void h_TossStuff_r(register struct host *host); /* * Make sure the subnet macros have been defined. @@ -490,56 +490,6 @@ hpr_GetCPS(afs_int32 id, prlist *CPS) static short consolePort = 0; -int -h_Release(register struct host *host) -{ - H_LOCK; - h_Release_r(host); - H_UNLOCK; - return 0; -} - -/** - * If this thread does not have a hold on this host AND - * if other threads also dont have any holds on this host AND - * If either the HOSTDELETED or CLIENTDELETED flags are set - * then toss the host - */ -int -h_Release_r(register struct host *host) -{ - - if (!((host)->holds[h_holdSlot()] & ~h_holdbit())) { - if (!h_OtherHolds_r(host)) { - /* must avoid masking this until after h_OtherHolds_r runs - * but it should be run before h_TossStuff_r */ - (host)->holds[h_holdSlot()] &= ~h_holdbit(); - if ((host->hostFlags & HOSTDELETED) - || (host->hostFlags & CLIENTDELETED)) { - h_TossStuff_r(host); - } - } else - (host)->holds[h_holdSlot()] &= ~h_holdbit(); - } else - (host)->holds[h_holdSlot()] &= ~h_holdbit(); - - return 0; -} - -int -h_OtherHolds_r(register struct host *host) -{ - register int i, bit, slot; - bit = h_holdbit(); - slot = h_holdSlot(); - for (i = 0; i < h_maxSlots; i++) { - if (host->holds[i] != ((i == slot) ? bit : 0)) { - return 1; - } - } - return 0; -} - int h_Lock_r(register struct host *host) { @@ -742,14 +692,12 @@ void h_flushhostcps(register afs_uint32 hostaddr, register afs_uint16 hport) { struct host *host; - int held = 0; H_LOCK; - h_Lookup_r(hostaddr, hport, &held, &host); + h_Lookup_r(hostaddr, hport, &host); if (host) { host->hcpsfailed = 1; - if (!held) - h_Release_r(host); + h_Release_r(host); } H_UNLOCK; return; @@ -836,12 +784,14 @@ h_SetupCallbackConn_r(struct host * host) rx_SetConnHardDeadTime(host->callback_rxcon, AFS_HARDDEADTIME); } -/* Lookup a host given an IP address and UDP port number. */ -/* hostaddr and hport are in network order */ -/* Note: host should be released by caller if 0 == *heldp and non-null */ -/* hostaddr and hport are in network order */ +/* h_Lookup_r + * Lookup a host given an IP address and UDP port number. + * hostaddr and hport are in network order + * hostaddr and hport are in network order + * On return, refCount is incremented. + */ int -h_Lookup_r(afs_uint32 haddr, afs_uint16 hport, int *heldp, struct host **hostp) +h_Lookup_r(afs_uint32 haddr, afs_uint16 hport, struct host **hostp) { afs_int32 now; struct host *host = NULL; @@ -860,14 +810,11 @@ h_Lookup_r(afs_uint32 haddr, afs_uint16 hport, int *heldp, struct host **hostp) *hostp = 0; return VBUSY; } - *heldp = h_Held_r(host); - if (!*heldp) - h_Hold_r(host); + h_Hold_r(host); h_Lock_r(host); if (host->hostFlags & HOSTDELETED) { h_Unlock_r(host); - if (!*heldp) - h_Release_r(host); + h_Release_r(host); goto restart; } h_Unlock_r(host); @@ -879,7 +826,7 @@ h_Lookup_r(afs_uint32 haddr, afs_uint16 hport, int *heldp, struct host **hostp) * first time that the host is added to a group. Also * here we also retry on previous legitimate hcps failures. * - * If we get here we still have a host hold. + * If we get here refCount is elevated. */ h_gethostcps_r(host, now); } @@ -911,27 +858,18 @@ h_LookupUuid_r(afsUUID * uuidp) } /*h_Lookup */ -/* - * h_Hold_r: Establish a hold by the current LWP on this host--the host - * or its clients will not be physically deleted until all holds have - * been released. - * NOTE: h_Hold_r is a macro defined in host.h. - */ - /* h_TossStuff_r: Toss anything in the host structure (the host or * clients marked for deletion. Called from h_Release_r ONLY. * To be called, there must be no holds, and either host->deleted * or host->clientDeleted must be set. */ -static void +void h_TossStuff_r(register struct host *host) { register struct client **cp, *client; - int i; /* if somebody still has this host held */ - for (i = 0; (i < h_maxSlots) && (!(host)->holds[i]); i++); - if (i != h_maxSlots) + if (host->refCount > 0) return; /* if somebody still has this host locked */ @@ -1031,18 +969,24 @@ h_TossStuff_r(register struct host *host) } /*h_TossStuff_r */ + /* h_Enumerate: Calls (*proc)(host, held, param) for at least each host in the * system at the start of the enumeration (perhaps more). Hosts may be deleted - * (have delete flag set); ditto for clients. (*proc) is always called with - * host h_held(). The hold state of the host with respect to this lwp is passed - * to (*proc) as the param held. The proc should return 0 if the host should be - * released, 1 if it should be held after enumeration. + * (have delete flag set); ditto for clients. refCount is always incremented + * before (*proc) is called. The param flags is passed to (*proc) as the + * param flags, permitting (*proc) to stop the enumeration (BAIL). + * + * Needed? Why not always h_Hold_r and h_Release_r in (*proc), or even -never- + * h_Hold_r or h_Release_r in (*proc)? + * + * **The proc should return 0 if the host should be released, 1 if it should + * be held after enumeration. */ void h_Enumerate(int (*proc) (struct host*, int, void *), void *param) { register struct host *host, **list; - register int *held; + register int *flags; register int i, count; H_LOCK; @@ -1052,18 +996,17 @@ h_Enumerate(int (*proc) (struct host*, int, void *), void *param) } list = (struct host **)malloc(hostCount * sizeof(struct host *)); if (!list) { - ViceLog(0, ("Failed malloc in h_Enumerate\n")); + ViceLog(0, ("Failed malloc in h_Enumerate (list)\n")); assert(0); } - held = (int *)malloc(hostCount * sizeof(int)); - if (!held) { - ViceLog(0, ("Failed malloc in h_Enumerate\n")); + flags = (int *)malloc(hostCount * sizeof(int)); + if (!flags) { + ViceLog(0, ("Failed malloc in h_Enumerate (flags)\n")); assert(0); } for (count = 0, host = hostList; host; host = host->next, count++) { list[count] = host; - if (!(held[count] = h_Held_r(host))) - h_Hold_r(host); + h_Hold_r(host); } if (count != hostCount) { ViceLog(0, ("h_Enumerate found %d of %d hosts\n", count, hostCount)); @@ -1071,25 +1014,28 @@ h_Enumerate(int (*proc) (struct host*, int, void *), void *param) assert(count <= hostCount); H_UNLOCK; for (i = 0; i < count; i++) { - held[i] = (*proc) (list[i], held[i], param); - if (!H_ENUMERATE_ISSET_HELD(held[i])) - h_Release(list[i]); /* this might free up the host */ + flags[i] = (*proc) (list[i], flags[i], param); + h_Release_r(list[i]); /* bail out of the enumeration early */ - if (H_ENUMERATE_ISSET_BAIL(held[i])) + if (H_ENUMERATE_ISSET_BAIL(flags[i])) break; } free((void *)list); - free((void *)held); -} /*h_Enumerate */ + free((void *)flags); +} /* h_Enumerate */ + /* h_Enumerate_r (revised): - * Calls (*proc)(host, held, param) for each host in hostList, starting - * at enumstart - * Hosts may be deleted (have delete flag set); ditto for clients. - * (*proc) is always called with - * host h_held() and the global host lock (H_LOCK) locked.The hold state of the - * host with respect to this lwp is passed to (*proc) as the param held. - * The proc should return 0 if the host should be released, 1 if it should + * Calls (*proc)(host, flags, param) for each host in hostList, starting + * at enumstart. Called only under H_LOCK. Hosts may be deleted (have + * delete flag set); ditto for clients. refCount is always incremented + * before (*proc) is called. The param flags is passed to (*proc) as the + * param flags, permitting (*proc) to stop the enumeration (BAIL). + * + * Needed? Why not always h_Hold_r and h_Release_r in (*proc), or even -never- + * h_Hold_r or h_Release_r in (*proc)? + * + * **The proc should return 0 if the host should be released, 1 if it should * be held after enumeration. */ void @@ -1097,28 +1043,26 @@ h_Enumerate_r(int (*proc) (struct host *, int, void *), struct host *enumstart, void *param) { register struct host *host, *next; - int held = 0; - int nheld = 0; + int flags = 0; + int nflags = 0; if (hostCount == 0) { return; } - if (enumstart && !(held = h_Held_r(enumstart))) - h_Hold_r(enumstart); - for (host = enumstart; host; host = next, held = nheld) { + h_Hold_r(enumstart); + for (host = enumstart; host; host = next, flags = nflags) { next = host->next; - if (next && !(nheld = h_Held_r(next)) && !H_ENUMERATE_ISSET_BAIL(held)) + if (next && !H_ENUMERATE_ISSET_BAIL(flags)) h_Hold_r(next); - held = (*proc) (host, held, param); - if (!H_ENUMERATE_ISSET_HELD(held)) + flags = (*proc) (host, flags, param); + if (H_ENUMERATE_ISSET_BAIL(flags)) { h_Release_r(host); /* this might free up the host */ - if (H_ENUMERATE_ISSET_BAIL(held)) { - if (!H_ENUMERATE_ISSET_HELD(nheld)) - h_Release_r(next); /* this might free up the host */ break; } + h_Release_r(host); /* this might free up the host */ } -} /*h_Enumerate_r */ +} /*h_Enumerate_r */ + /* inserts a new HashChain structure corresponding to this UUID */ void @@ -1509,14 +1453,13 @@ h_threadquota(int waiting) return 0; } -/* Host is returned held */ +/* If found, host is returned with refCount incremented */ struct host * h_GetHost_r(struct rx_connection *tcon) { struct host *host; struct host *oldHost; int code; - int held; struct interfaceAddr interf; int interfValid = 0; struct Identity *identP = NULL; @@ -1542,7 +1485,7 @@ h_GetHost_r(struct rx_connection *tcon) caps.Capabilities_len = 0; code = 0; - if (h_Lookup_r(haddr, hport, &held, &host)) + if (h_Lookup_r(haddr, hport, &host)) return 0; identP = (struct Identity *)rx_GetSpecific(tcon, rxcon_ident_key); if (host && !identP && !(host->Console & 1)) { @@ -1552,7 +1495,6 @@ h_GetHost_r(struct rx_connection *tcon) */ if ((host->hostFlags & HWHO_INPROGRESS) && h_threadquota(host->lock.num_waiting)) { - if (!held) h_Release_r(host); host = NULL; goto gethost_out; @@ -1561,8 +1503,7 @@ h_GetHost_r(struct rx_connection *tcon) if (!(host->hostFlags & ALTADDR)) { /* Another thread is doing initialization */ h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); ViceLog(125, ("Host %" AFS_PTR_FMT " (%s:%d) starting h_Lookup again\n", host, afs_inet_ntoa_r(host->host, hoststr), @@ -1636,8 +1577,7 @@ h_GetHost_r(struct rx_connection *tcon) host->hostFlags |= HOSTDELETED; host->hostFlags &= ~HWHO_INPROGRESS; h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); host = NULL; goto retry; } @@ -1652,8 +1592,7 @@ h_GetHost_r(struct rx_connection *tcon) host->hostFlags &= ~HWHO_INPROGRESS; host->hostFlags |= ALTADDR; h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); host = NULL; goto retry; } @@ -1673,11 +1612,10 @@ h_GetHost_r(struct rx_connection *tcon) if (!host->interface || !afs_uuid_equal(&interf.uuid, &host->interface->uuid)) { if (cb_in) { - ViceLog(25, - ("Uuid doesn't match connection (%s:%d).\n", - afs_inet_ntoa_r(haddr, hoststr), ntohs(hport))); - - removeAddress_r(host, haddr, hport); + ViceLog(25, + ("Uuid doesn't match connection (%s:%d).\n", + afs_inet_ntoa_r(haddr, hoststr), ntohs(hport))); + removeAddress_r(host, haddr, hport); } else { ViceLog(25, ("Uuid doesn't match host %" AFS_PTR_FMT " (%s:%d).\n", @@ -1688,8 +1626,7 @@ h_GetHost_r(struct rx_connection *tcon) host->hostFlags &= ~HWHO_INPROGRESS; host->hostFlags |= ALTADDR; h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); host = NULL; goto retry; } else if (cb_in) { @@ -1737,14 +1674,12 @@ h_GetHost_r(struct rx_connection *tcon) cb_in = NULL; if (rxconn) { - struct client *client; /* - * If rx_DestroyConnection calls h_FreeConnection we will - * deadlock on the host_glock_mutex. Work around the problem - * by unhooking the client from the connection before - * destroying the connection. + * If rx_DestroyConnection calls h_FreeConnection we + * will deadlock on the host_glock_mutex. Work around + * the problem by unhooking the client from the + * connection before destroying the connection. */ - client = rx_GetSpecific(rxconn, rxcon_client_key); rx_SetSpecific(rxconn, rxcon_client_key, (void *)0); rx_DestroyConnection(rxconn); } @@ -1763,8 +1698,7 @@ h_GetHost_r(struct rx_connection *tcon) host->hostFlags &= ~HWHO_INPROGRESS; host->hostFlags |= ALTADDR; h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); host = NULL; rx_DestroyConnection(cb_in); cb_in = NULL; @@ -1794,8 +1728,7 @@ h_GetHost_r(struct rx_connection *tcon) ntohs(host->port))); h_Lock_r(host); h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); ViceLog(125, ("Host %" AFS_PTR_FMT " (%s:%d) starting h_Lookup again\n", host, afs_inet_ntoa_r(host->host, hoststr), @@ -1827,8 +1760,7 @@ h_GetHost_r(struct rx_connection *tcon) h_Lock_r(host); host->hostFlags |= HOSTDELETED; h_Unlock_r(host); - if (!held) - h_Release_r(host); + h_Release_r(host); goto retry; } } else { @@ -1901,8 +1833,7 @@ h_GetHost_r(struct rx_connection *tcon) if (oldHost) { int probefail = 0; - if (!h_Held_r(oldHost)) - h_Hold_r(oldHost); + h_Hold_r(oldHost); h_Lock_r(oldHost); oldHost->hostFlags |= HWHO_INPROGRESS; @@ -2205,8 +2136,8 @@ h_ID2Client(afs_int32 vid) * by one. The caller must call h_ReleaseClient_r when finished with * the client. * - * the client->host is returned held. h_ReleaseClient_r does not release - * the hold on client->host. + * The refCount on client->host is returned incremented. h_ReleaseClient_r + * does not decrement the refCount on client->host. */ struct client * h_FindClient_r(struct rx_connection *tcon) @@ -2298,7 +2229,7 @@ h_FindClient_r(struct rx_connection *tcon) } if (!client) { /* loop */ - host = h_GetHost_r(tcon); /* Returns it h_Held */ + host = h_GetHost_r(tcon); /* Returns with incremented refCount */ if (!host) return 0; @@ -2561,7 +2492,7 @@ h_PrintStats(void) static int -h_PrintClient(register struct host *host, int held, void *rock) +h_PrintClient(register struct host *host, int flags, void *rock) { StreamHandle_t *file = (StreamHandle_t *)rock; register struct client *client; @@ -2575,7 +2506,7 @@ h_PrintClient(register struct host *host, int held, void *rock) LastCall = host->LastCall; if (host->hostFlags & HOSTDELETED) { H_UNLOCK; - return held; + return flags; } (void)afs_snprintf(tmpStr, sizeof tmpStr, "Host %s:%d down = %d, LastCall %s", @@ -2613,7 +2544,7 @@ h_PrintClient(register struct host *host, int held, void *rock) } } H_UNLOCK; - return held; + return flags; } /*h_PrintClient */ @@ -2651,7 +2582,7 @@ h_PrintClients(void) static int -h_DumpHost(register struct host *host, int held, void *rock) +h_DumpHost(register struct host *host, int flags, void *rock) { StreamHandle_t *file = (StreamHandle_t *)rock; @@ -2685,19 +2616,11 @@ h_DumpHost(register struct host *host, int held, void *rock) ntohs(host->interface->interface[i].port)); (void)STREAM_WRITE(tmpStr, strlen(tmpStr), 1, file); } - sprintf(tmpStr, "] holds: "); - (void)STREAM_WRITE(tmpStr, strlen(tmpStr), 1, file); - - for (i = 0; i < h_maxSlots; i++) { - sprintf(tmpStr, "%04x", host->holds[i]); - (void)STREAM_WRITE(tmpStr, strlen(tmpStr), 1, file); - } - sprintf(tmpStr, " slot/bit: %ld/%d\n", (long int) h_holdSlot(), - h_holdbit()); + sprintf(tmpStr, "] refCount: %d\n", host->refCount); (void)STREAM_WRITE(tmpStr, strlen(tmpStr), 1, file); H_UNLOCK; - return held; + return flags; } /*h_DumpHost */ @@ -2734,10 +2657,10 @@ h_DumpHosts(void) static int h_stateFillHeader(struct host_state_header * hdr); static int h_stateCheckHeader(struct host_state_header * hdr); static int h_stateAllocMap(struct fs_dump_state * state); -static int h_stateSaveHost(struct host * host, int held, void *rock); +static int h_stateSaveHost(struct host * host, int flags, void *rock); static int h_stateRestoreHost(struct fs_dump_state * state); -static int h_stateRestoreIndex(struct host * h, int held, void *rock); -static int h_stateVerifyHost(struct host * h, int held, void *rock); +static int h_stateRestoreIndex(struct host * h, int flags, void *rock); +static int h_stateVerifyHost(struct host * h, int flags, void *rock); static int h_stateVerifyAddrHash(struct fs_dump_state * state, struct host * h, afs_uint32 addr, afs_uint16 port); static int h_stateVerifyUuidHash(struct fs_dump_state * state, struct host * h); static void h_hostToDiskEntry_r(struct host * in, struct hostDiskEntry * out); @@ -2829,13 +2752,13 @@ h_stateRestoreIndices(struct fs_dump_state * state) } static int -h_stateRestoreIndex(struct host * h, int held, void *rock) +h_stateRestoreIndex(struct host * h, int flags, void *rock) { struct fs_dump_state *state = (struct fs_dump_state *)rock; if (cb_OldToNew(state, h->cblist, &h->cblist)) { - return H_ENUMERATE_BAIL(held); + return H_ENUMERATE_BAIL(flags); } - return held; + return flags; } int @@ -2846,14 +2769,14 @@ h_stateVerify(struct fs_dump_state * state) } static int -h_stateVerifyHost(struct host * h, int held, void* rock) +h_stateVerifyHost(struct host * h, int flags, void* rock) { struct fs_dump_state *state = (struct fs_dump_state *)rock; int i; if (h == NULL) { ViceLog(0, ("h_stateVerifyHost: error: NULL host pointer in linked list\n")); - return H_ENUMERATE_BAIL(held); + return H_ENUMERATE_BAIL(flags); } if (h->interface) { @@ -2874,7 +2797,7 @@ h_stateVerifyHost(struct host * h, int held, void* rock) state->bail = 1; } - return held; + return flags; } static int @@ -3021,7 +2944,7 @@ h_stateAllocMap(struct fs_dump_state * state) /* function called by h_Enumerate to save a host to disk */ static int -h_stateSaveHost(struct host * host, int held, void* rock) +h_stateSaveHost(struct host * host, int flags, void* rock) { struct fs_dump_state *state = (struct fs_dump_state *) rock; int if_len=0, hcps_len=0; @@ -3087,9 +3010,9 @@ h_stateSaveHost(struct host * host, int held, void* rock) if (hcps) free(hcps); if (state->bail) { - return H_ENUMERATE_BAIL(held); + return H_ENUMERATE_BAIL(flags); } - return held; + return flags; } /* restores a host from disk */ @@ -3481,7 +3404,7 @@ static struct AFSFid zerofid; */ #if 0 static int -CheckHost(register struct host *host, int held, void *rock) +CheckHost(register struct host *host, int flags, void *rock) { register struct client *client; struct rx_connection *cb_conn = NULL; @@ -3492,7 +3415,7 @@ CheckHost(register struct host *host, int held, void *rock) FS_STATE_RDLOCK; if (fs_state.mode == FS_MODE_SHUTDOWN) { FS_STATE_UNLOCK; - return H_ENUMERATE_BAIL(held); + return H_ENUMERATE_BAIL(flags); } FS_STATE_UNLOCK; #endif @@ -3587,7 +3510,7 @@ CheckHost(register struct host *host, int held, void *rock) #endif int -CheckHost_r(register struct host *host, int held, void *dummy) +CheckHost_r(register struct host *host, int flags, void *dummy) { register struct client *client; struct rx_connection *cb_conn = NULL; @@ -3598,7 +3521,7 @@ CheckHost_r(register struct host *host, int held, void *dummy) FS_STATE_RDLOCK; if (fs_state.mode == FS_MODE_SHUTDOWN) { FS_STATE_UNLOCK; - return H_ENUMERATE_BAIL(held); + return H_ENUMERATE_BAIL(flags); } FS_STATE_UNLOCK; #endif @@ -3683,7 +3606,7 @@ CheckHost_r(register struct host *host, int held, void *dummy) } h_Unlock_r(host); } - return held; + return flags; } /*CheckHost_r */ diff --git a/src/viced/host.h b/src/viced/host.h index 932e3c48b..bbcdb7d79 100644 --- a/src/viced/host.h +++ b/src/viced/host.h @@ -42,13 +42,6 @@ extern pthread_key_t viced_uclient_key; #define h_HTSPERBLOCK 512 /* Power of 2 */ #define h_HTSHIFT 9 /* log base 2 of HTSPERBLOCK */ -#define h_threadsPerSlot 32 /* bits per afs_int32 */ -#define h_threadsShift 5 /* for multiply/divide */ -#define h_threadsMask 31 /* for remainder */ - -/* size of the hold array for each host */ -#define h_maxSlots (((MAX_FILESERVER_THREAD+h_threadsPerSlot-1)>>h_threadsShift)+1) - struct Identity { char valid; /* zero if UUID is unknown */ afsUUID uuid; @@ -70,10 +63,7 @@ struct Interface { struct host { struct host *next, *prev; /* linked list of all hosts */ struct rx_connection *callback_rxcon; /* rx callback connection */ - afs_int32 holds[h_maxSlots]; - /* holds on this host; 1 bit per lwp. - * A hold prevents this structure and - * inferior structures from disappearing */ + afs_uint32 refCount; /* reference count */ afs_uint32 host; /* IP address of host interface that is * currently being used, in network * byte order */ @@ -164,30 +154,6 @@ extern int rxcon_client_key; for the client, then the client must have a connection */ /* N.B. h_UserName returns pointer to static data; also relatively expensive */ extern char *h_UserName(struct client *client); - -/* all threads whose thread-id is greater than the size of the hold array, -** then use the most significant bit in the 'hold' field in the host structure -*/ -#ifdef AFS_PTHREAD_ENV -#define h_lwpIndex() ( ((long)(pthread_getspecific(rx_thread_id_key)) > \ - ((h_maxSlots << h_threadsShift)-1)) ? \ - (h_maxSlots << h_threadsShift) -1 : \ - (long)(pthread_getspecific(rx_thread_id_key)) ) -#else /* AFS_PTHREAD_ENV */ -#define h_lwpIndex() ( (LWP_Index() > ((h_maxSlots << h_threadsShift)-1)) ? \ - (h_maxSlots << h_threadsShift) -1 : \ - LWP_Index() ) -#endif /* AFS_PTHREAD_ENV */ -#define h_holdIndex()( h_lwpIndex() & h_threadsMask) -#define h_holdSlot() ( h_lwpIndex() >> h_threadsShift) /*index in 'holds' */ -#define h_holdbit() ( 1<holds[h_holdSlot()] |= h_holdbit()) -extern int h_Release(register struct host *host); -extern int h_Release_r(register struct host *host); - -#define h_Held_r(host) ((h_holdbit() & (host)->holds[h_holdSlot()]) != 0) -extern int h_OtherHolds_r(register struct host *host); #define h_Lock(host) ObtainWriteLock(&(host)->lock) extern int h_Lock_r(register struct host *host); #define h_Unlock(host) ReleaseWriteLock(&(host)->lock) @@ -197,6 +163,25 @@ extern int h_Lock_r(register struct host *host); #define AddVolCallBack(host, fid) AddCallBack1((host), (fid), (afs_uint32 *)0, 3/*CB_VOLUME*/, 0) #define AddBulkCallBack(host, fid) AddCallBack1((host), (fid), (afs_uint32 *)0, 4/*CB_BULK*/, 0) +/* A simple refCount replaces per-thread hold mechanism. The former + * hold semantics are not different from refcounting, except with respect + * to cross-thread assertions. In this change, refcount is protected by + * H_LOCK, just like former hold bitmap. A future change will replace locks + * th lock-free operations. */ + +#define h_Hold_r(x) \ +do { \ + ++((x)->refCount); \ +} while(0) + +#define h_Release_r(x) \ +do { \ + --((x)->refCount); \ + if (((x)->refCount < 1) && \ + (((x)->hostFlags & HOSTDELETED) || \ + ((x)->hostFlags & CLIENTDELETED))) h_TossStuff_r((x)); \ +} while(0) + /* operations on the global linked list of hosts */ #define h_InsertList_r(h) (h)->next = hostList; \ (h)->prev = 0; \ @@ -230,13 +215,14 @@ extern void ShutDownAndCore(int dopanic); extern struct host *h_Alloc(register struct rx_connection *r_con); extern struct host *h_Alloc_r(register struct rx_connection *r_con); extern int h_Lookup_r(afs_uint32 hostaddr, afs_uint16 hport, - int *heldp, struct host **hostp); + struct host **hostp); extern struct host *h_LookupUuid_r(afsUUID * uuidp); extern void h_Enumerate(int (*proc) (struct host *, int, void *), void *param); extern void h_Enumerate_r(int (*proc) (struct host *, int, void *), struct host *enumstart, void *param); extern struct host *h_GetHost_r(struct rx_connection *tcon); extern struct client *h_FindClient_r(struct rx_connection *tcon); extern int h_ReleaseClient_r(struct client *client); +extern void h_TossStuff_r(register struct host *host); extern struct client *h_ID2Client(afs_int32 vid); extern int GetClient(struct rx_connection *tcon, struct client **cp); extern int PutClient(struct client **cp); @@ -272,9 +258,9 @@ extern int h_SaveState(void); extern int h_RestoreState(void); #endif -#define H_ENUMERATE_BAIL(held) ((held)|0x80000000) -#define H_ENUMERATE_ISSET_BAIL(held) ((held)&0x80000000) -#define H_ENUMERATE_ISSET_HELD(held) ((held)&0x7FFFFFFF) +#define H_ENUMERATE_BAIL(flags) ((flags)|0x80000000) +#define H_ENUMERATE_ISSET_BAIL(flags) ((flags)&0x80000000) +#define H_ENUMERATE_ISSET_HELD(flags) ((flags)&0x7FFFFFFF) struct host *(hosttableptrs[h_MAXHOSTTABLES]); /* Used by h_itoh */ #define h_htoi(host) ((host)->index) /* index isn't zeroed, no need to lock */ diff --git a/src/viced/viced.h b/src/viced/viced.h index 549b40d5a..704242e93 100644 --- a/src/viced/viced.h +++ b/src/viced/viced.h @@ -190,9 +190,8 @@ extern int saneacls; #define DONTPANIC 0 #define PANIC 1 -#define MAX_FILESERVER_THREAD 128 /* max number of threads in fileserver, subject to system limits. match to FD_HANDLE_SETASIDE */ - -#define FILESERVER_HELPER_THREADS 8 /* Listner, IOMGR, FiveMinute, FsyncCk +#define MAX_FILESERVER_THREAD 16384 /* max number of threads in fileserver */ +#define FILESERVER_HELPER_THREADS 8 /* Listner, IOMGR, FiveMinute, FsyncCk * HostCheck, Signal, min 2 for RXSTATS */ #ifdef AFS_PTHREAD_ENV #include -- 2.39.5