From 954442e207d8a05d2f26992ff966b56f7af41e1e Mon Sep 17 00:00:00 2001 From: Andrew Deason Date: Wed, 29 Jun 2011 13:51:22 -0500 Subject: [PATCH] SOLARIS: Granular multiPage detection Currently, a struct vcache has a multiPage counter, indicating how many afs_getpage requests are in-flight for that vcache that involve retrieving multiple pages. Any dcache associated with such vcaches are then avoided when choosing dcache entries to evict from the cache, since we may deadlock when trying to evict a dcache entry from one of the earlier afs_GetOnePage calls in a particular afs_getpage request. This behavior can cause the client to become unusable if the cache becomes full, and the only items in the cache are dcache entries in a file that has an in-flight multi-page afs_getpage request. Since, in that case, we cannot kick out any entries from the cache, and so we wait forever to wait for the cache utilization to go down. To prevent this from occurring, record exactly which ranges in the file have in-flight multi-page afs_getpage requests, and just avoid dcache entries in those ranges. This way afs_GetDownD can evict dcache entries in the same file, but still avoid entries that would cause a deadlock. Also add some comments explaining this situation a bit more. Change-Id: Idd39fd4811ea03aa7eee62f85f1a0c74c9c5e402 Reviewed-on: http://gerrit.openafs.org/4896 Reviewed-by: Derrick Brashear Tested-by: BuildBot (cherry picked from commit 31a001f60e5fe729b315f679d1d43b367bd74ea5) Reviewed-on: http://gerrit.openafs.org/4927 Tested-by: Derrick Brashear --- src/afs/SOLARIS/osi_vcache.c | 2 ++ src/afs/SOLARIS/osi_vm.c | 28 ++++++++++++++++++++++++++++ src/afs/SOLARIS/osi_vnodeops.c | 29 +++++++++++++++++++++++++++-- src/afs/afs.h | 17 ++++++++++++++++- src/afs/afs_dcache.c | 12 ++++++++---- src/afs/afs_prototypes.h | 1 + 6 files changed, 82 insertions(+), 7 deletions(-) diff --git a/src/afs/SOLARIS/osi_vcache.c b/src/afs/SOLARIS/osi_vcache.c index dd47176aa..8c2ce1115 100644 --- a/src/afs/SOLARIS/osi_vcache.c +++ b/src/afs/SOLARIS/osi_vcache.c @@ -35,6 +35,8 @@ void osi_PrePopulateVCache(struct vcache *avc) { memset(avc, 0, sizeof(struct vcache)); + QInit(&avc->multiPage); + AFS_RWLOCK_INIT(&avc->vlock, "vcache vlock"); rw_init(&avc->rwlock, "vcache rwlock", RW_DEFAULT, NULL); diff --git a/src/afs/SOLARIS/osi_vm.c b/src/afs/SOLARIS/osi_vm.c index a6a58f5ef..19fad3c9d 100644 --- a/src/afs/SOLARIS/osi_vm.c +++ b/src/afs/SOLARIS/osi_vm.c @@ -54,6 +54,34 @@ osi_VM_GetDownD(struct vcache *avc, struct dcache *adc) return code; } +/* Does this dcache conflict with a multiPage request for this vcache? + * + * This function only exists for Solaris. This is used by afs_GetDownD to + * calculate if trying to evict the given dcache may deadlock with an + * in-progress afs_getpage call that is trying to get more than one page at + * once. See afs_getpage for details. We return 0 if we do NOT conflict, + * nonzero otherwise. If we return nonzero, we should NOT try to evict the + * given dcache entry from the cache. + * + * Locking: tvc->vlock is write-locked on entry (and GLOCK is held) + */ +int +osi_VM_MultiPageConflict(struct vcache *avc, struct dcache *adc) +{ + struct multiPage_range *range; + for (range = (struct multiPage_range *)avc->multiPage.next; + range != &avc->multiPage; + range = (struct multiPage_range *)QNext(&range->q)) { + + if (adc->f.chunk >= AFS_CHUNK(range->off) && + adc->f.chunk <= AFS_CHUNK(range->off + range->len - 1)) { + return 1; + } + } + + return 0; +} + /* Try to discard pages, in order to recycle a vcache entry. * * We also make some sanity checks: ref count, open count, held locks. diff --git a/src/afs/SOLARIS/osi_vnodeops.c b/src/afs/SOLARIS/osi_vnodeops.c index cfd112419..6eb228ea1 100644 --- a/src/afs/SOLARIS/osi_vnodeops.c +++ b/src/afs/SOLARIS/osi_vnodeops.c @@ -162,9 +162,34 @@ afs_getpage(struct vnode *vp, offset_t off, u_int len, u_int *protp, #endif off, len, protp, pl, plsz, seg, addr, rw, acred); else { + struct multiPage_range range; struct vcache *vcp = VTOAFS(vp); + + /* We've been asked to get more than one page. We must return all + * requested pages at once, all of them locked, which means all of + * these dcache entries cannot be kicked out of the cache before we + * return (since their pages cannot be invalidated). + * + * afs_GetOnePage will be called multiple times by pvn_getpages in + * order to get all of the requested pages. One of the later + * afs_GetOnePage calls may need to evict some cache entries in order + * to perform its read. If we try to kick out one of the entries an + * earlier afs_GetOnePage call used, we will deadlock since we have + * the page locked. So, to tell afs_GetDownD that it should skip over + * any entries we've read in due to this afs_getpage call, record the + * offset and length in avc->multiPage. + * + * Ideally we would just set something in each dcache as we get it, + * but that is rather difficult, since pvn_getpages doesn't let us + * retain any information between calls to afs_GetOnePage. So instead + * just record the offset and length, and let afs_GetDownD calculate + * which dcache entries should be skipped. */ + + range.off = off; + range.len = len; + ObtainWriteLock(&vcp->vlock, 548); - vcp->multiPage++; + QAdd(&vcp->multiPage, &range.q); ReleaseWriteLock(&vcp->vlock); afs_BozonLock(&vcp->pvnLock, vcp); code = @@ -175,7 +200,7 @@ afs_getpage(struct vnode *vp, offset_t off, u_int len, u_int *protp, off, len, protp, pl, plsz, seg, addr, rw, acred); afs_BozonUnlock(&vcp->pvnLock, vcp); ObtainWriteLock(&vcp->vlock, 549); - vcp->multiPage--; + QRemove(&range.q); ReleaseWriteLock(&vcp->vlock); } AFS_GUNLOCK(); diff --git a/src/afs/afs.h b/src/afs/afs.h index fcc4c707a..5c3c95eaa 100644 --- a/src/afs/afs.h +++ b/src/afs/afs.h @@ -750,6 +750,21 @@ struct fvcache { struct afs_vnuniq oldParent; }; +#ifdef AFS_SUN5_ENV +/* + * This is for the multiPage field in struct vcache. Each one of these + * represents an outstanding getpage request that is larger than a single page. + * Recording these is necessary to prevent afs_GetOnePage from trying to evict + * a dcache entry that an earlier afs_GetOnePage call got in the same getpage + * request. See osi_VM_MultiPageConflict and afs_getpage. + */ +struct multiPage_range { + struct afs_q q; + offset_t off; /**< offset of getpage request */ + u_int len; /**< length of getpage request */ +}; +#endif + /* INVARIANTs: (vlruq.next != NULL) == (vlruq.prev != NULL) * nextfree => !vlruq.next && ! vlruq.prev * !(avc->nextfree) && !avc->vlruq.next => (FreeVCList == avc->nextfree) @@ -870,7 +885,7 @@ struct vcache { afs_ucred_t *uncred; int asynchrony; /* num kbytes to store behind */ #ifdef AFS_SUN5_ENV - short multiPage; /* count of multi-page getpages in progress */ + struct afs_q multiPage; /* list of multiPage_range structs */ #endif }; diff --git a/src/afs/afs_dcache.c b/src/afs/afs_dcache.c index c926a0840..6584bd247 100644 --- a/src/afs/afs_dcache.c +++ b/src/afs/afs_dcache.c @@ -589,7 +589,9 @@ afs_GetDownD(int anumber, int *aneedSpace, afs_int32 buckethint) * we don't reclaim active entries, or other than target bucket. * Set to 1, we reclaim even active ones in target bucket. * Set to 2, we reclaim any inactive one. - * Set to 3, we reclaim even active ones. + * Set to 3, we reclaim even active ones. On Solaris, we also reclaim + * entries whose corresponding vcache has a nonempty multiPage list, when + * possible. */ if (splitdcache) { phase = 0; @@ -724,9 +726,11 @@ afs_GetDownD(int anumber, int *aneedSpace, afs_int32 buckethint) ReleaseWriteLock(&afs_xdcache); ObtainWriteLock(&tvc->vlock, 543); - if (tvc->multiPage) { - skip = 1; - goto endmultipage; + if (!QEmpty(&tvc->multiPage)) { + if (phase < 3 || osi_VM_MultiPageConflict(tvc, tdc)) { + skip = 1; + goto endmultipage; + } } /* block locking pages */ tvc->vstates |= VPageCleaning; diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h index 5b2f63bb5..8ef955aa1 100644 --- a/src/afs/afs_prototypes.h +++ b/src/afs/afs_prototypes.h @@ -741,6 +741,7 @@ extern int osi_VM_Setup(struct vcache *avc, int force); #ifdef AFS_SUN5_ENV extern int osi_VM_GetDownD(struct vcache *avc, struct dcache *adc); +extern int osi_VM_MultiPageConflict(struct vcache *avc, struct dcache *adc); extern void osi_VM_PreTruncate(struct vcache *avc, int alen, afs_ucred_t *acred); #endif -- 2.39.5