From 8886b07c7986410664a3a30d16f6e1142a90135a Mon Sep 17 00:00:00 2001 From: Tom Keiser Date: Mon, 30 May 2005 06:23:30 +0000 Subject: [PATCH] STABLE14-parallel-attaching-20050529 FIXES 18999 allow parallelization of VInitVolumePackage so attaching volumes can be sped up (cherry picked from commit b9ae6e9547471c4006e4ac00879b3cb9bcefe47b) --- src/viced/viced.c | 11 ++++ src/vol/volume.c | 141 ++++++++++++++++++++++++++++++++++++++++++---- src/vol/volume.h | 8 +-- 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/src/viced/viced.c b/src/viced/viced.c index 3204df654..85119aba0 100644 --- a/src/viced/viced.c +++ b/src/viced/viced.c @@ -739,6 +739,9 @@ FlagMsg() strcat(buffer, "[-rxpck ] "); strcat(buffer, "[-rxdbg (enable rx debugging)] "); strcat(buffer, "[-rxdbge (enable rxevent debugging)] "); +#if AFS_PTHREAD_ENV + strcat(buffer, "[-vattachpar ] "); +#endif #ifdef AFS_AIX32_ENV strcat(buffer, "[-m ] "); #endif @@ -934,6 +937,14 @@ ParseArgs(int argc, char *argv[]) return -1; } rxpackets = atoi(argv[++i]); +#ifdef AFS_PTHREAD_ENV + } else if (!strcmp(argv[i], "-vattachpar")) { + if ((i + 1) >= argc) { + fprintf(stderr, "missing argument for -vattachpar\n"); + return -1; + } + vol_attach_threads = atoi(argv[++i]); +#endif /* AFS_PTHREAD_ENV */ } else if (!strcmp(argv[i], "-s")) { Sawsmall = 1; if ((i + 1) >= argc) { diff --git a/src/vol/volume.c b/src/vol/volume.c index 48eeecf81..9954bbf32 100644 --- a/src/vol/volume.c +++ b/src/vol/volume.c @@ -152,6 +152,7 @@ pthread_mutex_t vol_fsync_mutex; pthread_mutex_t vol_trans_mutex; pthread_cond_t vol_put_volume_cond; pthread_cond_t vol_sleep_cond; +int vol_attach_threads = 1; #endif /* AFS_PTHREAD_ENV */ #ifdef AFS_OSF_ENV @@ -209,6 +210,20 @@ ffs(x) } #endif /* !AFS_HAVE_FFS */ +#ifdef AFS_PTHREAD_ENV +#include "rx/rx_queue.h" +typedef struct diskpartition_queue_t { + struct rx_queue queue; + struct DiskPartition * diskP; +} diskpartition_queue_t; +typedef struct vinitvolumepackage_thread_t { + struct rx_queue queue; + pthread_cond_t thread_done_cv; + int n_threads_complete; +} vinitvolumepackage_thread_t; +static void * VInitVolumePackageThread(void * args); +#endif /* AFS_PTHREAD_ENV */ + struct Lock vol_listLock; /* Lock obtained when listing volumes: prevents a volume from being missed if the volume is attached during a list volumes */ extern struct Lock FSYNC_handler_lock; @@ -229,7 +244,8 @@ int VInit; /* 0 - uninitialized, bit32 VolumeCacheCheck; /* Incremented everytime a volume goes on line-- * used to stamp volume headers and in-core * vnodes. When the volume goes on-line the - * vnode will be invalidated */ + * vnode will be invalidated + * access only with VOL_LOCK held */ int VolumeCacheSize = 200, VolumeGets = 0, VolumeReplacements = 0, Vlooks = 0; @@ -284,10 +300,48 @@ VInitVolumePackage(ProgramType pt, int nLargeVnodes, int nSmallVnodes, return -1; if (programType == fileServer) { - DIR *dirp; - struct dirent *dp; struct DiskPartition *diskP; +#ifdef AFS_PTHREAD_ENV + struct vinitvolumepackage_thread_t params; + struct diskpartition_queue_t * dpq; + int i, len; + pthread_t tid; + pthread_attr_t attrs; + + assert(pthread_cond_init(¶ms.thread_done_cv,NULL) == 0); + queue_Init(¶ms); + params.n_threads_complete = 0; + + /* create partition work queue */ + for (len=0, diskP = DiskPartitionList; diskP; diskP = diskP->next, len++) { + dpq = (diskpartition_queue_t *) malloc(sizeof(struct diskpartition_queue_t)); + assert(dpq != NULL); + dpq->diskP = diskP; + queue_Prepend(¶ms,dpq); + } + assert(pthread_attr_init(&attrs) == 0); + assert(pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED) == 0); + + len = MIN(len, vol_attach_threads); + + VOL_LOCK; + for (i=0; i < len; i++) { + assert(pthread_create + (&tid, &attrs, &VInitVolumePackageThread, + ¶ms) == 0); + } + + while(params.n_threads_complete < len) { + pthread_cond_wait(¶ms.thread_done_cv,&vol_glock_mutex); + } + VOL_UNLOCK; + + assert(pthread_cond_destroy(¶ms.thread_done_cv) == 0); + +#else /* AFS_PTHREAD_ENV */ + DIR *dirp; + struct dirent *dp; /* Attach all the volumes in this partition */ for (diskP = DiskPartitionList; diskP; diskP = diskP->next) { @@ -319,6 +373,7 @@ VInitVolumePackage(ProgramType pt, int nLargeVnodes, int nSmallVnodes, Log("Partition %s: attached %d volumes; %d volumes not attached\n", diskP->name, nAttached, nUnattached); closedir(dirp); } +#endif /* AFS_PTHREAD_ENV */ } VInit = 2; /* Initialized, and all volumes have been attached */ @@ -332,6 +387,65 @@ VInitVolumePackage(ProgramType pt, int nLargeVnodes, int nSmallVnodes, return 0; } +#ifdef AFS_PTHREAD_ENV +static void * +VInitVolumePackageThread(void * args) { + int errors = 0; /* Number of errors while finding vice partitions. */ + + DIR *dirp; + struct dirent *dp; + struct DiskPartition *diskP; + struct vinitvolumepackage_thread_t * params; + struct diskpartition_queue_t * dpq; + int nAttached = 0, nUnattached = 0; + + params = (vinitvolumepackage_thread_t *) args; + + VOL_LOCK; + /* Attach all the volumes in this partition */ + while (queue_IsNotEmpty(params)) { + dpq = queue_First(params,diskpartition_queue_t); + queue_Remove(dpq); + VOL_UNLOCK; + diskP = dpq->diskP; + free(dpq); + + Log("Partition %s: attaching volumes\n", diskP->name); + dirp = opendir(VPartitionPath(diskP)); + assert(dirp); + while ((dp = readdir(dirp))) { + char *p; + p = strrchr(dp->d_name, '.'); + if (p != NULL && strcmp(p, VHDREXT) == 0) { + Error error; + Volume *vp; + vp = VAttachVolumeByName(&error, diskP->name, dp->d_name, + V_VOLUPD); + (*(vp ? &nAttached : &nUnattached))++; + if (error == VOFFLINE) + Log("Volume %d stays offline (/vice/offline/%s exists)\n", VolumeNumber(dp->d_name), dp->d_name); + else if (LogLevel >= 5) { + Log("Partition %s: attached volume %d (%s)\n", + diskP->name, VolumeNumber(dp->d_name), + dp->d_name); + } + if (vp) { + VPutVolume(vp); + } + } + } + Log("Partition %s: attached %d volumes; %d volumes not attached\n", diskP->name, nAttached, nUnattached); + closedir(dirp); + VOL_LOCK; + } + + params->n_threads_complete++; + pthread_cond_signal(¶ms->thread_done_cv); + VOL_UNLOCK; + return NULL; +} +#endif /* AFS_PTHREAD_ENV */ + /* This must be called by any volume utility which needs to run while the file server is also running. This is separated from VInitVolumePackage so that a utility can fork--and each of the children can independently @@ -725,6 +839,7 @@ attach2(Error * ec, char *path, register struct VolumeHeader * header, register Volume *vp; VOL_UNLOCK; + vp = (Volume *) calloc(1, sizeof(Volume)); assert(vp != NULL); vp->specialStatus = (byte) (isbusy ? VBUSY : 0); @@ -737,18 +852,21 @@ attach2(Error * ec, char *path, register struct VolumeHeader * header, IH_INIT(vp->diskDataHandle, partp->device, header->parent, header->volumeInfo); IH_INIT(vp->linkHandle, partp->device, header->parent, header->linkTable); - vp->cacheCheck = ++VolumeCacheCheck; - /* just in case this ever rolls over */ - if (!vp->cacheCheck) - vp->cacheCheck = ++VolumeCacheCheck; vp->shuttingDown = 0; vp->goingOffline = 0; vp->nUsers = 1; + VOL_LOCK; + vp->cacheCheck = ++VolumeCacheCheck; + /* just in case this ever rolls over */ + if (!vp->cacheCheck) + vp->cacheCheck = ++VolumeCacheCheck; GetVolumeHeader(vp); VOL_UNLOCK; + (void)ReadHeader(ec, V_diskDataHandle(vp), (char *)&V_disk(vp), sizeof(V_disk(vp)), VOLUMEINFOMAGIC, VOLUMEINFOVERSION); + VOL_LOCK; if (*ec) { Log("VAttachVolume: Error reading diskDataHandle vol header %s; error=%u\n", path, *ec); @@ -1060,7 +1178,7 @@ VGetVolume_r(Error * ec, VolId volumeId) Log("Volume %u: couldn't reread volume header\n", vp->hashid); FreeVolume(vp); - vp = 0; + vp = NULL; break; } } @@ -1068,7 +1186,7 @@ VGetVolume_r(Error * ec, VolId volumeId) if (vp->shuttingDown) { V8++; *ec = VNOVOL; - vp = 0; + vp = NULL; break; } if (programType == fileServer) { @@ -1811,11 +1929,11 @@ GetVolumeHeader(register Volume * vp) int old; static int everLogged = 0; - old = (vp->header != 0); /* old == volume already has a header */ + old = (vp->header != NULL); /* old == volume already has a header */ if (programType != fileServer) { if (!vp->header) { hd = (struct volHeader *)calloc(1, sizeof(*vp->header)); - assert(hd != 0); + assert(hd != NULL); vp->header = hd; hd->back = vp; } @@ -1946,3 +2064,4 @@ VPrintCacheStats(void) VPrintCacheStats_r(); VOL_UNLOCK; } + diff --git a/src/vol/volume.h b/src/vol/volume.h index 779e9b4c0..68fc0f98c 100644 --- a/src/vol/volume.h +++ b/src/vol/volume.h @@ -34,10 +34,10 @@ extern pthread_mutex_t vol_fsync_mutex; extern pthread_mutex_t vol_trans_mutex; extern pthread_cond_t vol_put_volume_cond; extern pthread_cond_t vol_sleep_cond; -#define VATTACH_LOCK \ - assert(pthread_mutex_lock(&vol_attach_mutex) == 0) -#define VATTACH_UNLOCK \ - assert(pthread_mutex_unlock(&vol_attach_mutex) == 0) +extern int vol_attach_threads; +/* this lock has been deprecated */ +#define VATTACH_LOCK +#define VATTACH_UNLOCK #define VOL_LOCK \ assert(pthread_mutex_lock(&vol_glock_mutex) == 0) #define VOL_UNLOCK \ -- 2.39.5