From: shadow Date: Tue, 30 Sep 2008 16:15:23 +0000 (+0000) Subject: parallel llog init X-Git-Tag: v1_9_80~41 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=f36d9a3f365f9548b1c07dc36c0d9cee4a13ff01 parallel llog init Branch HEAD b=16679 i=umka --- diff --git a/lustre/include/lustre_log.h b/lustre/include/lustre_log.h index 1ec4384..d1b8e82 100644 --- a/lustre/include/lustre_log.h +++ b/lustre/include/lustre_log.h @@ -184,7 +184,7 @@ int llog_obd_origin_add(struct llog_ctxt *ctxt, struct llog_cookie *logcookies, int numcookies); int llog_cat_initialize(struct obd_device *obd, struct obd_llog_group *olg, - int count, struct obd_uuid *uuid); + int idx, struct obd_uuid *uuid); int obd_llog_init(struct obd_device *obd, struct obd_llog_group *olg, struct obd_device *disk_obd, int count, struct llog_catid *logid, struct obd_uuid *uuid); @@ -247,7 +247,8 @@ struct llog_operations { /* llog_lvfs.c */ extern struct llog_operations llog_lvfs_ops; int llog_get_cat_list(struct obd_device *obd, struct obd_device *disk_obd, - char *name, int count, struct llog_catid *idarray); + char *name, int idx, int count, + struct llog_catid *idarray); struct llog_ctxt { int loc_idx; /* my index the obd array of ctxt's */ diff --git a/lustre/lov/lov_log.c b/lustre/lov/lov_log.c index b97a2d8..e7210af 100644 --- a/lustre/lov/lov_log.c +++ b/lustre/lov/lov_log.c @@ -198,14 +198,17 @@ static struct llog_operations lov_size_repl_logops = { }; int lov_llog_init(struct obd_device *obd, struct obd_llog_group *olg, - struct obd_device *tgt, int count, struct llog_catid *logid, + struct obd_device *tgt, int count, struct llog_catid *logid, struct obd_uuid *uuid) { struct lov_obd *lov = &obd->u.lov; struct obd_device *child; - int i, rc = 0, err = 0; + int i, rc = 0; ENTRY; + /* allow init only one target at one time */ + LASSERT(uuid); + LASSERT(olg == &obd->obd_olg); rc = llog_setup(obd, olg, LLOG_MDS_OST_ORIG_CTXT, tgt, 0, NULL, &lov_mds_ost_orig_logops); @@ -219,25 +222,23 @@ int lov_llog_init(struct obd_device *obd, struct obd_llog_group *olg, lov_getref(obd); /* count may not match lov->desc.ld_tgt_count during dynamic ost add */ - for (i = 0; i < count; i++) { + for (i = 0; i < lov->desc.ld_tgt_count; i++) { if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active) continue; - if (uuid && !obd_uuid_equals(uuid, &lov->lov_tgts[i]->ltd_uuid)) + if (!obd_uuid_equals(uuid, &lov->lov_tgts[i]->ltd_uuid)) continue; - CDEBUG(D_CONFIG, "init %d/%d\n", i, count); + LASSERT(lov->lov_tgts[i]->ltd_exp); child = lov->lov_tgts[i]->ltd_exp->exp_obd; - rc = obd_llog_init(child, &child->obd_olg, tgt, 1, logid + i, uuid); - if (rc) { + rc = obd_llog_init(child, &child->obd_olg, tgt, 1, logid, uuid); + if (rc) CERROR("error osc_llog_init idx %d osc '%s' tgt '%s' " "(rc=%d)\n", i, child->obd_name, tgt->obd_name, rc); - if (!err) - err = rc; - } + break; } lov_putref(obd); - RETURN(err); + RETURN(rc); } int lov_llog_finish(struct obd_device *obd, int count) diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index ee256b2..27e6bb0 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -232,8 +232,6 @@ int mds_postrecov(struct obd_device *obd) RETURN(0); LASSERT(!obd->obd_recovering); - LASSERT(!llog_ctxt_null(obd, LLOG_MDS_OST_ORIG_CTXT)); - /* clean PENDING dir */ #if 0 if (strncmp(obd->obd_name, MDD_OBD_NAME, strlen(MDD_OBD_NAME))) diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 07444ac..9073cc8 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -341,7 +341,8 @@ static int mds_lov_set_one_nextid(struct obd_device *obd, __u32 idx, obd_id *id) } /* Update the lov desc for a new size lov. */ -static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) +static int mds_lov_update_desc(struct obd_device *obd, int idx, + struct obd_uuid *uuid) { struct mds_obd *mds = &obd->u.mds; struct lov_desc *ld; @@ -353,7 +354,7 @@ static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) if (!ld) RETURN(-ENOMEM); - rc = obd_get_info(lov, sizeof(KEY_LOVDESC), KEY_LOVDESC, + rc = obd_get_info(mds->mds_osc_exp, sizeof(KEY_LOVDESC), KEY_LOVDESC, &valsize, ld, NULL); if (rc) GOTO(out, rc); @@ -376,8 +377,9 @@ static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) /* If we added a target we have to reconnect the llogs */ /* We only _need_ to do this at first add (idx), or the first time after recovery. However, it should now be safe to call anytime. */ - rc = llog_cat_initialize(obd, &obd->obd_olg, - mds->mds_lov_desc.ld_tgt_count, NULL); + rc = llog_cat_initialize(obd, &obd->obd_olg, idx, uuid); + if (rc) + GOTO(out, rc); /*XXX this notifies the MDD until lov handling use old mds code */ if (obd->obd_upcall.onu_owner) { @@ -406,7 +408,7 @@ static int mds_lov_update_mds(struct obd_device *obd, /* Don't let anyone else mess with mds_lov_objids now */ mutex_down(&obd->obd_dev_sem); - rc = mds_lov_update_desc(obd, mds->mds_osc_exp); + rc = mds_lov_update_desc(obd, idx, &watched->u.cli.cl_target_uuid); if (rc) GOTO(out, rc); @@ -508,34 +510,6 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) CERROR("cannot read %s: rc = %d\n", "lov_objids", rc); GOTO(err_reg, rc); } - - rc = mds_lov_update_desc(obd, mds->mds_osc_exp); - if (rc) - GOTO(err_reg, rc); - - /* tgt_count may be 0! */ - rc = llog_cat_initialize(obd, &obd->obd_olg, - mds->mds_lov_desc.ld_tgt_count, NULL); - if (rc) { - CERROR("failed to initialize catalog %d\n", rc); - GOTO(err_reg, rc); - } - - /* If we're mounting this code for the first time on an existing FS, - * we need to populate the objids array from the real OST values */ - if (mds->mds_lov_desc.ld_tgt_count > mds->mds_lov_objid_count) { - __u32 i = mds->mds_lov_objid_count; - for(; i <= mds->mds_lov_desc.ld_tgt_count; i++) { - rc = mds_lov_get_objid(obd, i); - if (rc != 0) - break; - } - if (rc == 0) - rc = mds_lov_write_objids(obd); - if (rc) - CERROR("got last objids from OSTs, but error " - "in update objids file: %d\n", rc); - } mutex_up(&obd->obd_dev_sem); /* I want to see a callback happen when the OBD moves to a @@ -815,22 +789,14 @@ int mds_notify(struct obd_device *obd, struct obd_device *watched, after the mdt in the config log. They didn't make it into mds_lov_connect. */ mutex_down(&obd->obd_dev_sem); - rc = mds_lov_update_desc(obd, obd->u.mds.mds_osc_exp); - if (rc) { - mutex_up(&obd->obd_dev_sem); - RETURN(rc); - } - /* We should update init llog here too for replay unlink and - * possiable llog init race when recovery complete */ - llog_cat_initialize(obd, &obd->obd_olg, - obd->u.mds.mds_lov_desc.ld_tgt_count, - &watched->u.cli.cl_target_uuid); + rc = mds_lov_update_desc(obd, *(__u32 *)data, + &watched->u.cli.cl_target_uuid); mutex_up(&obd->obd_dev_sem); - mds_allow_cli(obd, CONFIG_SYNC); + if (rc == 0) + mds_allow_cli(obd, CONFIG_SYNC); RETURN(rc); } - LASSERT(!llog_ctxt_null(obd, LLOG_MDS_OST_ORIG_CTXT)); rc = mds_lov_start_synchronize(obd, watched, data, !(ev == OBD_NOTIFY_SYNC)); diff --git a/lustre/obdclass/llog_internal.h b/lustre/obdclass/llog_internal.h index a992c1b..4c31016 100644 --- a/lustre/obdclass/llog_internal.h +++ b/lustre/obdclass/llog_internal.h @@ -49,7 +49,9 @@ struct llog_process_info { }; int llog_put_cat_list(struct obd_device *obd, struct obd_device *disk_obd, - char *name, int count, struct llog_catid *idarray); + char *name, int idx, int count, + struct llog_catid *idarray); + int llog_cat_id2handle(struct llog_handle *cathandle, struct llog_handle **res, struct llog_logid *logid); int class_config_dump_handler(struct llog_handle * handle, diff --git a/lustre/obdclass/llog_ioctl.c b/lustre/obdclass/llog_ioctl.c index d1c3fa5..48b9cc1 100644 --- a/lustre/obdclass/llog_ioctl.c +++ b/lustre/obdclass/llog_ioctl.c @@ -432,12 +432,11 @@ int llog_catalog_list(struct obd_device *obd, int count, ENTRY; size = sizeof(*idarray) * count; - OBD_ALLOC(idarray, size); + OBD_VMALLOC(idarray, size); if (!idarray) RETURN(-ENOMEM); - memset(idarray, 0, size); - rc = llog_get_cat_list(obd, obd, name, count, idarray); + rc = llog_get_cat_list(obd, obd, name, 0, count, idarray); if (rc) { OBD_FREE(idarray, size); RETURN(rc); @@ -457,7 +456,7 @@ int llog_catalog_list(struct obd_device *obd, int count, break; } } - OBD_FREE(idarray, size); + OBD_VFREE(idarray, size); RETURN(0); } diff --git a/lustre/obdclass/llog_lvfs.c b/lustre/obdclass/llog_lvfs.c index 4f3c913..f2a5cd8 100644 --- a/lustre/obdclass/llog_lvfs.c +++ b/lustre/obdclass/llog_lvfs.c @@ -768,13 +768,13 @@ static int llog_lvfs_destroy(struct llog_handle *handle) /* reads the catalog list */ int llog_get_cat_list(struct obd_device *obd, struct obd_device *disk_obd, - char *name, int count, struct llog_catid *idarray) + char *name, int idx, int count, struct llog_catid *idarray) { struct lvfs_run_ctxt saved; struct l_file *file; - int rc; + int rc, rc1 = 0; int size = sizeof(*idarray) * count; - loff_t off = 0; + loff_t off = idx * sizeof(*idarray); ENTRY; if (!count) @@ -788,7 +788,7 @@ int llog_get_cat_list(struct obd_device *obd, struct obd_device *disk_obd, name, rc); GOTO(out, rc); } - + if (!S_ISREG(file->f_dentry->d_inode->i_mode)) { CERROR("%s is not a regular file!: mode = %o\n", name, file->f_dentry->d_inode->i_mode); @@ -798,6 +798,11 @@ int llog_get_cat_list(struct obd_device *obd, struct obd_device *disk_obd, CDEBUG(D_CONFIG, "cat list: disk size=%d, read=%d\n", (int)i_size_read(file->f_dentry->d_inode), size); + /* read for new ost index or for empty file */ + memset(idarray, 0, size); + if (i_size_read(file->f_dentry->d_inode) < off) + GOTO(out, rc = 0); + rc = fsfilt_read_record(disk_obd, file, idarray, size, &off); if (rc) { CERROR("OBD filter: error reading %s: rc %d\n", name, rc); @@ -808,22 +813,24 @@ int llog_get_cat_list(struct obd_device *obd, struct obd_device *disk_obd, out: pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); if (file && !IS_ERR(file)) - rc = filp_close(file, 0); + rc1 = filp_close(file, 0); + if (rc == 0) + rc = rc1; return rc; } EXPORT_SYMBOL(llog_get_cat_list); /* writes the cat list */ int llog_put_cat_list(struct obd_device *obd, struct obd_device *disk_obd, - char *name, int count, struct llog_catid *idarray) + char *name, int idx, int count, struct llog_catid *idarray) { struct lvfs_run_ctxt saved; struct l_file *file; - int rc; + int rc, rc1 = 0; int size = sizeof(*idarray) * count; - loff_t off = 0; + loff_t off = idx * sizeof(*idarray); - if (!count) + if (!count) return (0); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); @@ -843,7 +850,7 @@ int llog_put_cat_list(struct obd_device *obd, struct obd_device *disk_obd, rc = fsfilt_write_record(disk_obd, file, idarray, size, &off, 1); if (rc) { - CDEBUG(D_INODE,"OBD filter: error reading %s: rc %d\n", + CDEBUG(D_INODE,"OBD filter: error writeing %s: rc %d\n", name, rc); GOTO(out, rc); } @@ -851,7 +858,10 @@ int llog_put_cat_list(struct obd_device *obd, struct obd_device *disk_obd, out: pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); if (file && !IS_ERR(file)) - rc = filp_close(file, 0); + rc1 = filp_close(file, 0); + + if (rc == 0) + rc = rc1; RETURN(rc); } @@ -920,14 +930,14 @@ static int llog_lvfs_destroy(struct llog_handle *handle) } int llog_get_cat_list(struct obd_device *obd, struct obd_device *disk_obd, - char *name, int count, struct llog_catid *idarray) + char *name, int idx, int count, struct llog_catid *idarray) { LBUG(); return 0; } int llog_put_cat_list(struct obd_device *obd, struct obd_device *disk_obd, - char *name, int count, struct llog_catid *idarray) + char *name, int idx, int count, struct llog_catid *idarray) { LBUG(); return 0; diff --git a/lustre/obdclass/llog_obd.c b/lustre/obdclass/llog_obd.c index 2e4b3a2..0542aa0 100644 --- a/lustre/obdclass/llog_obd.c +++ b/lustre/obdclass/llog_obd.c @@ -149,33 +149,11 @@ int llog_setup(struct obd_device *obd, struct obd_llog_group *olg, int index, RETURN(-EFAULT); LASSERT(olg != NULL); - ctxt = llog_group_get_ctxt(olg, index); - /* in some recovery cases, obd_llog_ctxt might already be set, - * but llogs might still be zero, for example in obd_filter recovery */ - if (ctxt) { - /* mds_lov_update_mds might call here multiple times. So if the - llog is already set up then don't to do it again. */ - CDEBUG(D_CONFIG, "obd %s ctxt %d already set up\n", - obd->obd_name, index); - LASSERT(ctxt->loc_olg == olg); - LASSERT(ctxt->loc_obd == obd); - LASSERT(ctxt->loc_exp == disk_obd->obd_self_export); - LASSERT(ctxt->loc_logops == op); - llog_ctxt_put(ctxt); - GOTO(out, rc = 0); - } ctxt = llog_new_ctxt(obd); if (!ctxt) GOTO(out, rc = -ENOMEM); - rc = llog_group_set_ctxt(olg, ctxt, index); - if (rc) { - llog_ctxt_destroy(ctxt); - if (rc == -EEXIST) - rc = 0; - GOTO(out, rc); - } ctxt->loc_obd = obd; ctxt->loc_exp = class_export_get(disk_obd->obd_self_export); ctxt->loc_olg = olg; @@ -183,6 +161,28 @@ int llog_setup(struct obd_device *obd, struct obd_llog_group *olg, int index, ctxt->loc_logops = op; sema_init(&ctxt->loc_sem, 1); + rc = llog_group_set_ctxt(olg, ctxt, index); + if (rc) { + llog_ctxt_destroy(ctxt); + if (rc == -EEXIST) { + /* sanity check */ + ctxt = llog_group_get_ctxt(olg, index); + + /* mds_lov_update_mds might call here multiple times. + * So if the llog is already set up then don't to do + * it again. */ + CDEBUG(D_CONFIG, "obd %s ctxt %d already set up\n", + obd->obd_name, index); + LASSERT(ctxt->loc_olg == olg); + LASSERT(ctxt->loc_obd == obd); + LASSERT(ctxt->loc_exp == disk_obd->obd_self_export); + LASSERT(ctxt->loc_logops == op); + llog_ctxt_put(ctxt); + rc = 0; + } + GOTO(out, rc); + } + if (op->lop_setup) rc = op->lop_setup(obd, olg, index, disk_obd, count, logid); @@ -407,41 +407,32 @@ int llog_obd_origin_add(struct llog_ctxt *ctxt, EXPORT_SYMBOL(llog_obd_origin_add); int llog_cat_initialize(struct obd_device *obd, struct obd_llog_group *olg, - int count, struct obd_uuid *uuid) + int idx, struct obd_uuid *uuid) { char name[32] = CATLIST; - struct llog_catid *idarray = NULL; - int size = sizeof(*idarray) * count; + struct llog_catid idarray; int rc; ENTRY; - if (count) { - OBD_VMALLOC(idarray, size); - if (!idarray) - RETURN(-ENOMEM); - } - - rc = llog_get_cat_list(obd, obd, name, count, idarray); + rc = llog_get_cat_list(obd, obd, name, idx, 1, &idarray); if (rc) { CERROR("rc: %d\n", rc); GOTO(out, rc); } - rc = obd_llog_init(obd, olg, obd, count, idarray, uuid); + rc = obd_llog_init(obd, olg, obd, 1, &idarray, uuid); if (rc) { CERROR("rc: %d\n", rc); GOTO(out, rc); } - rc = llog_put_cat_list(obd, obd, name, count, idarray); + rc = llog_put_cat_list(obd, obd, name, idx, 1, &idarray); if (rc) { CERROR("rc: %d\n", rc); GOTO(out, rc); } out: - if (idarray) - OBD_VFREE(idarray, size); RETURN(rc); } EXPORT_SYMBOL(llog_cat_initialize); diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 34518f3..2912dc3 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -2041,7 +2041,7 @@ int filter_common_setup(struct obd_device *obd, struct lustre_cfg* lcfg, ptlrpc_init_client(LDLM_CB_REQUEST_PORTAL, LDLM_CB_REPLY_PORTAL, "filter_ldlm_cb_client", &obd->obd_ldlm_client); - rc = llog_cat_initialize(obd, &obd->obd_olg, 1, NULL); + rc = obd_llog_init(obd, &obd->obd_olg, obd, 1, NULL, NULL); if (rc) { CERROR("failed to setup llogging subsystems\n"); GOTO(err_post, rc); diff --git a/lustre/ptlrpc/llog_server.c b/lustre/ptlrpc/llog_server.c index da21e87..93c03ec 100644 --- a/lustre/ptlrpc/llog_server.c +++ b/lustre/ptlrpc/llog_server.c @@ -560,7 +560,6 @@ static int llog_catinfo_deletions(struct obd_device *obd, char *buf, struct lvfs_run_ctxt saved; int size, i, count; struct llog_catid *idarray; - struct llog_logid *id; char name[32] = CATLIST; struct cb_data data; struct llog_ctxt *ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); @@ -572,11 +571,11 @@ static int llog_catinfo_deletions(struct obd_device *obd, char *buf, count = mds->mds_lov_desc.ld_tgt_count; size = sizeof(*idarray) * count; - OBD_ALLOC(idarray, size); + OBD_VMALLOC(idarray, size); if (!idarray) GOTO(release_ctxt, rc = -ENOMEM); - rc = llog_get_cat_list(obd, obd, name, count, idarray); + rc = llog_get_cat_list(obd, obd, name, 0, count, idarray); if (rc) GOTO(out_free, rc); @@ -588,8 +587,7 @@ static int llog_catinfo_deletions(struct obd_device *obd, char *buf, for (i = 0; i < count; i++) { int l, index, uncanceled = 0; - id = &idarray[i].lci_logid; - rc = llog_create(ctxt, &handle, id, NULL); + rc = llog_create(ctxt, &handle, &idarray[i].lci_logid, NULL); if (rc) GOTO(out_pop, rc); rc = llog_init_handle(handle, 0, NULL); @@ -604,8 +602,9 @@ static int llog_catinfo_deletions(struct obd_device *obd, char *buf, l = snprintf(data.out, data.remains, "\n[Catlog ID]: #"LPX64"#"LPX64"#%08x " "[Log Count]: %d\n", - id->lgl_oid, id->lgl_ogr, id->lgl_ogen, - uncanceled); + idarray[i].lci_logid.lgl_oid, + idarray[i].lci_logid.lgl_ogr, + idarray[i].lci_logid.lgl_ogen, uncanceled); data.out += l; data.remains -= l; @@ -617,14 +616,15 @@ static int llog_catinfo_deletions(struct obd_device *obd, char *buf, if (data.remains <= 0) break; } + EXIT; out_pop: pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL); out_free: - OBD_FREE(idarray, size); + OBD_VFREE(idarray, size); release_ctxt: llog_ctxt_put(ctxt); - RETURN(rc); + return(rc); } int llog_catinfo(struct ptlrpc_request *req) diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 5f4ce6e..c14cf9a 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -1073,6 +1073,9 @@ test_33a() { # bug 12333, was test_33 mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || rc=2 echo "ok." + cp /etc/hosts $MOUNT2/ || rc=3 + $LFS getstripe $MOUNT2/hosts + umount -d $MOUNT2 stop fs2ost -f stop fs2mds -f