From: Thomas Leibovici Date: Thu, 13 Jun 2013 10:54:45 +0000 (+0200) Subject: LU-3365 lmv: support DNE with HSM. X-Git-Tag: 2.4.52~60 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=848f9e20320cb7c01eaf7f1b5c27f5efd54e4818;hp=1b2547843817b4b7adbeb87ea9b070d9cac35c90 LU-3365 lmv: support DNE with HSM. Send HSM requests to the appropriate MDT. Split lists of fids of HSM actions into one list per MDT. Move kuc registration/unregistration from MDC to LMV as this is not MDT related. Signed-off-by: Thomas Leibovici Change-Id: Iecba75dccbf15abc667dcca4800113c70a824d5c Reviewed-on: http://review.whamcloud.com/6714 Tested-by: Hudson Reviewed-by: John L. Hammond Reviewed-by: jacques-Charles Lafoucriere Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 0868280..c3a93bf 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -45,6 +45,7 @@ #include #include #include +#include #else #include #endif @@ -812,6 +813,125 @@ out_fid2path: RETURN(rc); } +static int lmv_hsm_req_count(struct lmv_obd *lmv, + const struct hsm_user_request *hur, + const struct lmv_tgt_desc *tgt_mds) +{ + int i, nr = 0; + struct lmv_tgt_desc *curr_tgt; + + /* count how many requests must be sent to the given target */ + for (i = 0; i < hur->hur_request.hr_itemcount; i++) { + curr_tgt = lmv_find_target(lmv, &hur->hur_user_item[i].hui_fid); + if (obd_uuid_equals(&curr_tgt->ltd_uuid, &tgt_mds->ltd_uuid)) + nr++; + } + return nr; +} + +static void lmv_hsm_req_build(struct lmv_obd *lmv, + struct hsm_user_request *hur_in, + const struct lmv_tgt_desc *tgt_mds, + struct hsm_user_request *hur_out) +{ + int i, nr_out; + struct lmv_tgt_desc *curr_tgt; + + /* build the hsm_user_request for the given target */ + hur_out->hur_request = hur_in->hur_request; + nr_out = 0; + for (i = 0; i < hur_in->hur_request.hr_itemcount; i++) { + curr_tgt = lmv_find_target(lmv, + &hur_in->hur_user_item[i].hui_fid); + if (obd_uuid_equals(&curr_tgt->ltd_uuid, &tgt_mds->ltd_uuid)) { + hur_out->hur_user_item[nr_out] = + hur_in->hur_user_item[i]; + nr_out++; + } + } + hur_out->hur_request.hr_itemcount = nr_out; + memcpy(hur_data(hur_out), hur_data(hur_in), + hur_in->hur_request.hr_data_len); +} + +static int lmv_hsm_ct_unregister(struct lmv_obd *lmv, unsigned int cmd, int len, + struct lustre_kernelcomm *lk, void *uarg) +{ + int i, rc = 0; + ENTRY; + + /* unregister request (call from llapi_hsm_copytool_fini) */ + for (i = 0; i < lmv->desc.ld_tgt_count; i++) { + /* best effort: try to clean as much as possible + * (continue on error) */ + obd_iocontrol(cmd, lmv->tgts[i]->ltd_exp, len, lk, uarg); + } + + /* Whatever the result, remove copytool from kuc groups. + * Unreached coordinators will get EPIPE on next requests + * and will unregister automatically. + */ + rc = libcfs_kkuc_group_rem(lk->lk_uid, lk->lk_group); + RETURN(rc); +} + +static int lmv_hsm_ct_register(struct lmv_obd *lmv, unsigned int cmd, int len, + struct lustre_kernelcomm *lk, void *uarg) +{ + struct file *filp; + int i, j, err; + int rc = 0; + bool any_set = false; + ENTRY; + + /* All or nothing: try to register to all MDS. + * In case of failure, unregister from previous MDS, + * except if it because of inactive target. */ + for (i = 0; i < lmv->desc.ld_tgt_count; i++) { + err = obd_iocontrol(cmd, lmv->tgts[i]->ltd_exp, + len, lk, uarg); + if (err) { + if (lmv->tgts[i]->ltd_active) { + /* permanent error */ + CERROR("error: iocontrol MDC %s on MDT" + "idx %d cmd %x: err = %d\n", + lmv->tgts[i]->ltd_uuid.uuid, + i, cmd, err); + rc = err; + lk->lk_flags |= LK_FLG_STOP; + /* unregister from previous MDS */ + for (j = 0; j < i; j++) + obd_iocontrol(cmd, + lmv->tgts[j]->ltd_exp, + len, lk, uarg); + RETURN(rc); + } + /* else: transient error. + * kuc will register to the missing MDT + * when it is back */ + } else { + any_set = true; + } + } + + if (!any_set) + /* no registration done: return error */ + RETURN(-ENOTCONN); + + /* at least one registration done, with no failure */ + filp = fget(lk->lk_wfd); + if (filp == NULL) { + RETURN(-EBADF); + } + rc = libcfs_kkuc_group_add(filp, lk->lk_uid, lk->lk_group, lk->lk_data); + if (rc != 0 && filp != NULL) + fput(filp); + RETURN(rc); +} + + + + static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, int len, void *karg, void *uarg) { @@ -937,7 +1057,77 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, } case LL_IOC_HSM_STATE_GET: case LL_IOC_HSM_STATE_SET: - case LL_IOC_HSM_ACTION: + case LL_IOC_HSM_ACTION: { + struct md_op_data *op_data = karg; + struct lmv_tgt_desc *tgt; + + tgt = lmv_find_target(lmv, &op_data->op_fid1); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); + + if (tgt->ltd_exp == NULL) + RETURN(-EINVAL); + + rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); + break; + } + case LL_IOC_HSM_PROGRESS: { + const struct hsm_progress_kernel *hpk = karg; + struct lmv_tgt_desc *tgt; + + tgt = lmv_find_target(lmv, &hpk->hpk_fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); + rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); + break; + } + case LL_IOC_HSM_REQUEST: { + struct hsm_user_request *hur = karg; + struct lmv_tgt_desc *tgt; + unsigned int reqcount = hur->hur_request.hr_itemcount; + + if (reqcount == 0) + RETURN(0); + + /* if the request is about a single fid + * or if there is a single MDS, no need to split + * the request. */ + if (reqcount == 1 || count == 1) { + tgt = lmv_find_target(lmv, + &hur->hur_user_item[0].hui_fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); + rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); + } else { + /* split fid list to their respective MDS */ + for (i = 0; i < count; i++) { + unsigned int nr, reqlen; + int rc1; + struct hsm_user_request *req; + + nr = lmv_hsm_req_count(lmv, hur, lmv->tgts[i]); + if (nr == 0) /* nothing for this MDS */ + continue; + + /* build a request with fids for this MDS */ + reqlen = offsetof(typeof(*hur), + hur_user_item[nr]) + + hur->hur_request.hr_data_len; + OBD_ALLOC_LARGE(req, reqlen); + if (req == NULL) + RETURN(-ENOMEM); + + lmv_hsm_req_build(lmv, hur, lmv->tgts[i], req); + + rc1 = obd_iocontrol(cmd, lmv->tgts[i]->ltd_exp, + reqlen, req, uarg); + if (rc1 != 0 && rc == 0) + rc = rc1; + OBD_FREE_LARGE(req, reqlen); + } + } + break; + } case LL_IOC_LOV_SWAP_LAYOUTS: { struct md_op_data *op_data = karg; struct lmv_tgt_desc *tgt1, *tgt2; @@ -960,6 +1150,14 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, rc = obd_iocontrol(cmd, tgt1->ltd_exp, len, karg, uarg); break; } + case LL_IOC_HSM_CT_START: { + struct lustre_kernelcomm *lk = karg; + if (lk->lk_flags & LK_FLG_STOP) + rc = lmv_hsm_ct_unregister(lmv, cmd, len, lk, uarg); + else + rc = lmv_hsm_ct_register(lmv, cmd, len, lk, uarg); + break; + } default: for (i = 0; i < count; i++) { struct obd_device *mdc_obd; diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 2d2167c..50f9600 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1824,6 +1824,9 @@ static int mdc_iocontrol(unsigned int cmd, struct obd_export *exp, int len, GOTO(out, rc); case LL_IOC_HSM_CT_START: rc = mdc_ioc_hsm_ct_start(exp, karg); + /* ignore if it was already registered on this MDS. */ + if (rc == -EEXIST) + rc = 0; GOTO(out, rc); case LL_IOC_HSM_PROGRESS: rc = mdc_ioc_hsm_progress(exp, karg); @@ -2043,19 +2046,10 @@ static int mdc_ioc_hsm_ct_start(struct obd_export *exp, lk->lk_uid, lk->lk_group, lk->lk_flags); if (lk->lk_flags & LK_FLG_STOP) { - rc = libcfs_kkuc_group_rem(lk->lk_uid, lk->lk_group); /* Unregister with the coordinator */ - if (rc == 0) - rc = mdc_ioc_hsm_ct_unregister(imp); + rc = mdc_ioc_hsm_ct_unregister(imp); } else { - struct file *fp = fget(lk->lk_wfd); - - rc = libcfs_kkuc_group_add(fp, lk->lk_uid, lk->lk_group, - lk->lk_data); - if (rc && fp) - fput(fp); - if (rc == 0) - rc = mdc_ioc_hsm_ct_register(imp, archive); + rc = mdc_ioc_hsm_ct_register(imp, archive); } return rc; @@ -2380,7 +2374,7 @@ static int mdc_import_event(struct obd_device *obd, struct obd_import *imp, } case IMP_EVENT_ACTIVE: rc = obd_notify_observer(obd, obd, OBD_NOTIFY_ACTIVE, NULL); - /* restore re-establish kuc registration after reconnecting */ + /* redo the kuc registration after reconnecting */ if (rc == 0) rc = mdc_kuc_reregister(imp); break; diff --git a/lustre/utils/liblustreapi_hsm.c b/lustre/utils/liblustreapi_hsm.c index 4d32588..c4b5437 100644 --- a/lustre/utils/liblustreapi_hsm.c +++ b/lustre/utils/liblustreapi_hsm.c @@ -128,18 +128,18 @@ int llapi_hsm_copytool_start(struct hsm_copytool_private **priv, char *fsname, ct->kuc.lk_data = ct->archives; rc = root_ioctl(ct->fsname, LL_IOC_HSM_CT_START, &(ct->kuc), NULL, WANT_ERROR); - /* ignore if it was already registered on coordinator */ - if (rc == -EEXIST) - rc = 0; /* Only the kernel reference keeps the write side open */ close(ct->kuc.lk_wfd); ct->kuc.lk_wfd = 0; if (rc < 0) - goto out_err; + goto out_kuc; *priv = ct; return 0; +out_kuc: + /* cleanup the kuc channel */ + libcfs_ukuc_stop(&ct->kuc); out_err: if (ct->fsname) free(ct->fsname);