Whamcloud - gitweb
LU-3365 lmv: support DNE with HSM. 14/6714/6
authorThomas Leibovici <thomas.leibovici@cea.fr>
Thu, 13 Jun 2013 10:54:45 +0000 (12:54 +0200)
committerOleg Drokin <oleg.drokin@intel.com>
Fri, 28 Jun 2013 19:21:32 +0000 (19:21 +0000)
Send HSM requests to the appropriate MDT. Split lists of fids of HSM
actions into one list per MDT.
Move kuc registration/unregistration from MDC to LMV as this is not
MDT related.

Signed-off-by: Thomas Leibovici <thomas.leibovici@cea.fr>
Change-Id: Iecba75dccbf15abc667dcca4800113c70a824d5c
Reviewed-on: http://review.whamcloud.com/6714
Tested-by: Hudson
Reviewed-by: John L. Hammond <john.hammond@intel.com>
Reviewed-by: jacques-Charles Lafoucriere <jacques-charles.lafoucriere@cea.fr>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
lustre/lmv/lmv_obd.c
lustre/mdc/mdc_request.c
lustre/utils/liblustreapi_hsm.c

index 0868280..c3a93bf 100644 (file)
@@ -45,6 +45,7 @@
 #include <asm/div64.h>
 #include <linux/seq_file.h>
 #include <linux/namei.h>
+#include <asm/uaccess.h>
 #else
 #include <liblustre.h>
 #endif
@@ -812,6 +813,125 @@ out_fid2path:
        RETURN(rc);
 }
 
+static int lmv_hsm_req_count(struct lmv_obd *lmv,
+                            const struct hsm_user_request *hur,
+                            const struct lmv_tgt_desc *tgt_mds)
+{
+       int                     i, nr = 0;
+       struct lmv_tgt_desc    *curr_tgt;
+
+       /* count how many requests must be sent to the given target */
+       for (i = 0; i < hur->hur_request.hr_itemcount; i++) {
+               curr_tgt = lmv_find_target(lmv, &hur->hur_user_item[i].hui_fid);
+               if (obd_uuid_equals(&curr_tgt->ltd_uuid, &tgt_mds->ltd_uuid))
+                       nr++;
+       }
+       return nr;
+}
+
+static void lmv_hsm_req_build(struct lmv_obd *lmv,
+                             struct hsm_user_request *hur_in,
+                             const struct lmv_tgt_desc *tgt_mds,
+                             struct hsm_user_request *hur_out)
+{
+       int                     i, nr_out;
+       struct lmv_tgt_desc    *curr_tgt;
+
+       /* build the hsm_user_request for the given target */
+       hur_out->hur_request = hur_in->hur_request;
+       nr_out = 0;
+       for (i = 0; i < hur_in->hur_request.hr_itemcount; i++) {
+               curr_tgt = lmv_find_target(lmv,
+                                       &hur_in->hur_user_item[i].hui_fid);
+               if (obd_uuid_equals(&curr_tgt->ltd_uuid, &tgt_mds->ltd_uuid)) {
+                       hur_out->hur_user_item[nr_out] =
+                               hur_in->hur_user_item[i];
+                       nr_out++;
+               }
+       }
+       hur_out->hur_request.hr_itemcount = nr_out;
+       memcpy(hur_data(hur_out), hur_data(hur_in),
+              hur_in->hur_request.hr_data_len);
+}
+
+static int lmv_hsm_ct_unregister(struct lmv_obd *lmv, unsigned int cmd, int len,
+                                struct lustre_kernelcomm *lk, void *uarg)
+{
+       int     i, rc = 0;
+       ENTRY;
+
+       /* unregister request (call from llapi_hsm_copytool_fini) */
+       for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
+               /* best effort: try to clean as much as possible
+                * (continue on error) */
+               obd_iocontrol(cmd, lmv->tgts[i]->ltd_exp, len, lk, uarg);
+       }
+
+       /* Whatever the result, remove copytool from kuc groups.
+        * Unreached coordinators will get EPIPE on next requests
+        * and will unregister automatically.
+        */
+       rc = libcfs_kkuc_group_rem(lk->lk_uid, lk->lk_group);
+       RETURN(rc);
+}
+
+static int lmv_hsm_ct_register(struct lmv_obd *lmv, unsigned int cmd, int len,
+                              struct lustre_kernelcomm *lk, void *uarg)
+{
+       struct file     *filp;
+       int              i, j, err;
+       int              rc = 0;
+       bool             any_set = false;
+       ENTRY;
+
+       /* All or nothing: try to register to all MDS.
+        * In case of failure, unregister from previous MDS,
+        * except if it because of inactive target. */
+       for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
+               err = obd_iocontrol(cmd, lmv->tgts[i]->ltd_exp,
+                                  len, lk, uarg);
+               if (err) {
+                       if (lmv->tgts[i]->ltd_active) {
+                               /* permanent error */
+                               CERROR("error: iocontrol MDC %s on MDT"
+                                      "idx %d cmd %x: err = %d\n",
+                                       lmv->tgts[i]->ltd_uuid.uuid,
+                                       i, cmd, err);
+                               rc = err;
+                               lk->lk_flags |= LK_FLG_STOP;
+                               /* unregister from previous MDS */
+                               for (j = 0; j < i; j++)
+                                       obd_iocontrol(cmd,
+                                                 lmv->tgts[j]->ltd_exp,
+                                                 len, lk, uarg);
+                               RETURN(rc);
+                       }
+                       /* else: transient error.
+                        * kuc will register to the missing MDT
+                        * when it is back */
+               } else {
+                       any_set = true;
+               }
+       }
+
+       if (!any_set)
+               /* no registration done: return error */
+               RETURN(-ENOTCONN);
+
+       /* at least one registration done, with no failure */
+       filp = fget(lk->lk_wfd);
+       if (filp == NULL) {
+               RETURN(-EBADF);
+       }
+       rc = libcfs_kkuc_group_add(filp, lk->lk_uid, lk->lk_group, lk->lk_data);
+       if (rc != 0 && filp != NULL)
+               fput(filp);
+       RETURN(rc);
+}
+
+
+
+
 static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
                          int len, void *karg, void *uarg)
 {
@@ -937,7 +1057,77 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
        }
        case LL_IOC_HSM_STATE_GET:
        case LL_IOC_HSM_STATE_SET:
-       case LL_IOC_HSM_ACTION:
+       case LL_IOC_HSM_ACTION: {
+               struct md_op_data       *op_data = karg;
+               struct lmv_tgt_desc     *tgt;
+
+               tgt = lmv_find_target(lmv, &op_data->op_fid1);
+               if (IS_ERR(tgt))
+                               RETURN(PTR_ERR(tgt));
+
+               if (tgt->ltd_exp == NULL)
+                               RETURN(-EINVAL);
+
+               rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg);
+               break;
+       }
+       case LL_IOC_HSM_PROGRESS: {
+               const struct hsm_progress_kernel *hpk = karg;
+               struct lmv_tgt_desc     *tgt;
+
+               tgt = lmv_find_target(lmv, &hpk->hpk_fid);
+               if (IS_ERR(tgt))
+                       RETURN(PTR_ERR(tgt));
+               rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg);
+               break;
+       }
+       case LL_IOC_HSM_REQUEST: {
+               struct hsm_user_request *hur = karg;
+               struct lmv_tgt_desc     *tgt;
+               unsigned int reqcount = hur->hur_request.hr_itemcount;
+
+               if (reqcount == 0)
+                       RETURN(0);
+
+               /* if the request is about a single fid
+                * or if there is a single MDS, no need to split
+                * the request. */
+               if (reqcount == 1 || count == 1) {
+                       tgt = lmv_find_target(lmv,
+                                             &hur->hur_user_item[0].hui_fid);
+                       if (IS_ERR(tgt))
+                               RETURN(PTR_ERR(tgt));
+                       rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg);
+               } else {
+                       /* split fid list to their respective MDS */
+                       for (i = 0; i < count; i++) {
+                               unsigned int            nr, reqlen;
+                               int                     rc1;
+                               struct hsm_user_request *req;
+
+                               nr = lmv_hsm_req_count(lmv, hur, lmv->tgts[i]);
+                               if (nr == 0) /* nothing for this MDS */
+                                       continue;
+
+                               /* build a request with fids for this MDS */
+                               reqlen = offsetof(typeof(*hur),
+                                                 hur_user_item[nr])
+                                        + hur->hur_request.hr_data_len;
+                               OBD_ALLOC_LARGE(req, reqlen);
+                               if (req == NULL)
+                                       RETURN(-ENOMEM);
+
+                               lmv_hsm_req_build(lmv, hur, lmv->tgts[i], req);
+
+                               rc1 = obd_iocontrol(cmd, lmv->tgts[i]->ltd_exp,
+                                                   reqlen, req, uarg);
+                               if (rc1 != 0 && rc == 0)
+                                       rc = rc1;
+                               OBD_FREE_LARGE(req, reqlen);
+                       }
+               }
+               break;
+       }
        case LL_IOC_LOV_SWAP_LAYOUTS: {
                struct md_op_data       *op_data = karg;
                struct lmv_tgt_desc     *tgt1, *tgt2;
@@ -960,6 +1150,14 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
                rc = obd_iocontrol(cmd, tgt1->ltd_exp, len, karg, uarg);
                break;
        }
+       case LL_IOC_HSM_CT_START: {
+               struct lustre_kernelcomm *lk = karg;
+               if (lk->lk_flags & LK_FLG_STOP)
+                       rc = lmv_hsm_ct_unregister(lmv, cmd, len, lk, uarg);
+               else
+                       rc = lmv_hsm_ct_register(lmv, cmd, len, lk, uarg);
+               break;
+       }
        default:
                for (i = 0; i < count; i++) {
                        struct obd_device *mdc_obd;
index 2d2167c..50f9600 100644 (file)
@@ -1824,6 +1824,9 @@ static int mdc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
                GOTO(out, rc);
        case LL_IOC_HSM_CT_START:
                rc = mdc_ioc_hsm_ct_start(exp, karg);
+               /* ignore if it was already registered on this MDS. */
+               if (rc == -EEXIST)
+                       rc = 0;
                GOTO(out, rc);
        case LL_IOC_HSM_PROGRESS:
                rc = mdc_ioc_hsm_progress(exp, karg);
@@ -2043,19 +2046,10 @@ static int mdc_ioc_hsm_ct_start(struct obd_export *exp,
               lk->lk_uid, lk->lk_group, lk->lk_flags);
 
        if (lk->lk_flags & LK_FLG_STOP) {
-               rc = libcfs_kkuc_group_rem(lk->lk_uid, lk->lk_group);
                /* Unregister with the coordinator */
-               if (rc == 0)
-                       rc = mdc_ioc_hsm_ct_unregister(imp);
+               rc = mdc_ioc_hsm_ct_unregister(imp);
        } else {
-               struct file *fp = fget(lk->lk_wfd);
-
-               rc = libcfs_kkuc_group_add(fp, lk->lk_uid, lk->lk_group,
-                                          lk->lk_data);
-               if (rc && fp)
-                       fput(fp);
-               if (rc == 0)
-                       rc = mdc_ioc_hsm_ct_register(imp, archive);
+               rc = mdc_ioc_hsm_ct_register(imp, archive);
        }
 
        return rc;
@@ -2380,7 +2374,7 @@ static int mdc_import_event(struct obd_device *obd, struct obd_import *imp,
         }
        case IMP_EVENT_ACTIVE:
                rc = obd_notify_observer(obd, obd, OBD_NOTIFY_ACTIVE, NULL);
-               /* restore re-establish kuc registration after reconnecting */
+               /* redo the kuc registration after reconnecting */
                if (rc == 0)
                        rc = mdc_kuc_reregister(imp);
                break;
index 4d32588..c4b5437 100644 (file)
@@ -128,18 +128,18 @@ int llapi_hsm_copytool_start(struct hsm_copytool_private **priv, char *fsname,
        ct->kuc.lk_data = ct->archives;
        rc = root_ioctl(ct->fsname, LL_IOC_HSM_CT_START, &(ct->kuc), NULL,
                        WANT_ERROR);
-       /* ignore if it was already registered on coordinator */
-       if (rc == -EEXIST)
-               rc = 0;
        /* Only the kernel reference keeps the write side open */
        close(ct->kuc.lk_wfd);
        ct->kuc.lk_wfd = 0;
        if (rc < 0)
-               goto out_err;
+               goto out_kuc;
 
        *priv = ct;
        return 0;
 
+out_kuc:
+       /* cleanup the kuc channel */
+       libcfs_ukuc_stop(&ct->kuc);
 out_err:
        if (ct->fsname)
                free(ct->fsname);