LU-3882 hsm: Prevent duplicate CT registrations

Associate copytool registration to a given MDC import so that
multiple mounts of the same filesystem do not lead to having the
copytool registered multiple time.

Signed-off-by: Henri Doreau <henri.doreau@cea.fr>
Change-Id: I5767df50331675a1650dda1ab5fc8440aad2a52e
Reviewed-on: http://review.whamcloud.com/7612
Reviewed-by: John L. Hammond <john.hammond@intel.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/libcfs/include/libcfs/libcfs_kernelcomm.h b/libcfs/include/libcfs/libcfs_kernelcomm.h
index c630ba3..ff1002f 100644 (file)
--- a/libcfs/include/libcfs/libcfs_kernelcomm.h
+++ b/libcfs/include/libcfs/libcfs_kernelcomm.h
@@ -78,7 +78,7 @@ enum kuc_generic_message_type {
 };
 
 /* prototype for callback function on kuc groups */
 };
 
 /* prototype for callback function on kuc groups */

-typedef int (*libcfs_kkuc_cb_t)(__u32 data, void *cb_arg);
+typedef int (*libcfs_kkuc_cb_t)(void *data, void *cb_arg);

 
 /* KUC Broadcast Groups. This determines which userspace process hears which
  * messages.  Mutliple transports may be used within a group, or multiple
 
 /* KUC Broadcast Groups. This determines which userspace process hears which
  * messages.  Mutliple transports may be used within a group, or multiple


@@ -93,8 +93,8 @@ typedef int (*libcfs_kkuc_cb_t)(__u32 data, void *cb_arg);
 extern int libcfs_kkuc_msg_put(struct file *fp, void *payload);
 extern int libcfs_kkuc_group_put(int group, void *payload);
 extern int libcfs_kkuc_group_add(struct file *fp, int uid, int group,
 extern int libcfs_kkuc_msg_put(struct file *fp, void *payload);
 extern int libcfs_kkuc_group_put(int group, void *payload);
 extern int libcfs_kkuc_group_add(struct file *fp, int uid, int group,

-                                __u32 data);
-extern int libcfs_kkuc_group_rem(int uid, int group);
+                                void *data);
+extern int libcfs_kkuc_group_rem(int uid, int group, void **pdata);

 extern int libcfs_kkuc_group_foreach(int group, libcfs_kkuc_cb_t cb_func,
                                     void *cb_arg);
 
 extern int libcfs_kkuc_group_foreach(int group, libcfs_kkuc_cb_t cb_func,
                                     void *cb_arg);
 

diff --git a/libcfs/libcfs/kernel_user_comm.c b/libcfs/libcfs/kernel_user_comm.c
index c322c85..96bee49 100644 (file)
--- a/libcfs/libcfs/kernel_user_comm.c
+++ b/libcfs/libcfs/kernel_user_comm.c
@@ -191,23 +191,25 @@ EXPORT_SYMBOL(libcfs_kkuc_msg_put);
 /* Broadcast groups are global across all mounted filesystems;
  * i.e. registering for a group on 1 fs will get messages for that
  * group from any fs */
 /* Broadcast groups are global across all mounted filesystems;
  * i.e. registering for a group on 1 fs will get messages for that
  * group from any fs */

-/** A single group reigstration has a uid and a file pointer */
+/** A single group registration has a uid and a file pointer */

 struct kkuc_reg {
 struct kkuc_reg {

-       cfs_list_t      kr_chain;
-       int             kr_uid;
+       cfs_list_t       kr_chain;
+       int              kr_uid;

        struct file     *kr_fp;
        struct file     *kr_fp;

-       __u32           kr_data;
+       void            *kr_data;

 };
 };

+

 static cfs_list_t kkuc_groups[KUC_GRP_MAX+1] = {};
 /* Protect message sending against remove and adds */
 static DECLARE_RWSEM(kg_sem);
 
 /** Add a receiver to a broadcast group
  * @param filp pipe to write into
 static cfs_list_t kkuc_groups[KUC_GRP_MAX+1] = {};
 /* Protect message sending against remove and adds */
 static DECLARE_RWSEM(kg_sem);
 
 /** Add a receiver to a broadcast group
  * @param filp pipe to write into

- * @param uid identidier for this receiver
+ * @param uid identifier for this receiver

  * @param group group number
  * @param group group number

+ * @param data user data

  */
  */

-int libcfs_kkuc_group_add(struct file *filp, int uid, int group, __u32 data)
+int libcfs_kkuc_group_add(struct file *filp, int uid, int group, void *data)

 {
         struct kkuc_reg *reg;
 
 {
         struct kkuc_reg *reg;
 


@@ -241,7 +243,7 @@ int libcfs_kkuc_group_add(struct file *filp, int uid, int group, __u32 data)
 }
 EXPORT_SYMBOL(libcfs_kkuc_group_add);
 
 }
 EXPORT_SYMBOL(libcfs_kkuc_group_add);
 

-int libcfs_kkuc_group_rem(int uid, int group)
+int libcfs_kkuc_group_rem(int uid, int group, void **pdata)

 {
         struct kkuc_reg *reg, *next;
         ENTRY;
 {
         struct kkuc_reg *reg, *next;
         ENTRY;


@@ -268,6 +270,8 @@ int libcfs_kkuc_group_rem(int uid, int group)
                                reg->kr_uid, reg->kr_fp, group);
                         if (reg->kr_fp != NULL)
                                fput(reg->kr_fp);
                                reg->kr_uid, reg->kr_fp, group);
                         if (reg->kr_fp != NULL)
                                fput(reg->kr_fp);

+                       if (pdata != NULL)
+                               *pdata = reg->kr_data;

                        kfree(reg);
                }
        }
                        kfree(reg);
                }
        }


@@ -311,14 +315,14 @@ EXPORT_SYMBOL(libcfs_kkuc_group_put);
  * Calls a callback function for each link of the given kuc group.
  * @param group the group to call the function on.
  * @param cb_func the function to be called.
  * Calls a callback function for each link of the given kuc group.
  * @param group the group to call the function on.
  * @param cb_func the function to be called.

- * @param cb_arg iextra argument to be passed to the callback function.
+ * @param cb_arg extra argument to be passed to the callback function.

  */
 int libcfs_kkuc_group_foreach(int group, libcfs_kkuc_cb_t cb_func,
                               void *cb_arg)
 {
  */
 int libcfs_kkuc_group_foreach(int group, libcfs_kkuc_cb_t cb_func,
                               void *cb_arg)
 {

-        struct kkuc_reg *reg;
-        int rc = 0;
-        ENTRY;
+       struct kkuc_reg *reg;
+       int              rc = 0;
+       ENTRY;

 
         if (group > KUC_GRP_MAX) {
                 CDEBUG(D_WARNING, "Kernelcomm: bad group %d\n", group);
 
         if (group > KUC_GRP_MAX) {
                 CDEBUG(D_WARNING, "Kernelcomm: bad group %d\n", group);

diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h
index fabea52..0b1f80e 100644 (file)
--- a/lustre/include/lustre_export.h
+++ b/lustre/include/lustre_export.h
@@ -382,6 +382,13 @@ static inline bool imp_connect_lvb_type(struct obd_import *imp)
 extern struct obd_export *class_conn2export(struct lustre_handle *conn);
 extern struct obd_device *class_conn2obd(struct lustre_handle *conn);
 
 extern struct obd_export *class_conn2export(struct lustre_handle *conn);
 extern struct obd_device *class_conn2obd(struct lustre_handle *conn);
 

+#define KKUC_CT_DATA_MAGIC     0x092013cea
+struct kkuc_ct_data {
+       __u32           kcd_magic;
+       struct obd_uuid kcd_uuid;
+       __u32           kcd_archive;
+};
+

 /** @} export */
 
 #endif /* __EXPORT_H */
 /** @} export */
 
 #endif /* __EXPORT_H */

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 7f8166e..7825758 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -858,8 +858,9 @@ static void lmv_hsm_req_build(struct lmv_obd *lmv,
 static int lmv_hsm_ct_unregister(struct lmv_obd *lmv, unsigned int cmd, int len,
                                 struct lustre_kernelcomm *lk, void *uarg)
 {
 static int lmv_hsm_ct_unregister(struct lmv_obd *lmv, unsigned int cmd, int len,
                                 struct lustre_kernelcomm *lk, void *uarg)
 {

-       __u32   i;
-       int     rc;
+       __u32                    i;
+       int                      rc;
+       struct kkuc_ct_data     *kcd = NULL;

        ENTRY;
 
        /* unregister request (call from llapi_hsm_copytool_fini) */
        ENTRY;
 
        /* unregister request (call from llapi_hsm_copytool_fini) */


@@ -873,17 +874,21 @@ static int lmv_hsm_ct_unregister(struct lmv_obd *lmv, unsigned int cmd, int len,
         * Unreached coordinators will get EPIPE on next requests
         * and will unregister automatically.
         */
         * Unreached coordinators will get EPIPE on next requests
         * and will unregister automatically.
         */

-       rc = libcfs_kkuc_group_rem(lk->lk_uid, lk->lk_group);
+       rc = libcfs_kkuc_group_rem(lk->lk_uid, lk->lk_group, (void **)&kcd);
+       if (kcd != NULL)
+               OBD_FREE_PTR(kcd);
+

        RETURN(rc);
 }
 
 static int lmv_hsm_ct_register(struct lmv_obd *lmv, unsigned int cmd, int len,
                               struct lustre_kernelcomm *lk, void *uarg)
 {
        RETURN(rc);
 }
 
 static int lmv_hsm_ct_register(struct lmv_obd *lmv, unsigned int cmd, int len,
                               struct lustre_kernelcomm *lk, void *uarg)
 {

-       struct file     *filp;
-       __u32            i, j;
-       int              err, rc;
-       bool             any_set = false;
+       struct file             *filp;
+       __u32                    i, j;
+       int                      err, rc;
+       bool                     any_set = false;
+       struct kkuc_ct_data     *kcd;

        ENTRY;
 
        /* All or nothing: try to register to all MDS.
        ENTRY;
 
        /* All or nothing: try to register to all MDS.


@@ -923,12 +928,25 @@ static int lmv_hsm_ct_register(struct lmv_obd *lmv, unsigned int cmd, int len,
 
        /* at least one registration done, with no failure */
        filp = fget(lk->lk_wfd);
 
        /* at least one registration done, with no failure */
        filp = fget(lk->lk_wfd);

-       if (filp == NULL) {
+       if (filp == NULL)

                RETURN(-EBADF);
                RETURN(-EBADF);

-       }
-       rc = libcfs_kkuc_group_add(filp, lk->lk_uid, lk->lk_group, lk->lk_data);
-       if (rc != 0 && filp != NULL)
+
+       OBD_ALLOC_PTR(kcd);
+       if (kcd == NULL) {

                fput(filp);
                fput(filp);

+               RETURN(-ENOMEM);
+       }
+       kcd->kcd_magic = KKUC_CT_DATA_MAGIC;
+       kcd->kcd_uuid = lmv->cluuid;
+       kcd->kcd_archive = lk->lk_data;
+
+       rc = libcfs_kkuc_group_add(filp, lk->lk_uid, lk->lk_group, kcd);
+       if (rc != 0) {
+               if (filp != NULL)
+                       fput(filp);
+               OBD_FREE_PTR(kcd);
+       }
+

        RETURN(rc);
 }
 
        RETURN(rc);
 }
 

diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index 24b50c7..b9e5cac 100644 (file)
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -2125,21 +2125,27 @@ static int mdc_hsm_copytool_send(int len, void *val)
 /**
  * callback function passed to kuc for re-registering each HSM copytool
  * running on MDC, after MDT shutdown/recovery.
 /**
  * callback function passed to kuc for re-registering each HSM copytool
  * running on MDC, after MDT shutdown/recovery.

- * @param data archive id served by the copytool
+ * @param data copytool registration data

  * @param cb_arg callback argument (obd_import)
  */
  * @param cb_arg callback argument (obd_import)
  */

-static int mdc_hsm_ct_reregister(__u32 data, void *cb_arg)
+static int mdc_hsm_ct_reregister(void *data, void *cb_arg)

 {
 {

+       struct kkuc_ct_data     *kcd = data;

        struct obd_import       *imp = (struct obd_import *)cb_arg;
        struct obd_import       *imp = (struct obd_import *)cb_arg;

-       __u32                    archive = data;

        int                      rc;
 
        int                      rc;
 

-       CDEBUG(D_HA, "recover copytool registration to MDT (archive=%#x)\n",
-              archive);
-       rc = mdc_ioc_hsm_ct_register(imp, archive);
+       if (kcd == NULL || kcd->kcd_magic != KKUC_CT_DATA_MAGIC)
+               return -EPROTO;
+
+       if (!obd_uuid_equals(&kcd->kcd_uuid, &imp->imp_obd->obd_uuid))
+               return 0;
+
+       CDEBUG(D_HA, "%s: recover copytool registration to MDT (archive=%#x)\n",
+              imp->imp_obd->obd_name, kcd->kcd_archive);
+       rc = mdc_ioc_hsm_ct_register(imp, kcd->kcd_archive);

 
        /* ignore error if the copytool is already registered */
 
        /* ignore error if the copytool is already registered */

-       return ((rc != 0) && (rc != -EEXIST)) ? rc : 0;
+       return (rc == -EEXIST) ? 0 : rc;

 }
 
 /**
 }
 
 /**


@@ -2541,9 +2547,9 @@ static int mdc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
         case OBD_CLEANUP_EARLY:
                 break;
         case OBD_CLEANUP_EXPORTS:
         case OBD_CLEANUP_EARLY:
                 break;
         case OBD_CLEANUP_EXPORTS:

-                /* Failsafe, ok if racy */
-                if (obd->obd_type->typ_refcnt <= 1)
-                        libcfs_kkuc_group_rem(0, KUC_GRP_HSM);
+               /* Failsafe, ok if racy */
+               if (obd->obd_type->typ_refcnt <= 1)
+                       libcfs_kkuc_group_rem(0, KUC_GRP_HSM, NULL);

 
                 obd_cleanup_client_import(obd);
                 ptlrpc_lprocfs_unregister_obd(obd);
 
                 obd_cleanup_client_import(obd);
                 ptlrpc_lprocfs_unregister_obd(obd);