Whamcloud - gitweb
LU-3951 lfsck: LWP connection from OST-x to MDT-y 66/7666/24
authorFan Yong <fan.yong@intel.com>
Fri, 24 Jan 2014 19:41:42 +0000 (03:41 +0800)
committerOleg Drokin <oleg.drokin@intel.com>
Fri, 31 Jan 2014 04:19:41 +0000 (04:19 +0000)
When client sends object-based RPC to the OST-x, the RPC service
thread on the OST-x needs to verify whether the given parent FID
information in the client RPC matches the parent FID information
stored in the OST-object. If not match, it will query the MDT-y
according to the client given parent FID information. The query
RPC from the OST-x to the MDT-y is sent via LWP connection.

The other use case is that the LFSCK on the OST-x needs to talk
with the LFSCK on the MDT-y, such control/query RPC will be via
above LWP connection from OST-x to MDT-y.

Currently, we only support LWP connection frm OST-x to the MDT-0.
This patch enhance that to enable LWP connection from any OST to
any MDT.

Test-Parameters: allwaysuploadlogs
Signed-off-by: Fan Yong <fan.yong@intel.com>
Change-Id: Ie98be82b3af90456d1838d53b6d77c12956f7bd7
Reviewed-on: http://review.whamcloud.com/7666
Tested-by: Jenkins
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Alex Zhuravlev <alexey.zhuravlev@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
13 files changed:
libcfs/include/libcfs/libcfs_string.h
libcfs/libcfs/libcfs_string.c
lustre/include/lustre_disk.h
lustre/include/obd.h
lustre/include/obd_class.h
lustre/mdt/mdt_handler.c
lustre/obdclass/obd_config.c
lustre/obdclass/obd_mount.c
lustre/obdclass/obd_mount_server.c
lustre/ofd/ofd_fs.c
lustre/ofd/ofd_objects.c
lustre/osp/lwp_dev.c
lustre/quota/qsd_lib.c

index e402b55..82c5c27 100644 (file)
@@ -44,6 +44,7 @@
 #define __LIBCFS_STRING_H__
 
 /* libcfs_string.c */
+char *cfs_strrstr(const char *haystack, const char *needle);
 /* string comparison ignoring case */
 int cfs_strncasecmp(const char *s1, const char *s2, size_t n);
 /* Convert a text string to a bitmask */
index be26f93..1d7a941 100644 (file)
 
 #include <libcfs/libcfs.h>
 
+char *cfs_strrstr(const char *haystack, const char *needle)
+{
+       char *ptr;
+
+       if (unlikely(haystack == NULL || needle == NULL))
+               return NULL;
+
+       if (strlen(needle) == 1)
+               return strrchr(haystack, needle[0]);
+
+       ptr = strstr(haystack, needle);
+       if (ptr != NULL) {
+               while (1) {
+                       char *tmp;
+
+                       tmp = strstr(&ptr[1], needle);
+                       if (tmp == NULL)
+                               return ptr;
+
+                       ptr = tmp;
+               }
+       }
+
+       return NULL;
+}
+EXPORT_SYMBOL(cfs_strrstr);
+
 /* non-0 = don't match */
 int cfs_strncasecmp(const char *s1, const char *s2, size_t n)
 {
index 09b72a3..a61f7be 100644 (file)
@@ -500,6 +500,9 @@ struct lustre_sb_info {
        char                      lsi_fstype[16];
        struct backing_dev_info   lsi_bdi;     /* each client mountpoint needs
                                                  own backing_dev_info */
+       struct list_head          lsi_lwp_list;
+       spinlock_t                lsi_lwp_lock;
+       unsigned long             lsi_lwp_started:1;
 };
 
 #define LSI_UMOUNT_FAILOVER              0x00200000
@@ -527,7 +530,6 @@ struct lustre_mount_info {
 #ifdef __KERNEL__
 /* obd_mount.c */
 int server_name2fsname(const char *svname, char *fsname, const char **endptr);
-int server_name2index(const char *svname, __u32 *idx, const char **endptr);
 int server_name2svname(const char *label, char *svname, const char **endptr,
                       size_t svsize);
 int server_name_is_ost(const char *svname);
index e19aab1..c18052b 100644 (file)
@@ -752,6 +752,7 @@ struct obd_device {
        cfs_list_t              obd_exports;
        cfs_list_t              obd_unlinked_exports;
        cfs_list_t              obd_delayed_exports;
+       struct list_head        obd_lwp_list;
        int                     obd_num_exports;
        spinlock_t              obd_nid_lock;
        struct ldlm_namespace  *obd_namespace;
@@ -769,6 +770,7 @@ struct obd_device {
        struct rw_semaphore     obd_observer_link_sem;
         struct obd_notify_upcall obd_upcall;
         struct obd_export       *obd_self_export;
+       struct obd_export       *obd_lwp_export;
         /* list of exports in LRU order, for ping evictor, with obd_dev_lock */
         cfs_list_t              obd_exports_timed;
         time_t                  obd_eviction_timer; /* for ping evictor */
index 11917f3..da400c2 100644 (file)
@@ -167,13 +167,14 @@ int class_add_uuid(const char *uuid, __u64 nid);
 
 /* Passed as data param to class_config_parse_llog */
 struct config_llog_instance {
-        char               *cfg_obdname;
-        void               *cfg_instance;
-        struct super_block *cfg_sb;
-        struct obd_uuid     cfg_uuid;
-       llog_cb_t           cfg_callback;
-        int                 cfg_last_idx; /* for partial llog processing */
-        int                 cfg_flags;
+       char                    *cfg_obdname;
+       void                    *cfg_instance;
+       struct super_block      *cfg_sb;
+       struct obd_uuid          cfg_uuid;
+       llog_cb_t                cfg_callback;
+       int                      cfg_last_idx; /* for partial llog processing */
+       int                      cfg_flags;
+       __u32                    cfg_lwp_idx;
 };
 int class_config_parse_llog(const struct lu_env *env, struct llog_ctxt *ctxt,
                            char *name, struct config_llog_instance *cfg);
@@ -2272,7 +2273,8 @@ extern int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c);
 int lustre_register_lwp_item(const char *lwpname, struct obd_export **exp,
                             register_lwp_cb cb_func, void *cb_data);
 void lustre_deregister_lwp_item(struct obd_export **exp);
-int tgt_name2lwpname(const char *tgt_name, char *lwp_name);
+struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx);
+int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx);
 #endif /* HAVE_SERVER_SUPPORT */
 
 /* sysctl.c */
@@ -2298,4 +2300,10 @@ int raw_name2idx(int hashtype, int count, const char *name, int namelen);
 /* prng.c */
 #define ll_generate_random_uuid(uuid_out) cfs_get_random_bytes(uuid_out, sizeof(class_uuid_t))
 
+#ifdef __KERNEL__
+int server_name2index(const char *svname, __u32 *idx, const char **endptr);
+#else
+# define server_name2index(name, idx, ptr)     do {} while (0)
+#endif
+
 #endif /* __LINUX_OBD_CLASS_H */
index f882b4d..ed52643 100644 (file)
@@ -3505,7 +3505,7 @@ static int mdt_register_seq_exp(struct mdt_device *mdt)
        if (lwp_name == NULL)
                GOTO(out_free, rc = -ENOMEM);
 
-       rc = tgt_name2lwpname(mdt_obd_name(mdt), lwp_name);
+       rc = tgt_name2lwp_name(mdt_obd_name(mdt), lwp_name, MAX_OBD_NAME, 0);
        if (rc != 0)
                GOTO(out_free, rc);
 
index 83465e2..8fa81b3 100644 (file)
@@ -42,6 +42,7 @@
 #ifdef __KERNEL__
 #include <obd_class.h>
 #include <linux/string.h>
+#include <lustre_disk.h>
 #else
 #include <liblustre.h>
 #include <string.h>
@@ -410,6 +411,7 @@ int class_attach(struct lustre_cfg *lcfg)
        CFS_INIT_LIST_HEAD(&obd->obd_lock_replay_queue);
        CFS_INIT_LIST_HEAD(&obd->obd_final_req_queue);
        CFS_INIT_LIST_HEAD(&obd->obd_evict_list);
+       INIT_LIST_HEAD(&obd->obd_lwp_list);
 
         llog_group_init(&obd->obd_olg, FID_SEQ_LLOG);
 
@@ -1514,6 +1516,8 @@ int class_config_llog_handler(const struct lu_env *env,
                         if (marker->cm_flags & CM_START) {
                                 /* all previous flags off */
                                 clli->cfg_flags = CFG_F_MARKER;
+                               server_name2index(marker->cm_tgtname,
+                                                 &clli->cfg_lwp_idx, NULL);
                                 if (marker->cm_flags & CM_SKIP) {
                                         clli->cfg_flags |= CFG_F_SKIP;
                                         CDEBUG(D_CONFIG, "SKIP #%d\n",
index 0fa0463..d592459 100644 (file)
@@ -588,8 +588,10 @@ struct lustre_sb_info *lustre_init_lsi(struct super_block *sb)
 
         /* Default umount style */
         lsi->lsi_flags = LSI_UMOUNT_FAILOVER;
+       INIT_LIST_HEAD(&lsi->lsi_lwp_list);
+       spin_lock_init(&lsi->lsi_lwp_lock);
 
-        RETURN(lsi);
+       RETURN(lsi);
 }
 
 static int lustre_free_lsi(struct super_block *sb)
index cff3d80..0567556 100644 (file)
@@ -327,54 +327,28 @@ static int server_mgc_clear_fs(struct obd_device *mgc)
        RETURN(rc);
 }
 
-static int is_mdc_device(const char *devname)
+static inline bool is_mdc_device(const char *devname)
 {
        char *ptr;
 
        ptr = strrchr(devname, '-');
-       if (ptr != NULL && strcmp(ptr, "-mdc") == 0)
-               return 1;
-
-       return 0;
+       return ptr != NULL && strcmp(ptr, "-mdc") == 0;
 }
 
-static inline int tgt_is_mdt0(const char *tgtname)
+static inline bool tgt_is_mdt(const char *tgtname, __u32 *idx)
 {
-       __u32 idx;
-       int   type;
+       int type;
 
-       type = server_name2index(tgtname, &idx, NULL);
-       if (type != LDD_F_SV_TYPE_MDT)
-               return 0;
+       type = server_name2index(tgtname, idx, NULL);
 
-       return idx == 0;
-}
-
-static inline int is_mdc_for_mdt0(const char *devname)
-{
-       char   *ptr;
-
-       if (!is_mdc_device(devname))
-               return 0;
-
-       ptr = strrchr(devname, '-');
-       if (ptr == NULL)
-               return 0;
-
-       *ptr = 0;
-       if (tgt_is_mdt0(devname)) {
-               *ptr = '-';
-               return 1;
-       }
-       *ptr = '-';
-       return 0;
+       return type == LDD_F_SV_TYPE_MDT;
 }
 
 /**
- * Convert OST/MDT name(fsname-OSTxxxx) to a lwp name
- * (fsname-MDT0000-lwp-OSTxxxx)
+ * Convert OST/MDT name(fsname-{MDT,OST}xxxx) to a lwp name with the @idx:yyyy
+ * (fsname-MDTyyyy-lwp-{MDT,OST}xxxx)
  **/
-int tgt_name2lwpname(const char *svname, char *lwpname)
+int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx)
 {
        char            *fsname;
        const char      *tgt;
@@ -385,30 +359,35 @@ int tgt_name2lwpname(const char *svname, char *lwpname)
        if (fsname == NULL)
                RETURN(-ENOMEM);
 
-       rc = server_name2fsname(svname, fsname, &tgt);
+       rc = server_name2fsname(tgt_name, fsname, &tgt);
        if (rc != 0) {
-               CERROR("%s: failed to get fsname from svname. %d\n",
-                      svname, rc);
+               CERROR("%s: failed to get fsname from tgt_name: rc = %d\n",
+                      tgt_name, rc);
                GOTO(cleanup, rc);
        }
 
        if (*tgt != '-' && *tgt != ':') {
-               CERROR("%s: invalid svname name!\n", svname);
+               CERROR("%s: invalid tgt_name name!\n", tgt_name);
                GOTO(cleanup, rc = -EINVAL);
        }
 
        tgt++;
        if (strncmp(tgt, "OST", 3) != 0 && strncmp(tgt, "MDT", 3) != 0) {
-               CERROR("%s is not an OST or MDT target!\n", svname);
+               CERROR("%s is not an OST or MDT target!\n", tgt_name);
                GOTO(cleanup, rc = -EINVAL);
        }
-       sprintf(lwpname, "%s-MDT0000-%s-%s", fsname, LUSTRE_LWP_NAME, tgt);
+       snprintf(lwp_name, len, "%s-MDT%04x-%s-%s",
+                fsname, idx, LUSTRE_LWP_NAME, tgt);
+
+       GOTO(cleanup, rc = 0);
+
 cleanup:
        if (fsname != NULL)
                OBD_FREE(fsname, MTI_NAME_MAXLEN);
-       RETURN(rc);
+
+       return rc;
 }
-EXPORT_SYMBOL(tgt_name2lwpname);
+EXPORT_SYMBOL(tgt_name2lwp_name);
 
 static CFS_LIST_HEAD(lwp_register_list);
 DEFINE_MUTEX(lwp_register_list_lock);
@@ -478,6 +457,44 @@ void lustre_deregister_lwp_item(struct obd_export **exp)
 }
 EXPORT_SYMBOL(lustre_deregister_lwp_item);
 
+struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx)
+{
+       struct lustre_mount_info *lmi;
+       struct lustre_sb_info    *lsi;
+       struct obd_device        *lwp;
+       struct obd_export        *exp = NULL;
+       char                      fsname[16];
+       char                      lwp_name[24];
+       int                       rc;
+
+       lmi = server_get_mount_2(dev);
+       if (lmi == NULL)
+               return NULL;
+
+       lsi = s2lsi(lmi->lmi_sb);
+       rc = server_name2fsname(lsi->lsi_svname, fsname, NULL);
+       if (rc != 0) {
+               CERROR("%s: failed to get fsname: rc = %d\n",
+                      lsi->lsi_svname, rc);
+               return NULL;
+       }
+
+       snprintf(lwp_name, sizeof(lwp_name), "%s-MDT%04x", fsname, idx);
+       spin_lock(&lsi->lsi_lwp_lock);
+       list_for_each_entry(lwp, &lsi->lsi_lwp_list, obd_lwp_list) {
+               char *ptr = strstr(lwp->obd_name, lwp_name);
+
+               if (ptr != NULL) {
+                       exp = class_export_get(lwp->obd_lwp_export);
+                       break;
+               }
+       }
+       spin_unlock(&lsi->lsi_lwp_lock);
+
+       return exp;
+}
+EXPORT_SYMBOL(lustre_find_lwp_by_index);
+
 static void lustre_notify_lwp_list(struct obd_export *exp)
 {
        struct lwp_register_item *lri, *tmp;
@@ -539,10 +556,16 @@ static int lustre_lwp_connect(struct obd_device *lwp)
        /* Use lwp name as the uuid, so we find the export by lwp name later */
        memcpy(uuid->uuid, lwp->obd_name, strlen(lwp->obd_name));
        rc = obd_connect(&env, &exp, lwp, uuid, data, NULL);
-       if (rc != 0)
+       if (rc != 0) {
                CERROR("%s: connect failed: rc = %d\n", lwp->obd_name, rc);
-       else
+       } else {
+               if (unlikely(lwp->obd_lwp_export != NULL))
+                       class_export_put(lwp->obd_lwp_export);
+               lwp->obd_lwp_export = class_export_get(exp);
                lustre_notify_lwp_list(exp);
+       }
+
+       GOTO(out, rc);
 
 out:
        if (data != NULL)
@@ -554,14 +577,15 @@ out:
        lu_context_exit(&session_ctx);
        lu_context_fini(&session_ctx);
 
-       RETURN(rc);
+       return rc;
 }
 
 /**
- * lwp is used by slaves (Non-MDT0 targets) to manage the connection
- * to MDT0.
+ * lwp is used by slaves (Non-MDT0 targets) to manage the connection to MDT0,
+ * or from the OSTx to MDTy.
  **/
-static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi)
+static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi,
+                           __u32 idx)
 {
        struct obd_device       *obd;
        char                    *lwpname = NULL;
@@ -571,18 +595,18 @@ static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi)
 
        rc = class_add_uuid(lustre_cfg_string(lcfg, 1),
                            lcfg->lcfg_nid);
-       if (rc) {
+       if (rc != 0) {
                CERROR("%s: Can't add uuid: rc =%d\n", lsi->lsi_svname, rc);
-               GOTO(out, rc);
+               RETURN(rc);
        }
 
        OBD_ALLOC(lwpname, MTI_NAME_MAXLEN);
        if (lwpname == NULL)
                GOTO(out, rc = -ENOMEM);
 
-       rc = tgt_name2lwpname(lsi->lsi_svname, lwpname);
+       rc = tgt_name2lwp_name(lsi->lsi_svname, lwpname, MTI_NAME_MAXLEN, idx);
        if (rc != 0) {
-               CERROR("%s: failed to generate lwp name. %d\n",
+               CERROR("%s: failed to generate lwp name: rc = %d\n",
                       lsi->lsi_svname, rc);
                GOTO(out, rc);
        }
@@ -604,20 +628,29 @@ static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi)
        LASSERT(obd != NULL);
 
        rc = lustre_lwp_connect(obd);
-       if (rc != 0)
+       if (rc == 0) {
+               obd->u.cli.cl_max_mds_easize = MAX_MD_SIZE;
+               spin_lock(&lsi->lsi_lwp_lock);
+               list_add_tail(&obd->obd_lwp_list, &lsi->lsi_lwp_list);
+               spin_unlock(&lsi->lsi_lwp_lock);
+       } else {
                CERROR("%s: connect failed: rc = %d\n", lwpname, rc);
+       }
+
+       GOTO(out, rc);
+
 out:
        if (lwpname != NULL)
                OBD_FREE(lwpname, MTI_NAME_MAXLEN);
        if (lwpuuid != NULL)
                OBD_FREE(lwpuuid, MTI_NAME_MAXLEN);
 
-       RETURN(rc);
+       return rc;
 }
 
 /* the caller is responsible for memory free */
 static struct obd_device *lustre_find_lwp(struct lustre_sb_info *lsi,
-                                         char **lwpname, char **logname)
+                                         char **lwpname, __u32 idx)
 {
        struct obd_device       *lwp;
        int                      rc = 0;
@@ -630,22 +663,9 @@ static struct obd_device *lustre_find_lwp(struct lustre_sb_info *lsi,
        if (*lwpname == NULL)
                RETURN(ERR_PTR(-ENOMEM));
 
-       if (logname != NULL) {
-               OBD_ALLOC(*logname, MTI_NAME_MAXLEN);
-               if (*logname == NULL)
-                       GOTO(out, rc = -ENOMEM);
-               rc = server_name2fsname(lsi->lsi_svname, *lwpname, NULL);
-               if (rc != 0) {
-                       CERROR("%s: failed to get fsname from svname. %d\n",
-                              lsi->lsi_svname, rc);
-                       GOTO(out, rc = -EINVAL);
-               }
-               sprintf(*logname, "%s-client", *lwpname);
-       }
-
-       rc = tgt_name2lwpname(lsi->lsi_svname, *lwpname);
+       rc = tgt_name2lwp_name(lsi->lsi_svname, *lwpname, MTI_NAME_MAXLEN, idx);
        if (rc != 0) {
-               CERROR("%s: failed to generate lwp name. %d\n",
+               CERROR("%s: failed to generate lwp name: rc = %d\n",
                       lsi->lsi_svname, rc);
                GOTO(out, rc = -EINVAL);
        }
@@ -658,10 +678,6 @@ out:
                        OBD_FREE(*lwpname, MTI_NAME_MAXLEN);
                        *lwpname = NULL;
                }
-               if (logname != NULL && *logname != NULL) {
-                       OBD_FREE(*logname, MTI_NAME_MAXLEN);
-                       *logname = NULL;
-               }
                lwp = ERR_PTR(rc);
        }
 
@@ -669,7 +685,7 @@ out:
 }
 
 static int lustre_lwp_add_conn(struct lustre_cfg *cfg,
-                              struct lustre_sb_info *lsi)
+                              struct lustre_sb_info *lsi, __u32 idx)
 {
        struct lustre_cfg_bufs *bufs = NULL;
        struct lustre_cfg      *lcfg = NULL;
@@ -678,7 +694,7 @@ static int lustre_lwp_add_conn(struct lustre_cfg *cfg,
        int                     rc;
        ENTRY;
 
-       lwp = lustre_find_lwp(lsi, &lwpname, NULL);
+       lwp = lustre_find_lwp(lsi, &lwpname, idx);
        if (IS_ERR(lwp)) {
                CERROR("%s: can't find lwp device.\n", lsi->lsi_svname);
                GOTO(out, rc = PTR_ERR(lwp));
@@ -713,19 +729,19 @@ out:
  * Retrieve MDT nids from the client log, then start the lwp device.
  * there are only two scenarios which would include mdt nid.
  * 1.
- * marker   5 (flags=0x01, v2.1.54.0) lustre-MDT0000  'add mdc' xxx-
+ * marker   5 (flags=0x01, v2.1.54.0) lustre-MDTyyyy  'add mdc' xxx-
  * add_uuid  nid=192.168.122.162@tcp(0x20000c0a87aa2)  0:  1:192.168.122.162@tcp
- * attach    0:lustre-MDT0000-mdc  1:mdc  2:lustre-clilmv_UUID
- * setup     0:lustre-MDT0000-mdc  1:lustre-MDT0000_UUID  2:192.168.122.162@tcp
+ * attach    0:lustre-MDTyyyy-mdc  1:mdc  2:lustre-clilmv_UUID
+ * setup     0:lustre-MDTyyyy-mdc  1:lustre-MDTyyyy_UUID  2:192.168.122.162@tcp
  * add_uuid  nid=192.168.172.1@tcp(0x20000c0a8ac01)  0:  1:192.168.172.1@tcp
- * add_conn  0:lustre-MDT0000-mdc  1:192.168.172.1@tcp
- * modify_mdc_tgts add 0:lustre-clilmv  1:lustre-MDT0000_UUID xxxx
- * marker   5 (flags=0x02, v2.1.54.0) lustre-MDT0000  'add mdc' xxxx-
+ * add_conn  0:lustre-MDTyyyy-mdc  1:192.168.172.1@tcp
+ * modify_mdc_tgts add 0:lustre-clilmv  1:lustre-MDTyyyy_UUID xxxx
+ * marker   5 (flags=0x02, v2.1.54.0) lustre-MDTyyyy  'add mdc' xxxx-
  * 2.
- * marker   7 (flags=0x01, v2.1.54.0) lustre-MDT0000  'add failnid' xxxx-
+ * marker   7 (flags=0x01, v2.1.54.0) lustre-MDTyyyy  'add failnid' xxxx-
  * add_uuid  nid=192.168.122.2@tcp(0x20000c0a87a02)  0:  1:192.168.122.2@tcp
- * add_conn  0:lustre-MDT0000-mdc  1:192.168.122.2@tcp
- * marker   7 (flags=0x02, v2.1.54.0) lustre-MDT0000  'add failnid' xxxx-
+ * add_conn  0:lustre-MDTyyyy-mdc  1:192.168.122.2@tcp
+ * marker   7 (flags=0x02, v2.1.54.0) lustre-MDTyyyy  'add failnid' xxxx-
  **/
 static int client_lwp_config_process(const struct lu_env *env,
                                     struct llog_handle *handle,
@@ -768,7 +784,10 @@ static int client_lwp_config_process(const struct lu_env *env,
                    marker->cm_flags & CM_EXCLUDE)
                        GOTO(out, rc = 0);
 
-               if (!tgt_is_mdt0(marker->cm_tgtname))
+               if (!tgt_is_mdt(marker->cm_tgtname, &clli->cfg_lwp_idx))
+                       GOTO(out, rc = 0);
+
+               if (IS_MDT(lsi) && clli->cfg_lwp_idx != 0)
                        GOTO(out, rc = 0);
 
                if (!strncmp(marker->cm_comment, "add mdc", 7) ||
@@ -789,7 +808,7 @@ static int client_lwp_config_process(const struct lu_env *env,
        }
        case LCFG_ADD_UUID: {
                if (clli->cfg_flags == CFG_F_MARKER) {
-                       rc = lustre_lwp_setup(lcfg, lsi);
+                       rc = lustre_lwp_setup(lcfg, lsi, clli->cfg_lwp_idx);
                        /* XXX: process only the first nid as
                         * we don't need another instance of lwp */
                        clli->cfg_flags |= CFG_F_SKIP;
@@ -803,8 +822,25 @@ static int client_lwp_config_process(const struct lu_env *env,
                break;
        }
        case LCFG_ADD_CONN: {
-               if (is_mdc_for_mdt0(lustre_cfg_string(lcfg, 0)))
-                       rc = lustre_lwp_add_conn(lcfg, lsi);
+               char *devname = lustre_cfg_string(lcfg, 0);
+               char *ptr;
+               __u32 idx     = 0;
+
+               if (!is_mdc_device(devname))
+                       break;
+
+               ptr = strrchr(devname, '-');
+               if (ptr == NULL)
+                       break;
+
+               *ptr = 0;
+               if (!tgt_is_mdt(devname, &idx)) {
+                       *ptr = '-';
+                       break;
+               }
+
+               *ptr = '-';
+               rc = lustre_lwp_add_conn(lcfg, lsi, idx);
                break;
        }
        default:
@@ -816,67 +852,82 @@ out:
 
 static int lustre_disconnect_lwp(struct super_block *sb)
 {
-       struct lustre_sb_info           *lsi = s2lsi(sb);
+       struct lustre_sb_info           *lsi     = s2lsi(sb);
        struct obd_device               *lwp;
-       char                            *lwpname = NULL;
        char                            *logname = NULL;
-       struct lustre_cfg               *lcfg = NULL;
-       struct lustre_cfg_bufs          *bufs = NULL;
-       struct config_llog_instance     *cfg = NULL;
-       int                              rc;
+       struct lustre_cfg_bufs          *bufs    = NULL;
+       struct config_llog_instance     *cfg     = NULL;
+       int                              rc      = 0;
+       int                              rc1     = 0;
        ENTRY;
 
-       lwp = lustre_find_lwp(lsi, &lwpname, &logname);
-       if (IS_ERR(lwp) && PTR_ERR(lwp) != -ENOENT)
-               GOTO(out, rc = PTR_ERR(lwp));
+       if (likely(lsi->lsi_lwp_started)) {
+               OBD_ALLOC(logname, MTI_NAME_MAXLEN);
+               if (logname == NULL)
+                       RETURN(-ENOMEM);
 
-       LASSERT(lwpname != NULL);
-       LASSERT(logname != NULL);
+               rc = server_name2fsname(lsi->lsi_svname, logname, NULL);
+               if (rc != 0) {
+                       CERROR("%s: failed to get fsname from svname: "
+                              "rc = %d\n", lsi->lsi_svname, rc);
+                       GOTO(out, rc = -EINVAL);
+               }
 
-       OBD_ALLOC_PTR(cfg);
-       if (cfg == NULL)
-               GOTO(out, rc = -ENOMEM);
+               strcat(logname, "-client");
+               OBD_ALLOC_PTR(cfg);
+               if (cfg == NULL)
+                       GOTO(out, rc = -ENOMEM);
 
-       /* end log first */
-       cfg->cfg_instance = sb;
-       rc = lustre_end_log(sb, logname, cfg);
-       if (rc != 0) {
-               CERROR("%s: Can't end config log %s.\n", lwpname, logname);
-               GOTO(out, rc);
-       }
+               /* end log first */
+               cfg->cfg_instance = sb;
+               rc = lustre_end_log(sb, logname, cfg);
+               if (rc != 0)
+                       GOTO(out, rc);
 
-       if (PTR_ERR(lwp) == -ENOENT) {
-               CDEBUG(D_CONFIG, "%s: lwp device wasn't started.\n",
-                      lsi->lsi_svname);
-               GOTO(out, rc = 0);
+               lsi->lsi_lwp_started = 0;
        }
 
        OBD_ALLOC_PTR(bufs);
        if (bufs == NULL)
                GOTO(out, rc = -ENOMEM);
 
-       lustre_cfg_bufs_reset(bufs, lwp->obd_name);
-       lustre_cfg_bufs_set_string(bufs, 1, NULL);
-       lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs);
-       if (!lcfg)
-               GOTO(out, rc = -ENOMEM);
+       list_for_each_entry(lwp, &lsi->lsi_lwp_list, obd_lwp_list) {
+               struct lustre_cfg *lcfg;
 
-       /* Disconnect import first. NULL is passed for the '@env', since
-        * it will not be used. */
-       rc = lwp->obd_lu_dev->ld_ops->ldo_process_config(NULL, lwp->obd_lu_dev,
-                                                        lcfg);
-out:
-       if (lcfg)
+               if (likely(lwp->obd_lwp_export != NULL)) {
+                       class_export_put(lwp->obd_lwp_export);
+                       lwp->obd_lwp_export = NULL;
+               }
+
+               lustre_cfg_bufs_reset(bufs, lwp->obd_name);
+               lustre_cfg_bufs_set_string(bufs, 1, NULL);
+               lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs);
+               if (lcfg == NULL)
+                       GOTO(out, rc = -ENOMEM);
+
+               /* Disconnect import first. NULL is passed for the '@env',
+                * since it will not be used. */
+               rc = lwp->obd_lu_dev->ld_ops->ldo_process_config(NULL,
+                                                       lwp->obd_lu_dev, lcfg);
                lustre_cfg_free(lcfg);
-       if (bufs)
+               if (rc != 0 && rc != -ETIMEDOUT) {
+                       CERROR("%s: fail to disconnect LWP: rc = %d\n",
+                              lwp->obd_name, rc);
+                       rc1 = rc;
+               }
+       }
+
+       GOTO(out, rc);
+
+out:
+       if (bufs != NULL)
                OBD_FREE_PTR(bufs);
-       if (cfg)
+       if (cfg != NULL)
                OBD_FREE_PTR(cfg);
-       if (lwpname)
-               OBD_FREE(lwpname, MTI_NAME_MAXLEN);
-       if (logname)
+       if (logname != NULL)
                OBD_FREE(logname, MTI_NAME_MAXLEN);
-       RETURN(rc);
+
+       return rc1 != 0 ? rc1 : rc;
 }
 
 /**
@@ -885,69 +936,71 @@ out:
 static int lustre_stop_lwp(struct super_block *sb)
 {
        struct lustre_sb_info   *lsi = s2lsi(sb);
-       struct obd_device       *lwp = NULL;
-       char                    *lwpname = NULL;
-       int                      rc = 0;
+       struct obd_device       *lwp;
+       int                      rc  = 0;
+       int                      rc1 = 0;
        ENTRY;
 
-       lwp = lustre_find_lwp(lsi, &lwpname, NULL);
-       if (IS_ERR(lwp)) {
-               CDEBUG(PTR_ERR(lwp) == -ENOENT ? D_CONFIG : D_ERROR,
-                      "%s: lwp wasn't started.\n", lsi->lsi_svname);
-               GOTO(out, rc = 0);
+       while (!list_empty(&lsi->lsi_lwp_list)) {
+               lwp = list_entry(lsi->lsi_lwp_list.next, struct obd_device,
+                                obd_lwp_list);
+               list_del_init(&lwp->obd_lwp_list);
+               lwp->obd_force = 1;
+               rc = class_manual_cleanup(lwp);
+               if (rc != 0) {
+                       CERROR("%s: fail to stop LWP: rc = %d\n",
+                              lwp->obd_name, rc);
+                       rc1 = rc;
+               }
        }
 
-       lwp->obd_force = 1;
-       rc = class_manual_cleanup(lwp);
-
-out:
-       if (lwpname != NULL)
-               OBD_FREE(lwpname, MTI_NAME_MAXLEN);
-       RETURN(rc);
+       RETURN(rc1 != 0 ? rc1 : rc);
 }
 
 /**
- * Start the lwp(fsname-MDT0000-lwp-OSTxxxx) for an OST or MDT target,
- * which would be used to establish connection from OST to MDT0.
+ * Start the lwp(fsname-MDTyyyy-lwp-{MDT,OST}xxxx) for a MDT/OST or MDT target.
  **/
 static int lustre_start_lwp(struct super_block *sb)
 {
        struct lustre_sb_info       *lsi = s2lsi(sb);
        struct config_llog_instance *cfg = NULL;
-       struct obd_device           *lwp;
-       char                        *lwpname = NULL;
-       char                        *logname = NULL;
+       char                        *logname;
        int                          rc;
        ENTRY;
 
-       lwp = lustre_find_lwp(lsi, &lwpname, &logname);
+       if (unlikely(lsi->lsi_lwp_started))
+               RETURN(0);
 
-       /* the lwp device already stared */
-       if (lwp && !IS_ERR(lwp))
-               GOTO(out, rc = 0);
-
-       if (PTR_ERR(lwp) != -ENOENT)
-               GOTO(out, rc = PTR_ERR(lwp));
+       OBD_ALLOC(logname, MTI_NAME_MAXLEN);
+       if (logname == NULL)
+               RETURN(-ENOMEM);
 
-       LASSERT(lwpname != NULL);
-       LASSERT(logname != NULL);
+       rc = server_name2fsname(lsi->lsi_svname, logname, NULL);
+       if (rc != 0) {
+               CERROR("%s: failed to get fsname from svname: rc = %d\n",
+                      lsi->lsi_svname, rc);
+               GOTO(out, rc = -EINVAL);
+       }
 
+       strcat(logname, "-client");
        OBD_ALLOC_PTR(cfg);
        if (cfg == NULL)
                GOTO(out, rc = -ENOMEM);
 
        cfg->cfg_callback = client_lwp_config_process;
        cfg->cfg_instance = sb;
-
        rc = lustre_process_log(sb, logname, cfg);
+       if (rc == 0)
+               lsi->lsi_lwp_started = 1;
+
+       GOTO(out, rc);
+
 out:
-       if (lwpname != NULL)
-               OBD_FREE(lwpname, MTI_NAME_MAXLEN);
-       if (logname != NULL)
-               OBD_FREE(logname, MTI_NAME_MAXLEN);
+       OBD_FREE(logname, MTI_NAME_MAXLEN);
        if (cfg != NULL)
                OBD_FREE_PTR(cfg);
-       RETURN(rc);
+
+       return rc;
 }
 
 DEFINE_MUTEX(server_start_lock);
@@ -1414,9 +1467,10 @@ static void server_put_super(struct super_block *sb)
                int     rc;
 
                rc = lustre_disconnect_lwp(sb);
-               if (rc && rc != ETIMEDOUT)
-                       CERROR("%s: failed to disconnect lwp. (rc=%d)\n",
-                              tmpname, rc);
+               if (rc != 0 && rc != -ETIMEDOUT &&
+                   rc != -ENOTCONN && rc != -ESHUTDOWN)
+                       CWARN("%s: failed to disconnect lwp: rc= %d\n",
+                             tmpname, rc);
        }
 
        /* Stop the target */
index e5fd706..80a5ae4 100644 (file)
@@ -402,7 +402,7 @@ static int ofd_register_seq_exp(struct ofd_device *ofd)
        if (lwp_name == NULL)
                GOTO(out_free, rc = -ENOMEM);
 
-       rc = tgt_name2lwpname(ofd_name(ofd), lwp_name);
+       rc = tgt_name2lwp_name(ofd_name(ofd), lwp_name, MAX_OBD_NAME, 0);
        if (rc != 0)
                GOTO(out_free, rc);
 
index cd8c26a..6458498 100644 (file)
@@ -270,7 +270,7 @@ int ofd_precreate_objects(const struct lu_env *env, struct ofd_device *ofd,
 
         /* When the LFSCK scanning the whole device to verify the LAST_ID file
          * consistency, it will load the last_id into RAM firstly, and compare
-         * the last_id with echo OST-object's ID. If the later one is larger,
+         * the last_id with each OST-object's ID. If the later one is larger,
          * then it will regard the LAST_ID file crashed. But during the LFSCK
          * scanning, the OFD may continue to create new OST-objects. Those new
          * created OST-objects will have larger IDs than the LFSCK known ones.
index fce82a2..9560504 100644 (file)
@@ -37,6 +37,7 @@
 #include <obd_class.h>
 #include <lustre_param.h>
 #include <lustre_log.h>
+#include <libcfs/libcfs_string.h>
 
 struct lwp_device {
        struct lu_device        lpd_dev;
@@ -56,40 +57,17 @@ static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
        return &d->lpd_dev;
 }
 
-static int lwp_name2fsname(char *lwpname, char *fsname)
-{
-       char *ptr;
-
-       LASSERT(lwpname != NULL);
-       LASSERT(fsname != NULL);
-
-       sprintf(fsname, "-%s-", LUSTRE_LWP_NAME);
-
-       ptr = strstr(lwpname, fsname);
-       if (ptr == NULL)
-               return -EINVAL;
-
-       while (*(--ptr) != '-') {
-               if (ptr == lwpname)
-                       return -EINVAL;
-       }
-
-       strncpy(fsname, lwpname, ptr - lwpname);
-       fsname[ptr - lwpname] = '\0';
-
-       return 0;
-}
-
 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
                     char *nidstring)
 {
        struct lustre_cfg_bufs  *bufs = NULL;
        struct lustre_cfg       *lcfg = NULL;
-       char                    *lwpname = lwp->lpd_obd->obd_name;
-       char                    *fsname = NULL;
+       char                    *lwp_name = lwp->lpd_obd->obd_name;
        char                    *server_uuid = NULL;
+       char                    *ptr;
        class_uuid_t             uuid;
        struct obd_import       *imp;
+       int                      len = strlen(lwp_name);
        int                      rc;
        ENTRY;
 
@@ -97,23 +75,22 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
        if (bufs == NULL)
                RETURN(-ENOMEM);
 
-       OBD_ALLOC(fsname, strlen(lwpname));
-       if (fsname == NULL)
+       OBD_ALLOC(server_uuid, len);
+       if (server_uuid == NULL)
                GOTO(out, rc = -ENOMEM);
 
-       rc = lwp_name2fsname(lwpname, fsname);
-       if (rc) {
-               CERROR("%s: failed to get fsname from lwpname. %d\n",
-                      lwpname, rc);
-               GOTO(out, rc);
+       snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME);
+       ptr = cfs_strrstr(lwp_name, server_uuid);
+       if (ptr == NULL) {
+               CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
+                      lwp_name, -EINVAL);
+               GOTO(out, rc = -EINVAL);
        }
 
-       OBD_ALLOC(server_uuid, strlen(fsname) + 15);
-       if (server_uuid == NULL)
-               GOTO(out, rc = -ENOMEM);
-
-       sprintf(server_uuid, "%s-MDT0000_UUID", fsname);
-       lustre_cfg_bufs_reset(bufs, lwpname);
+       strncpy(server_uuid, lwp_name, ptr - lwp_name);
+       server_uuid[ptr - lwp_name] = '\0';
+       strncat(server_uuid, "_UUID", len - 1);
+       lustre_cfg_bufs_reset(bufs, lwp_name);
        lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
        lustre_cfg_bufs_set_string(bufs, 2, nidstring);
        lcfg = lustre_cfg_new(LCFG_SETUP, bufs);
@@ -138,9 +115,7 @@ out:
        if (bufs != NULL)
                OBD_FREE_PTR(bufs);
        if (server_uuid != NULL)
-               OBD_FREE(server_uuid, strlen(fsname) + 15);
-       if (fsname != NULL)
-               OBD_FREE(fsname, strlen(lwpname));
+               OBD_FREE(server_uuid, len);
        if (lcfg != NULL)
                lustre_cfg_free(lcfg);
        if (rc)
@@ -149,7 +124,7 @@ out:
        RETURN(rc);
 }
 
-int lwp_disconnect(struct lwp_device *d)
+static int lwp_disconnect(struct lwp_device *d)
 {
        struct obd_import *imp;
        int rc = 0;
@@ -169,12 +144,13 @@ int lwp_disconnect(struct lwp_device *d)
        /* Some non-replayable imports (MDS's OSCs) are pinged, so just
         * delete it regardless.  (It's safe to delete an import that was
         * never added.) */
-       (void)ptlrpc_pinger_del_import(imp);
-
+       ptlrpc_pinger_del_import(imp);
        rc = ptlrpc_disconnect_import(imp, 0);
-       if (rc && rc != -ETIMEDOUT)
-               CERROR("%s: can't disconnect: rc = %d\n",
-                      d->lpd_obd->obd_name, rc);
+       if (rc != 0 && rc != -ETIMEDOUT && rc != -ENOTCONN && rc != -ESHUTDOWN)
+               CWARN("%s: can't disconnect: rc = %d\n",
+                     d->lpd_obd->obd_name, rc);
+       else
+               rc = 0;
 
        ptlrpc_invalidate_import(imp);
 
@@ -352,8 +328,9 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
                           struct obd_connect_data *data, void *localdata)
 {
        struct lwp_device       *lwp = lu2lwp_dev(obd->obd_lu_dev);
+       struct client_obd       *cli = &lwp->lpd_obd->u.cli;
+       struct obd_import       *imp = cli->cl_import;
        struct obd_connect_data *ocd;
-       struct obd_import       *imp;
        struct lustre_handle     conn;
        int                      rc;
 
@@ -361,9 +338,11 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 
        CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
 
+       *exp = NULL;
+       down_write(&cli->cl_sem);
        rc = class_connect(&conn, obd, cluuid);
-       if (rc)
-               RETURN(rc);
+       if (rc != 0)
+               GOTO(out_sem, rc);
 
        *exp = class_conn2export(&conn);
        lwp->lpd_exp = *exp;
@@ -372,8 +351,10 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
        lwp->lpd_connects++;
        LASSERT(lwp->lpd_connects == 1);
 
-       imp = lwp->lpd_obd->u.cli.cl_import;
        imp->imp_dlm_handle = conn;
+       rc = ptlrpc_init_import(imp);
+       if (rc != 0)
+               GOTO(out_dis, rc);
 
        LASSERT(data != NULL);
        ocd = &imp->imp_connect_data;
@@ -385,15 +366,26 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
        imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 
        rc = ptlrpc_connect_import(imp);
-       if (rc) {
+       if (rc != 0) {
                CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
-               GOTO(out, rc);
+               GOTO(out_dis, rc);
        }
 
        ptlrpc_pinger_add_import(imp);
 
-out:
-       RETURN(rc);
+       GOTO(out_dis, rc = 0);
+
+out_dis:
+       if (rc != 0) {
+               class_disconnect(*exp);
+               *exp = NULL;
+               lwp->lpd_exp = NULL;
+       }
+
+out_sem:
+       up_write(&cli->cl_sem);
+
+       return rc;
 }
 
 static int lwp_obd_disconnect(struct obd_export *exp)
index 9d9887c..990cfee 100644 (file)
@@ -728,7 +728,8 @@ int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd)
        }
 
        /* generate osp name */
-       rc = tgt_name2lwpname((char *)qsd->qsd_svname, qti->qti_buf);
+       rc = tgt_name2lwp_name(qsd->qsd_svname, qti->qti_buf,
+                              MTI_NAME_MAXLEN, 0);
        if (rc) {
                CERROR("%s: failed to generate ospname (%d)\n",
                       qsd->qsd_svname, rc);