From 7a303a8250c7f2e20a72089cca6d9ac52daff632 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Sat, 25 Jan 2014 03:41:42 +0800 Subject: [PATCH] LU-3951 lfsck: LWP connection from OST-x to MDT-y When client sends object-based RPC to the OST-x, the RPC service thread on the OST-x needs to verify whether the given parent FID information in the client RPC matches the parent FID information stored in the OST-object. If not match, it will query the MDT-y according to the client given parent FID information. The query RPC from the OST-x to the MDT-y is sent via LWP connection. The other use case is that the LFSCK on the OST-x needs to talk with the LFSCK on the MDT-y, such control/query RPC will be via above LWP connection from OST-x to MDT-y. Currently, we only support LWP connection frm OST-x to the MDT-0. This patch enhance that to enable LWP connection from any OST to any MDT. Test-Parameters: allwaysuploadlogs Signed-off-by: Fan Yong Change-Id: Ie98be82b3af90456d1838d53b6d77c12956f7bd7 Reviewed-on: http://review.whamcloud.com/7666 Tested-by: Jenkins Reviewed-by: Andreas Dilger Reviewed-by: Alex Zhuravlev Tested-by: Maloo Reviewed-by: Oleg Drokin --- libcfs/include/libcfs/libcfs_string.h | 1 + libcfs/libcfs/libcfs_string.c | 27 +++ lustre/include/lustre_disk.h | 4 +- lustre/include/obd.h | 2 + lustre/include/obd_class.h | 24 ++- lustre/mdt/mdt_handler.c | 2 +- lustre/obdclass/obd_config.c | 4 + lustre/obdclass/obd_mount.c | 4 +- lustre/obdclass/obd_mount_server.c | 394 +++++++++++++++++++--------------- lustre/ofd/ofd_fs.c | 2 +- lustre/ofd/ofd_objects.c | 2 +- lustre/osp/lwp_dev.c | 104 +++++---- lustre/quota/qsd_lib.c | 3 +- 13 files changed, 333 insertions(+), 240 deletions(-) diff --git a/libcfs/include/libcfs/libcfs_string.h b/libcfs/include/libcfs/libcfs_string.h index e402b55..82c5c27 100644 --- a/libcfs/include/libcfs/libcfs_string.h +++ b/libcfs/include/libcfs/libcfs_string.h @@ -44,6 +44,7 @@ #define __LIBCFS_STRING_H__ /* libcfs_string.c */ +char *cfs_strrstr(const char *haystack, const char *needle); /* string comparison ignoring case */ int cfs_strncasecmp(const char *s1, const char *s2, size_t n); /* Convert a text string to a bitmask */ diff --git a/libcfs/libcfs/libcfs_string.c b/libcfs/libcfs/libcfs_string.c index be26f93..1d7a941 100644 --- a/libcfs/libcfs/libcfs_string.c +++ b/libcfs/libcfs/libcfs_string.c @@ -42,6 +42,33 @@ #include +char *cfs_strrstr(const char *haystack, const char *needle) +{ + char *ptr; + + if (unlikely(haystack == NULL || needle == NULL)) + return NULL; + + if (strlen(needle) == 1) + return strrchr(haystack, needle[0]); + + ptr = strstr(haystack, needle); + if (ptr != NULL) { + while (1) { + char *tmp; + + tmp = strstr(&ptr[1], needle); + if (tmp == NULL) + return ptr; + + ptr = tmp; + } + } + + return NULL; +} +EXPORT_SYMBOL(cfs_strrstr); + /* non-0 = don't match */ int cfs_strncasecmp(const char *s1, const char *s2, size_t n) { diff --git a/lustre/include/lustre_disk.h b/lustre/include/lustre_disk.h index 09b72a3..a61f7be 100644 --- a/lustre/include/lustre_disk.h +++ b/lustre/include/lustre_disk.h @@ -500,6 +500,9 @@ struct lustre_sb_info { char lsi_fstype[16]; struct backing_dev_info lsi_bdi; /* each client mountpoint needs own backing_dev_info */ + struct list_head lsi_lwp_list; + spinlock_t lsi_lwp_lock; + unsigned long lsi_lwp_started:1; }; #define LSI_UMOUNT_FAILOVER 0x00200000 @@ -527,7 +530,6 @@ struct lustre_mount_info { #ifdef __KERNEL__ /* obd_mount.c */ int server_name2fsname(const char *svname, char *fsname, const char **endptr); -int server_name2index(const char *svname, __u32 *idx, const char **endptr); int server_name2svname(const char *label, char *svname, const char **endptr, size_t svsize); int server_name_is_ost(const char *svname); diff --git a/lustre/include/obd.h b/lustre/include/obd.h index e19aab1..c18052b 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -752,6 +752,7 @@ struct obd_device { cfs_list_t obd_exports; cfs_list_t obd_unlinked_exports; cfs_list_t obd_delayed_exports; + struct list_head obd_lwp_list; int obd_num_exports; spinlock_t obd_nid_lock; struct ldlm_namespace *obd_namespace; @@ -769,6 +770,7 @@ struct obd_device { struct rw_semaphore obd_observer_link_sem; struct obd_notify_upcall obd_upcall; struct obd_export *obd_self_export; + struct obd_export *obd_lwp_export; /* list of exports in LRU order, for ping evictor, with obd_dev_lock */ cfs_list_t obd_exports_timed; time_t obd_eviction_timer; /* for ping evictor */ diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 11917f3..da400c2 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -167,13 +167,14 @@ int class_add_uuid(const char *uuid, __u64 nid); /* Passed as data param to class_config_parse_llog */ struct config_llog_instance { - char *cfg_obdname; - void *cfg_instance; - struct super_block *cfg_sb; - struct obd_uuid cfg_uuid; - llog_cb_t cfg_callback; - int cfg_last_idx; /* for partial llog processing */ - int cfg_flags; + char *cfg_obdname; + void *cfg_instance; + struct super_block *cfg_sb; + struct obd_uuid cfg_uuid; + llog_cb_t cfg_callback; + int cfg_last_idx; /* for partial llog processing */ + int cfg_flags; + __u32 cfg_lwp_idx; }; int class_config_parse_llog(const struct lu_env *env, struct llog_ctxt *ctxt, char *name, struct config_llog_instance *cfg); @@ -2272,7 +2273,8 @@ extern int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c); int lustre_register_lwp_item(const char *lwpname, struct obd_export **exp, register_lwp_cb cb_func, void *cb_data); void lustre_deregister_lwp_item(struct obd_export **exp); -int tgt_name2lwpname(const char *tgt_name, char *lwp_name); +struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx); +int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx); #endif /* HAVE_SERVER_SUPPORT */ /* sysctl.c */ @@ -2298,4 +2300,10 @@ int raw_name2idx(int hashtype, int count, const char *name, int namelen); /* prng.c */ #define ll_generate_random_uuid(uuid_out) cfs_get_random_bytes(uuid_out, sizeof(class_uuid_t)) +#ifdef __KERNEL__ +int server_name2index(const char *svname, __u32 *idx, const char **endptr); +#else +# define server_name2index(name, idx, ptr) do {} while (0) +#endif + #endif /* __LINUX_OBD_CLASS_H */ diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index f882b4d..ed52643 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -3505,7 +3505,7 @@ static int mdt_register_seq_exp(struct mdt_device *mdt) if (lwp_name == NULL) GOTO(out_free, rc = -ENOMEM); - rc = tgt_name2lwpname(mdt_obd_name(mdt), lwp_name); + rc = tgt_name2lwp_name(mdt_obd_name(mdt), lwp_name, MAX_OBD_NAME, 0); if (rc != 0) GOTO(out_free, rc); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index 83465e2..8fa81b3 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -42,6 +42,7 @@ #ifdef __KERNEL__ #include #include +#include #else #include #include @@ -410,6 +411,7 @@ int class_attach(struct lustre_cfg *lcfg) CFS_INIT_LIST_HEAD(&obd->obd_lock_replay_queue); CFS_INIT_LIST_HEAD(&obd->obd_final_req_queue); CFS_INIT_LIST_HEAD(&obd->obd_evict_list); + INIT_LIST_HEAD(&obd->obd_lwp_list); llog_group_init(&obd->obd_olg, FID_SEQ_LLOG); @@ -1514,6 +1516,8 @@ int class_config_llog_handler(const struct lu_env *env, if (marker->cm_flags & CM_START) { /* all previous flags off */ clli->cfg_flags = CFG_F_MARKER; + server_name2index(marker->cm_tgtname, + &clli->cfg_lwp_idx, NULL); if (marker->cm_flags & CM_SKIP) { clli->cfg_flags |= CFG_F_SKIP; CDEBUG(D_CONFIG, "SKIP #%d\n", diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 0fa0463..d592459 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -588,8 +588,10 @@ struct lustre_sb_info *lustre_init_lsi(struct super_block *sb) /* Default umount style */ lsi->lsi_flags = LSI_UMOUNT_FAILOVER; + INIT_LIST_HEAD(&lsi->lsi_lwp_list); + spin_lock_init(&lsi->lsi_lwp_lock); - RETURN(lsi); + RETURN(lsi); } static int lustre_free_lsi(struct super_block *sb) diff --git a/lustre/obdclass/obd_mount_server.c b/lustre/obdclass/obd_mount_server.c index cff3d80..0567556 100644 --- a/lustre/obdclass/obd_mount_server.c +++ b/lustre/obdclass/obd_mount_server.c @@ -327,54 +327,28 @@ static int server_mgc_clear_fs(struct obd_device *mgc) RETURN(rc); } -static int is_mdc_device(const char *devname) +static inline bool is_mdc_device(const char *devname) { char *ptr; ptr = strrchr(devname, '-'); - if (ptr != NULL && strcmp(ptr, "-mdc") == 0) - return 1; - - return 0; + return ptr != NULL && strcmp(ptr, "-mdc") == 0; } -static inline int tgt_is_mdt0(const char *tgtname) +static inline bool tgt_is_mdt(const char *tgtname, __u32 *idx) { - __u32 idx; - int type; + int type; - type = server_name2index(tgtname, &idx, NULL); - if (type != LDD_F_SV_TYPE_MDT) - return 0; + type = server_name2index(tgtname, idx, NULL); - return idx == 0; -} - -static inline int is_mdc_for_mdt0(const char *devname) -{ - char *ptr; - - if (!is_mdc_device(devname)) - return 0; - - ptr = strrchr(devname, '-'); - if (ptr == NULL) - return 0; - - *ptr = 0; - if (tgt_is_mdt0(devname)) { - *ptr = '-'; - return 1; - } - *ptr = '-'; - return 0; + return type == LDD_F_SV_TYPE_MDT; } /** - * Convert OST/MDT name(fsname-OSTxxxx) to a lwp name - * (fsname-MDT0000-lwp-OSTxxxx) + * Convert OST/MDT name(fsname-{MDT,OST}xxxx) to a lwp name with the @idx:yyyy + * (fsname-MDTyyyy-lwp-{MDT,OST}xxxx) **/ -int tgt_name2lwpname(const char *svname, char *lwpname) +int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx) { char *fsname; const char *tgt; @@ -385,30 +359,35 @@ int tgt_name2lwpname(const char *svname, char *lwpname) if (fsname == NULL) RETURN(-ENOMEM); - rc = server_name2fsname(svname, fsname, &tgt); + rc = server_name2fsname(tgt_name, fsname, &tgt); if (rc != 0) { - CERROR("%s: failed to get fsname from svname. %d\n", - svname, rc); + CERROR("%s: failed to get fsname from tgt_name: rc = %d\n", + tgt_name, rc); GOTO(cleanup, rc); } if (*tgt != '-' && *tgt != ':') { - CERROR("%s: invalid svname name!\n", svname); + CERROR("%s: invalid tgt_name name!\n", tgt_name); GOTO(cleanup, rc = -EINVAL); } tgt++; if (strncmp(tgt, "OST", 3) != 0 && strncmp(tgt, "MDT", 3) != 0) { - CERROR("%s is not an OST or MDT target!\n", svname); + CERROR("%s is not an OST or MDT target!\n", tgt_name); GOTO(cleanup, rc = -EINVAL); } - sprintf(lwpname, "%s-MDT0000-%s-%s", fsname, LUSTRE_LWP_NAME, tgt); + snprintf(lwp_name, len, "%s-MDT%04x-%s-%s", + fsname, idx, LUSTRE_LWP_NAME, tgt); + + GOTO(cleanup, rc = 0); + cleanup: if (fsname != NULL) OBD_FREE(fsname, MTI_NAME_MAXLEN); - RETURN(rc); + + return rc; } -EXPORT_SYMBOL(tgt_name2lwpname); +EXPORT_SYMBOL(tgt_name2lwp_name); static CFS_LIST_HEAD(lwp_register_list); DEFINE_MUTEX(lwp_register_list_lock); @@ -478,6 +457,44 @@ void lustre_deregister_lwp_item(struct obd_export **exp) } EXPORT_SYMBOL(lustre_deregister_lwp_item); +struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx) +{ + struct lustre_mount_info *lmi; + struct lustre_sb_info *lsi; + struct obd_device *lwp; + struct obd_export *exp = NULL; + char fsname[16]; + char lwp_name[24]; + int rc; + + lmi = server_get_mount_2(dev); + if (lmi == NULL) + return NULL; + + lsi = s2lsi(lmi->lmi_sb); + rc = server_name2fsname(lsi->lsi_svname, fsname, NULL); + if (rc != 0) { + CERROR("%s: failed to get fsname: rc = %d\n", + lsi->lsi_svname, rc); + return NULL; + } + + snprintf(lwp_name, sizeof(lwp_name), "%s-MDT%04x", fsname, idx); + spin_lock(&lsi->lsi_lwp_lock); + list_for_each_entry(lwp, &lsi->lsi_lwp_list, obd_lwp_list) { + char *ptr = strstr(lwp->obd_name, lwp_name); + + if (ptr != NULL) { + exp = class_export_get(lwp->obd_lwp_export); + break; + } + } + spin_unlock(&lsi->lsi_lwp_lock); + + return exp; +} +EXPORT_SYMBOL(lustre_find_lwp_by_index); + static void lustre_notify_lwp_list(struct obd_export *exp) { struct lwp_register_item *lri, *tmp; @@ -539,10 +556,16 @@ static int lustre_lwp_connect(struct obd_device *lwp) /* Use lwp name as the uuid, so we find the export by lwp name later */ memcpy(uuid->uuid, lwp->obd_name, strlen(lwp->obd_name)); rc = obd_connect(&env, &exp, lwp, uuid, data, NULL); - if (rc != 0) + if (rc != 0) { CERROR("%s: connect failed: rc = %d\n", lwp->obd_name, rc); - else + } else { + if (unlikely(lwp->obd_lwp_export != NULL)) + class_export_put(lwp->obd_lwp_export); + lwp->obd_lwp_export = class_export_get(exp); lustre_notify_lwp_list(exp); + } + + GOTO(out, rc); out: if (data != NULL) @@ -554,14 +577,15 @@ out: lu_context_exit(&session_ctx); lu_context_fini(&session_ctx); - RETURN(rc); + return rc; } /** - * lwp is used by slaves (Non-MDT0 targets) to manage the connection - * to MDT0. + * lwp is used by slaves (Non-MDT0 targets) to manage the connection to MDT0, + * or from the OSTx to MDTy. **/ -static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) +static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi, + __u32 idx) { struct obd_device *obd; char *lwpname = NULL; @@ -571,18 +595,18 @@ static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) rc = class_add_uuid(lustre_cfg_string(lcfg, 1), lcfg->lcfg_nid); - if (rc) { + if (rc != 0) { CERROR("%s: Can't add uuid: rc =%d\n", lsi->lsi_svname, rc); - GOTO(out, rc); + RETURN(rc); } OBD_ALLOC(lwpname, MTI_NAME_MAXLEN); if (lwpname == NULL) GOTO(out, rc = -ENOMEM); - rc = tgt_name2lwpname(lsi->lsi_svname, lwpname); + rc = tgt_name2lwp_name(lsi->lsi_svname, lwpname, MTI_NAME_MAXLEN, idx); if (rc != 0) { - CERROR("%s: failed to generate lwp name. %d\n", + CERROR("%s: failed to generate lwp name: rc = %d\n", lsi->lsi_svname, rc); GOTO(out, rc); } @@ -604,20 +628,29 @@ static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) LASSERT(obd != NULL); rc = lustre_lwp_connect(obd); - if (rc != 0) + if (rc == 0) { + obd->u.cli.cl_max_mds_easize = MAX_MD_SIZE; + spin_lock(&lsi->lsi_lwp_lock); + list_add_tail(&obd->obd_lwp_list, &lsi->lsi_lwp_list); + spin_unlock(&lsi->lsi_lwp_lock); + } else { CERROR("%s: connect failed: rc = %d\n", lwpname, rc); + } + + GOTO(out, rc); + out: if (lwpname != NULL) OBD_FREE(lwpname, MTI_NAME_MAXLEN); if (lwpuuid != NULL) OBD_FREE(lwpuuid, MTI_NAME_MAXLEN); - RETURN(rc); + return rc; } /* the caller is responsible for memory free */ static struct obd_device *lustre_find_lwp(struct lustre_sb_info *lsi, - char **lwpname, char **logname) + char **lwpname, __u32 idx) { struct obd_device *lwp; int rc = 0; @@ -630,22 +663,9 @@ static struct obd_device *lustre_find_lwp(struct lustre_sb_info *lsi, if (*lwpname == NULL) RETURN(ERR_PTR(-ENOMEM)); - if (logname != NULL) { - OBD_ALLOC(*logname, MTI_NAME_MAXLEN); - if (*logname == NULL) - GOTO(out, rc = -ENOMEM); - rc = server_name2fsname(lsi->lsi_svname, *lwpname, NULL); - if (rc != 0) { - CERROR("%s: failed to get fsname from svname. %d\n", - lsi->lsi_svname, rc); - GOTO(out, rc = -EINVAL); - } - sprintf(*logname, "%s-client", *lwpname); - } - - rc = tgt_name2lwpname(lsi->lsi_svname, *lwpname); + rc = tgt_name2lwp_name(lsi->lsi_svname, *lwpname, MTI_NAME_MAXLEN, idx); if (rc != 0) { - CERROR("%s: failed to generate lwp name. %d\n", + CERROR("%s: failed to generate lwp name: rc = %d\n", lsi->lsi_svname, rc); GOTO(out, rc = -EINVAL); } @@ -658,10 +678,6 @@ out: OBD_FREE(*lwpname, MTI_NAME_MAXLEN); *lwpname = NULL; } - if (logname != NULL && *logname != NULL) { - OBD_FREE(*logname, MTI_NAME_MAXLEN); - *logname = NULL; - } lwp = ERR_PTR(rc); } @@ -669,7 +685,7 @@ out: } static int lustre_lwp_add_conn(struct lustre_cfg *cfg, - struct lustre_sb_info *lsi) + struct lustre_sb_info *lsi, __u32 idx) { struct lustre_cfg_bufs *bufs = NULL; struct lustre_cfg *lcfg = NULL; @@ -678,7 +694,7 @@ static int lustre_lwp_add_conn(struct lustre_cfg *cfg, int rc; ENTRY; - lwp = lustre_find_lwp(lsi, &lwpname, NULL); + lwp = lustre_find_lwp(lsi, &lwpname, idx); if (IS_ERR(lwp)) { CERROR("%s: can't find lwp device.\n", lsi->lsi_svname); GOTO(out, rc = PTR_ERR(lwp)); @@ -713,19 +729,19 @@ out: * Retrieve MDT nids from the client log, then start the lwp device. * there are only two scenarios which would include mdt nid. * 1. - * marker 5 (flags=0x01, v2.1.54.0) lustre-MDT0000 'add mdc' xxx- + * marker 5 (flags=0x01, v2.1.54.0) lustre-MDTyyyy 'add mdc' xxx- * add_uuid nid=192.168.122.162@tcp(0x20000c0a87aa2) 0: 1:192.168.122.162@tcp - * attach 0:lustre-MDT0000-mdc 1:mdc 2:lustre-clilmv_UUID - * setup 0:lustre-MDT0000-mdc 1:lustre-MDT0000_UUID 2:192.168.122.162@tcp + * attach 0:lustre-MDTyyyy-mdc 1:mdc 2:lustre-clilmv_UUID + * setup 0:lustre-MDTyyyy-mdc 1:lustre-MDTyyyy_UUID 2:192.168.122.162@tcp * add_uuid nid=192.168.172.1@tcp(0x20000c0a8ac01) 0: 1:192.168.172.1@tcp - * add_conn 0:lustre-MDT0000-mdc 1:192.168.172.1@tcp - * modify_mdc_tgts add 0:lustre-clilmv 1:lustre-MDT0000_UUID xxxx - * marker 5 (flags=0x02, v2.1.54.0) lustre-MDT0000 'add mdc' xxxx- + * add_conn 0:lustre-MDTyyyy-mdc 1:192.168.172.1@tcp + * modify_mdc_tgts add 0:lustre-clilmv 1:lustre-MDTyyyy_UUID xxxx + * marker 5 (flags=0x02, v2.1.54.0) lustre-MDTyyyy 'add mdc' xxxx- * 2. - * marker 7 (flags=0x01, v2.1.54.0) lustre-MDT0000 'add failnid' xxxx- + * marker 7 (flags=0x01, v2.1.54.0) lustre-MDTyyyy 'add failnid' xxxx- * add_uuid nid=192.168.122.2@tcp(0x20000c0a87a02) 0: 1:192.168.122.2@tcp - * add_conn 0:lustre-MDT0000-mdc 1:192.168.122.2@tcp - * marker 7 (flags=0x02, v2.1.54.0) lustre-MDT0000 'add failnid' xxxx- + * add_conn 0:lustre-MDTyyyy-mdc 1:192.168.122.2@tcp + * marker 7 (flags=0x02, v2.1.54.0) lustre-MDTyyyy 'add failnid' xxxx- **/ static int client_lwp_config_process(const struct lu_env *env, struct llog_handle *handle, @@ -768,7 +784,10 @@ static int client_lwp_config_process(const struct lu_env *env, marker->cm_flags & CM_EXCLUDE) GOTO(out, rc = 0); - if (!tgt_is_mdt0(marker->cm_tgtname)) + if (!tgt_is_mdt(marker->cm_tgtname, &clli->cfg_lwp_idx)) + GOTO(out, rc = 0); + + if (IS_MDT(lsi) && clli->cfg_lwp_idx != 0) GOTO(out, rc = 0); if (!strncmp(marker->cm_comment, "add mdc", 7) || @@ -789,7 +808,7 @@ static int client_lwp_config_process(const struct lu_env *env, } case LCFG_ADD_UUID: { if (clli->cfg_flags == CFG_F_MARKER) { - rc = lustre_lwp_setup(lcfg, lsi); + rc = lustre_lwp_setup(lcfg, lsi, clli->cfg_lwp_idx); /* XXX: process only the first nid as * we don't need another instance of lwp */ clli->cfg_flags |= CFG_F_SKIP; @@ -803,8 +822,25 @@ static int client_lwp_config_process(const struct lu_env *env, break; } case LCFG_ADD_CONN: { - if (is_mdc_for_mdt0(lustre_cfg_string(lcfg, 0))) - rc = lustre_lwp_add_conn(lcfg, lsi); + char *devname = lustre_cfg_string(lcfg, 0); + char *ptr; + __u32 idx = 0; + + if (!is_mdc_device(devname)) + break; + + ptr = strrchr(devname, '-'); + if (ptr == NULL) + break; + + *ptr = 0; + if (!tgt_is_mdt(devname, &idx)) { + *ptr = '-'; + break; + } + + *ptr = '-'; + rc = lustre_lwp_add_conn(lcfg, lsi, idx); break; } default: @@ -816,67 +852,82 @@ out: static int lustre_disconnect_lwp(struct super_block *sb) { - struct lustre_sb_info *lsi = s2lsi(sb); + struct lustre_sb_info *lsi = s2lsi(sb); struct obd_device *lwp; - char *lwpname = NULL; char *logname = NULL; - struct lustre_cfg *lcfg = NULL; - struct lustre_cfg_bufs *bufs = NULL; - struct config_llog_instance *cfg = NULL; - int rc; + struct lustre_cfg_bufs *bufs = NULL; + struct config_llog_instance *cfg = NULL; + int rc = 0; + int rc1 = 0; ENTRY; - lwp = lustre_find_lwp(lsi, &lwpname, &logname); - if (IS_ERR(lwp) && PTR_ERR(lwp) != -ENOENT) - GOTO(out, rc = PTR_ERR(lwp)); + if (likely(lsi->lsi_lwp_started)) { + OBD_ALLOC(logname, MTI_NAME_MAXLEN); + if (logname == NULL) + RETURN(-ENOMEM); - LASSERT(lwpname != NULL); - LASSERT(logname != NULL); + rc = server_name2fsname(lsi->lsi_svname, logname, NULL); + if (rc != 0) { + CERROR("%s: failed to get fsname from svname: " + "rc = %d\n", lsi->lsi_svname, rc); + GOTO(out, rc = -EINVAL); + } - OBD_ALLOC_PTR(cfg); - if (cfg == NULL) - GOTO(out, rc = -ENOMEM); + strcat(logname, "-client"); + OBD_ALLOC_PTR(cfg); + if (cfg == NULL) + GOTO(out, rc = -ENOMEM); - /* end log first */ - cfg->cfg_instance = sb; - rc = lustre_end_log(sb, logname, cfg); - if (rc != 0) { - CERROR("%s: Can't end config log %s.\n", lwpname, logname); - GOTO(out, rc); - } + /* end log first */ + cfg->cfg_instance = sb; + rc = lustre_end_log(sb, logname, cfg); + if (rc != 0) + GOTO(out, rc); - if (PTR_ERR(lwp) == -ENOENT) { - CDEBUG(D_CONFIG, "%s: lwp device wasn't started.\n", - lsi->lsi_svname); - GOTO(out, rc = 0); + lsi->lsi_lwp_started = 0; } OBD_ALLOC_PTR(bufs); if (bufs == NULL) GOTO(out, rc = -ENOMEM); - lustre_cfg_bufs_reset(bufs, lwp->obd_name); - lustre_cfg_bufs_set_string(bufs, 1, NULL); - lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs); - if (!lcfg) - GOTO(out, rc = -ENOMEM); + list_for_each_entry(lwp, &lsi->lsi_lwp_list, obd_lwp_list) { + struct lustre_cfg *lcfg; - /* Disconnect import first. NULL is passed for the '@env', since - * it will not be used. */ - rc = lwp->obd_lu_dev->ld_ops->ldo_process_config(NULL, lwp->obd_lu_dev, - lcfg); -out: - if (lcfg) + if (likely(lwp->obd_lwp_export != NULL)) { + class_export_put(lwp->obd_lwp_export); + lwp->obd_lwp_export = NULL; + } + + lustre_cfg_bufs_reset(bufs, lwp->obd_name); + lustre_cfg_bufs_set_string(bufs, 1, NULL); + lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs); + if (lcfg == NULL) + GOTO(out, rc = -ENOMEM); + + /* Disconnect import first. NULL is passed for the '@env', + * since it will not be used. */ + rc = lwp->obd_lu_dev->ld_ops->ldo_process_config(NULL, + lwp->obd_lu_dev, lcfg); lustre_cfg_free(lcfg); - if (bufs) + if (rc != 0 && rc != -ETIMEDOUT) { + CERROR("%s: fail to disconnect LWP: rc = %d\n", + lwp->obd_name, rc); + rc1 = rc; + } + } + + GOTO(out, rc); + +out: + if (bufs != NULL) OBD_FREE_PTR(bufs); - if (cfg) + if (cfg != NULL) OBD_FREE_PTR(cfg); - if (lwpname) - OBD_FREE(lwpname, MTI_NAME_MAXLEN); - if (logname) + if (logname != NULL) OBD_FREE(logname, MTI_NAME_MAXLEN); - RETURN(rc); + + return rc1 != 0 ? rc1 : rc; } /** @@ -885,69 +936,71 @@ out: static int lustre_stop_lwp(struct super_block *sb) { struct lustre_sb_info *lsi = s2lsi(sb); - struct obd_device *lwp = NULL; - char *lwpname = NULL; - int rc = 0; + struct obd_device *lwp; + int rc = 0; + int rc1 = 0; ENTRY; - lwp = lustre_find_lwp(lsi, &lwpname, NULL); - if (IS_ERR(lwp)) { - CDEBUG(PTR_ERR(lwp) == -ENOENT ? D_CONFIG : D_ERROR, - "%s: lwp wasn't started.\n", lsi->lsi_svname); - GOTO(out, rc = 0); + while (!list_empty(&lsi->lsi_lwp_list)) { + lwp = list_entry(lsi->lsi_lwp_list.next, struct obd_device, + obd_lwp_list); + list_del_init(&lwp->obd_lwp_list); + lwp->obd_force = 1; + rc = class_manual_cleanup(lwp); + if (rc != 0) { + CERROR("%s: fail to stop LWP: rc = %d\n", + lwp->obd_name, rc); + rc1 = rc; + } } - lwp->obd_force = 1; - rc = class_manual_cleanup(lwp); - -out: - if (lwpname != NULL) - OBD_FREE(lwpname, MTI_NAME_MAXLEN); - RETURN(rc); + RETURN(rc1 != 0 ? rc1 : rc); } /** - * Start the lwp(fsname-MDT0000-lwp-OSTxxxx) for an OST or MDT target, - * which would be used to establish connection from OST to MDT0. + * Start the lwp(fsname-MDTyyyy-lwp-{MDT,OST}xxxx) for a MDT/OST or MDT target. **/ static int lustre_start_lwp(struct super_block *sb) { struct lustre_sb_info *lsi = s2lsi(sb); struct config_llog_instance *cfg = NULL; - struct obd_device *lwp; - char *lwpname = NULL; - char *logname = NULL; + char *logname; int rc; ENTRY; - lwp = lustre_find_lwp(lsi, &lwpname, &logname); + if (unlikely(lsi->lsi_lwp_started)) + RETURN(0); - /* the lwp device already stared */ - if (lwp && !IS_ERR(lwp)) - GOTO(out, rc = 0); - - if (PTR_ERR(lwp) != -ENOENT) - GOTO(out, rc = PTR_ERR(lwp)); + OBD_ALLOC(logname, MTI_NAME_MAXLEN); + if (logname == NULL) + RETURN(-ENOMEM); - LASSERT(lwpname != NULL); - LASSERT(logname != NULL); + rc = server_name2fsname(lsi->lsi_svname, logname, NULL); + if (rc != 0) { + CERROR("%s: failed to get fsname from svname: rc = %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc = -EINVAL); + } + strcat(logname, "-client"); OBD_ALLOC_PTR(cfg); if (cfg == NULL) GOTO(out, rc = -ENOMEM); cfg->cfg_callback = client_lwp_config_process; cfg->cfg_instance = sb; - rc = lustre_process_log(sb, logname, cfg); + if (rc == 0) + lsi->lsi_lwp_started = 1; + + GOTO(out, rc); + out: - if (lwpname != NULL) - OBD_FREE(lwpname, MTI_NAME_MAXLEN); - if (logname != NULL) - OBD_FREE(logname, MTI_NAME_MAXLEN); + OBD_FREE(logname, MTI_NAME_MAXLEN); if (cfg != NULL) OBD_FREE_PTR(cfg); - RETURN(rc); + + return rc; } DEFINE_MUTEX(server_start_lock); @@ -1414,9 +1467,10 @@ static void server_put_super(struct super_block *sb) int rc; rc = lustre_disconnect_lwp(sb); - if (rc && rc != ETIMEDOUT) - CERROR("%s: failed to disconnect lwp. (rc=%d)\n", - tmpname, rc); + if (rc != 0 && rc != -ETIMEDOUT && + rc != -ENOTCONN && rc != -ESHUTDOWN) + CWARN("%s: failed to disconnect lwp: rc= %d\n", + tmpname, rc); } /* Stop the target */ diff --git a/lustre/ofd/ofd_fs.c b/lustre/ofd/ofd_fs.c index e5fd706..80a5ae4 100644 --- a/lustre/ofd/ofd_fs.c +++ b/lustre/ofd/ofd_fs.c @@ -402,7 +402,7 @@ static int ofd_register_seq_exp(struct ofd_device *ofd) if (lwp_name == NULL) GOTO(out_free, rc = -ENOMEM); - rc = tgt_name2lwpname(ofd_name(ofd), lwp_name); + rc = tgt_name2lwp_name(ofd_name(ofd), lwp_name, MAX_OBD_NAME, 0); if (rc != 0) GOTO(out_free, rc); diff --git a/lustre/ofd/ofd_objects.c b/lustre/ofd/ofd_objects.c index cd8c26a..6458498 100644 --- a/lustre/ofd/ofd_objects.c +++ b/lustre/ofd/ofd_objects.c @@ -270,7 +270,7 @@ int ofd_precreate_objects(const struct lu_env *env, struct ofd_device *ofd, /* When the LFSCK scanning the whole device to verify the LAST_ID file * consistency, it will load the last_id into RAM firstly, and compare - * the last_id with echo OST-object's ID. If the later one is larger, + * the last_id with each OST-object's ID. If the later one is larger, * then it will regard the LAST_ID file crashed. But during the LFSCK * scanning, the OFD may continue to create new OST-objects. Those new * created OST-objects will have larger IDs than the LFSCK known ones. diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c index fce82a2..9560504 100644 --- a/lustre/osp/lwp_dev.c +++ b/lustre/osp/lwp_dev.c @@ -37,6 +37,7 @@ #include #include #include +#include struct lwp_device { struct lu_device lpd_dev; @@ -56,40 +57,17 @@ static inline struct lu_device *lwp2lu_dev(struct lwp_device *d) return &d->lpd_dev; } -static int lwp_name2fsname(char *lwpname, char *fsname) -{ - char *ptr; - - LASSERT(lwpname != NULL); - LASSERT(fsname != NULL); - - sprintf(fsname, "-%s-", LUSTRE_LWP_NAME); - - ptr = strstr(lwpname, fsname); - if (ptr == NULL) - return -EINVAL; - - while (*(--ptr) != '-') { - if (ptr == lwpname) - return -EINVAL; - } - - strncpy(fsname, lwpname, ptr - lwpname); - fsname[ptr - lwpname] = '\0'; - - return 0; -} - static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, char *nidstring) { struct lustre_cfg_bufs *bufs = NULL; struct lustre_cfg *lcfg = NULL; - char *lwpname = lwp->lpd_obd->obd_name; - char *fsname = NULL; + char *lwp_name = lwp->lpd_obd->obd_name; char *server_uuid = NULL; + char *ptr; class_uuid_t uuid; struct obd_import *imp; + int len = strlen(lwp_name); int rc; ENTRY; @@ -97,23 +75,22 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, if (bufs == NULL) RETURN(-ENOMEM); - OBD_ALLOC(fsname, strlen(lwpname)); - if (fsname == NULL) + OBD_ALLOC(server_uuid, len); + if (server_uuid == NULL) GOTO(out, rc = -ENOMEM); - rc = lwp_name2fsname(lwpname, fsname); - if (rc) { - CERROR("%s: failed to get fsname from lwpname. %d\n", - lwpname, rc); - GOTO(out, rc); + snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME); + ptr = cfs_strrstr(lwp_name, server_uuid); + if (ptr == NULL) { + CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n", + lwp_name, -EINVAL); + GOTO(out, rc = -EINVAL); } - OBD_ALLOC(server_uuid, strlen(fsname) + 15); - if (server_uuid == NULL) - GOTO(out, rc = -ENOMEM); - - sprintf(server_uuid, "%s-MDT0000_UUID", fsname); - lustre_cfg_bufs_reset(bufs, lwpname); + strncpy(server_uuid, lwp_name, ptr - lwp_name); + server_uuid[ptr - lwp_name] = '\0'; + strncat(server_uuid, "_UUID", len - 1); + lustre_cfg_bufs_reset(bufs, lwp_name); lustre_cfg_bufs_set_string(bufs, 1, server_uuid); lustre_cfg_bufs_set_string(bufs, 2, nidstring); lcfg = lustre_cfg_new(LCFG_SETUP, bufs); @@ -138,9 +115,7 @@ out: if (bufs != NULL) OBD_FREE_PTR(bufs); if (server_uuid != NULL) - OBD_FREE(server_uuid, strlen(fsname) + 15); - if (fsname != NULL) - OBD_FREE(fsname, strlen(lwpname)); + OBD_FREE(server_uuid, len); if (lcfg != NULL) lustre_cfg_free(lcfg); if (rc) @@ -149,7 +124,7 @@ out: RETURN(rc); } -int lwp_disconnect(struct lwp_device *d) +static int lwp_disconnect(struct lwp_device *d) { struct obd_import *imp; int rc = 0; @@ -169,12 +144,13 @@ int lwp_disconnect(struct lwp_device *d) /* Some non-replayable imports (MDS's OSCs) are pinged, so just * delete it regardless. (It's safe to delete an import that was * never added.) */ - (void)ptlrpc_pinger_del_import(imp); - + ptlrpc_pinger_del_import(imp); rc = ptlrpc_disconnect_import(imp, 0); - if (rc && rc != -ETIMEDOUT) - CERROR("%s: can't disconnect: rc = %d\n", - d->lpd_obd->obd_name, rc); + if (rc != 0 && rc != -ETIMEDOUT && rc != -ENOTCONN && rc != -ESHUTDOWN) + CWARN("%s: can't disconnect: rc = %d\n", + d->lpd_obd->obd_name, rc); + else + rc = 0; ptlrpc_invalidate_import(imp); @@ -352,8 +328,9 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp, struct obd_connect_data *data, void *localdata) { struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev); + struct client_obd *cli = &lwp->lpd_obd->u.cli; + struct obd_import *imp = cli->cl_import; struct obd_connect_data *ocd; - struct obd_import *imp; struct lustre_handle conn; int rc; @@ -361,9 +338,11 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp, CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects); + *exp = NULL; + down_write(&cli->cl_sem); rc = class_connect(&conn, obd, cluuid); - if (rc) - RETURN(rc); + if (rc != 0) + GOTO(out_sem, rc); *exp = class_conn2export(&conn); lwp->lpd_exp = *exp; @@ -372,8 +351,10 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp, lwp->lpd_connects++; LASSERT(lwp->lpd_connects == 1); - imp = lwp->lpd_obd->u.cli.cl_import; imp->imp_dlm_handle = conn; + rc = ptlrpc_init_import(imp); + if (rc != 0) + GOTO(out_dis, rc); LASSERT(data != NULL); ocd = &imp->imp_connect_data; @@ -385,15 +366,26 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp, imp->imp_connect_flags_orig = ocd->ocd_connect_flags; rc = ptlrpc_connect_import(imp); - if (rc) { + if (rc != 0) { CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc); - GOTO(out, rc); + GOTO(out_dis, rc); } ptlrpc_pinger_add_import(imp); -out: - RETURN(rc); + GOTO(out_dis, rc = 0); + +out_dis: + if (rc != 0) { + class_disconnect(*exp); + *exp = NULL; + lwp->lpd_exp = NULL; + } + +out_sem: + up_write(&cli->cl_sem); + + return rc; } static int lwp_obd_disconnect(struct obd_export *exp) diff --git a/lustre/quota/qsd_lib.c b/lustre/quota/qsd_lib.c index 9d9887c..990cfee 100644 --- a/lustre/quota/qsd_lib.c +++ b/lustre/quota/qsd_lib.c @@ -728,7 +728,8 @@ int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd) } /* generate osp name */ - rc = tgt_name2lwpname((char *)qsd->qsd_svname, qti->qti_buf); + rc = tgt_name2lwp_name(qsd->qsd_svname, qti->qti_buf, + MTI_NAME_MAXLEN, 0); if (rc) { CERROR("%s: failed to generate ospname (%d)\n", qsd->qsd_svname, rc); -- 1.8.3.1