From: wangdi Date: Wed, 9 Oct 2013 02:56:07 +0000 (-0700) Subject: LU-2183 osp: cleanup osp-on-ost X-Git-Tag: 2.3.60~1 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=1bdf0d6fcbe11efdf67ebdb09f04955b9ce88de4 LU-2183 osp: cleanup osp-on-ost With new quota design and FIDonOST introduced, a new connection from OST/MDTn to MDT0 is required for sending quota and sequence RPCs. This connection was implemented by adding a new special osp device named osp-on-ost, which introduced complexity to osp code, and many hacks of checking the osp device name in both client and server side code. In this patch, we removed the osp-on-ost, and introduced a new device type lwp (Light Weight Proxy) to manage the light weight connection established from OSTs/MDTs to MDT0. Signed-off-by: Niu Yawei Signed-off-by: Wang Di Change-Id: Ic1ba25f941a4f971a542dc4be8a81e28ee0f477b Reviewed-on: http://review.whamcloud.com/4923 Tested-by: Hudson Reviewed-by: Andreas Dilger Reviewed-by: Alex Zhuravlev Reviewed-by: Mike Pershin Tested-by: Maloo --- diff --git a/lustre/fld/fld_handler.c b/lustre/fld/fld_handler.c index 0ee6852..cc2bb42 100644 --- a/lustre/fld/fld_handler.c +++ b/lustre/fld/fld_handler.c @@ -242,8 +242,7 @@ static int fld_req_handle(struct ptlrpc_request *req, * Set it as 'LU_SEQ_RANGE_MDT' by default. * Old 2.0 liblustre client cannot talk with new 2.1 server. */ if (!(exp->exp_connect_flags & OBD_CONNECT_64BITHASH) && - !((exp->exp_connect_flags & OBD_CONNECT_MDS) && - (exp->exp_connect_flags & OBD_CONNECT_FID)) && + !(exp->exp_connect_flags & OBD_CONNECT_MDS_MDS) && !(exp->exp_connect_flags & OBD_CONNECT_LIGHTWEIGHT) && !exp->exp_libclient) out->lsr_flags = LU_SEQ_RANGE_MDT; diff --git a/lustre/include/obd.h b/lustre/include/obd.h index cdf0a5f..7b50f69 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -760,16 +760,15 @@ struct niobuf_local { #define LUSTRE_FLD_NAME "fld" #define LUSTRE_SEQ_NAME "seq" -#define LUSTRE_CMM_NAME "cmm" #define LUSTRE_MDD_NAME "mdd" #define LUSTRE_OSD_LDISKFS_NAME "osd-ldiskfs" #define LUSTRE_OSD_ZFS_NAME "osd-zfs" #define LUSTRE_VVP_NAME "vvp" #define LUSTRE_LMV_NAME "lmv" -#define LUSTRE_CMM_MDC_NAME "cmm-mdc" #define LUSTRE_SLP_NAME "slp" #define LUSTRE_LOD_NAME "lod" #define LUSTRE_OSP_NAME "osp" +#define LUSTRE_LWP_NAME "lwp" /* obd device type names */ /* FIXME all the references to LUSTRE_MDS_NAME should be swapped with LUSTRE_MDT_NAME */ @@ -783,7 +782,6 @@ struct niobuf_local { #define LUSTRE_MGS_NAME "mgs" #define LUSTRE_MGC_NAME "mgc" -#define LUSTRE_CACHEOBD_NAME "cobd" #define LUSTRE_ECHO_NAME "obdecho" #define LUSTRE_ECHO_CLIENT_NAME "echo_client" #define LUSTRE_QMT_NAME "qmt" @@ -826,70 +824,6 @@ static inline int is_osp_on_mdt(char *name) /* Don't conflict with on-wire flags OBD_BRW_WRITE, etc */ #define N_LOCAL_TEMP_PAGE 0x10000000 -/* Currently the connection osp is only for connecting MDT0, the - * name would either be - * fsname-MDT0000-osp-OSTxxxx or fsname-MDT0000-osp-MDT0000 */ -static inline int is_osp_for_connection(char *name) -{ - char *ptr; - int osp_on_mdt0 = 0; - char *endptr; - - ptr = strrchr(name, '-'); - if (ptr == NULL) { - CERROR("%s is not a obdname\n", name); - return 0; - } - - if (strncmp(ptr + 1, "OST", 3) != 0 && strncmp(ptr + 1, "MDT", 3) != 0) - return 0; - - if (strncmp(ptr + 1, "MDT", 3) == 0) { - int index; - -#ifdef __KERNEL__ - index = simple_strtoul(ptr + 4, &endptr, 16); -#else - index = strtoul(ptr + 4, &endptr, 16); -#endif - if (index != 0) - return 0; - osp_on_mdt0 = 1; - } - - /* match the "-osp" */ - if (ptr - name < strlen(LUSTRE_OSP_NAME) + 1) - return 0; - - ptr -= (strlen(LUSTRE_OSP_NAME) + 1); - if (*ptr != '-') - return 0; - - if (strncmp(ptr + 1, LUSTRE_OSP_NAME, strlen(LUSTRE_OSP_NAME)) != 0) - return 0; - - if (osp_on_mdt0) { - int index = 0; - while (*(--ptr) != '-' && ptr != name); - - if (ptr == name) { - CERROR("%s is not a valid osp name\n", name); - return 0; - } - - if (strncmp(ptr + 1, "MDT", 3) != 0) - return 0; -#ifdef __KERNEL__ - index = simple_strtoul(ptr + 4, &endptr, 16); -#else - index = strtoul(ptr + 4, &endptr, 16); -#endif - if (index != 0) - return 0; - } - return 1; -} - struct obd_trans_info { __u64 oti_transno; __u64 oti_xid; diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index b8b911f..dbc8038 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -2262,14 +2262,14 @@ static inline void fid2obdo(struct lu_fid *fid, struct obdo *oa) /* something here */ } -typedef int (*register_osp_cb)(void *data); - -struct osp_register_item { - struct obd_export **ori_exp; - register_osp_cb ori_cb_func; - void *ori_cb_data; - cfs_list_t ori_list; - char ori_name[MTI_NAME_MAXLEN]; +typedef int (*register_lwp_cb)(void *data); + +struct lwp_register_item { + struct obd_export **lri_exp; + register_lwp_cb lri_cb_func; + void *lri_cb_data; + cfs_list_t lri_list; + char lri_name[MTI_NAME_MAXLEN]; }; /* I'm as embarrassed about this as you are. @@ -2280,10 +2280,10 @@ extern int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c); /* obd_mount.c */ int server_name2fsname(char *svname, char *fsname, char **endptr); -int lustre_register_osp_item(char *ospname, struct obd_export **exp, - register_osp_cb cb_func, void *cb_data); -void lustre_deregister_osp_item(struct obd_export **exp); -int tgt_name2ospname(char *ost_name, char *osp_name); +int lustre_register_lwp_item(char *lwpname, struct obd_export **exp, + register_lwp_cb cb_func, void *cb_data); +void lustre_deregister_lwp_item(struct obd_export **exp); +int tgt_name2lwpname(char *tgt_name, char *lwp_name); /* sysctl.c */ extern void obd_sysctl_init (void); diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index b15c10a..312ee13 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -287,10 +287,10 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) cli->cl_sp_to = LUSTRE_SP_OST; ns_type = LDLM_NS_TYPE_OSC; } else if (!strcmp(name, LUSTRE_MDC_NAME) || + !strcmp(name, LUSTRE_LWP_NAME) || (!strcmp(name, LUSTRE_OSP_NAME) && - (is_osp_for_connection(cli_name) || (is_osp_on_mdt(cli_name) && - strstr(lustre_cfg_buf(lcfg, 1), "OST") == NULL)))) { + strstr(lustre_cfg_buf(lcfg, 1), "OST") == NULL))) { /* MDC or OSP_on_MDT for other MDTs */ rq_portal = MDS_REQUEST_PORTAL; rp_portal = MDC_REPLY_PORTAL; diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index c6d7e5b..bb5d965 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -125,6 +125,9 @@ struct obd_type *class_get_type(const char *name) if (strcmp(modname, "obdfilter") == 0) modname = "ofd"; + if (strcmp(modname, LUSTRE_LWP_NAME) == 0) + modname = LUSTRE_OSP_NAME; + if (!strncmp(modname, LUSTRE_MDS_NAME, strlen(LUSTRE_MDS_NAME))) modname = LUSTRE_MDT_NAME; diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index a9c1ceb..d7d7f60 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -784,6 +784,7 @@ int class_add_conn(struct obd_device *obd, struct lustre_cfg *lcfg) if (strcmp(obd->obd_type->typ_name, LUSTRE_MDC_NAME) && strcmp(obd->obd_type->typ_name, LUSTRE_OSC_NAME) && strcmp(obd->obd_type->typ_name, LUSTRE_OSP_NAME) && + strcmp(obd->obd_type->typ_name, LUSTRE_LWP_NAME) && strcmp(obd->obd_type->typ_name, LUSTRE_MGC_NAME)) { CERROR("can't add connection on non-client dev\n"); RETURN(-EINVAL); diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 5be2a5e..ccf5a6e 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -903,15 +903,13 @@ static int inline is_mdc_for_mdt0(char *devname) } /** - * Convert OST/MDT name(fsname-OSTxxxx) to an osp name - * (fsname-MDT0000-osp-OSTxxxx), which will be used to - * communicate with MDT0 for this target. + * Convert OST/MDT name(fsname-OSTxxxx) to a lwp name + * (fsname-MDT0000-lwp-OSTxxxx) **/ -int tgt_name2ospname(char *svname, char *ospname) +int tgt_name2lwpname(char *svname, char *lwpname) { - char *fsname; - char *tgt; - int rc; + char *fsname, *tgt; + int rc; ENTRY; OBD_ALLOC(fsname, MTI_NAME_MAXLEN); @@ -920,12 +918,13 @@ int tgt_name2ospname(char *svname, char *ospname) rc = server_name2fsname(svname, fsname, &tgt); if (rc != 0) { - CERROR("%s change fsname error: rc %d\n", svname, rc); + CERROR("%s: failed to get fsname from svname. %d\n", + svname, rc); GOTO(cleanup, rc); } if (*tgt != '-' && *tgt != ':') { - CERROR("%s wrong svname name!\n", svname); + CERROR("%s: invalid svname name!\n", svname); GOTO(cleanup, rc = -EINVAL); } @@ -934,101 +933,101 @@ int tgt_name2ospname(char *svname, char *ospname) CERROR("%s is not an OST or MDT target!\n", svname); GOTO(cleanup, rc = -EINVAL); } - sprintf(ospname, "%s-MDT0000-%s-%s", fsname, LUSTRE_OSP_NAME, tgt); + sprintf(lwpname, "%s-MDT0000-%s-%s", fsname, LUSTRE_LWP_NAME, tgt); cleanup: if (fsname != NULL) OBD_FREE(fsname, MTI_NAME_MAXLEN); RETURN(rc); } -EXPORT_SYMBOL(tgt_name2ospname); +EXPORT_SYMBOL(tgt_name2lwpname); -static CFS_LIST_HEAD(osp_register_list); -DEFINE_MUTEX(osp_register_list_lock); +static CFS_LIST_HEAD(lwp_register_list); +DEFINE_MUTEX(lwp_register_list_lock); -int lustre_register_osp_item(char *ospname, struct obd_export **exp, - register_osp_cb cb_func, void *cb_data) +int lustre_register_lwp_item(char *lwpname, struct obd_export **exp, + register_lwp_cb cb_func, void *cb_data) { - struct obd_device *osp; - struct osp_register_item *ori; + struct obd_device *lwp; + struct lwp_register_item *lri; ENTRY; - LASSERTF(strlen(ospname) < MTI_NAME_MAXLEN, "ospname is too long %s\n", - ospname); + LASSERTF(strlen(lwpname) < MTI_NAME_MAXLEN, "lwpname is too long %s\n", + lwpname); LASSERT(exp != NULL && *exp == NULL); - OBD_ALLOC_PTR(ori); - if (ori == NULL) + OBD_ALLOC_PTR(lri); + if (lri == NULL) RETURN(-ENOMEM); - mutex_lock(&osp_register_list_lock); + mutex_lock(&lwp_register_list_lock); - osp = class_name2obd(ospname); - if (osp != NULL && osp->obd_set_up == 1) { + lwp = class_name2obd(lwpname); + if (lwp != NULL && lwp->obd_set_up == 1) { struct obd_uuid *uuid; OBD_ALLOC_PTR(uuid); if (uuid == NULL) { - mutex_unlock(&osp_register_list_lock); - OBD_FREE_PTR(ori); + mutex_unlock(&lwp_register_list_lock); + OBD_FREE_PTR(lri); RETURN(-ENOMEM); } - memcpy(uuid->uuid, ospname, strlen(ospname)); - *exp = cfs_hash_lookup(osp->obd_uuid_hash, uuid); + memcpy(uuid->uuid, lwpname, strlen(lwpname)); + *exp = cfs_hash_lookup(lwp->obd_uuid_hash, uuid); OBD_FREE_PTR(uuid); } - memcpy(ori->ori_name, ospname, strlen(ospname)); - ori->ori_exp = exp; - ori->ori_cb_func = cb_func; - ori->ori_cb_data = cb_data; - CFS_INIT_LIST_HEAD(&ori->ori_list); - cfs_list_add(&ori->ori_list, &osp_register_list); + memcpy(lri->lri_name, lwpname, strlen(lwpname)); + lri->lri_exp = exp; + lri->lri_cb_func = cb_func; + lri->lri_cb_data = cb_data; + CFS_INIT_LIST_HEAD(&lri->lri_list); + cfs_list_add(&lri->lri_list, &lwp_register_list); if (*exp != NULL && cb_func != NULL) cb_func(cb_data); - mutex_unlock(&osp_register_list_lock); + mutex_unlock(&lwp_register_list_lock); RETURN(0); } -EXPORT_SYMBOL(lustre_register_osp_item); +EXPORT_SYMBOL(lustre_register_lwp_item); -void lustre_deregister_osp_item(struct obd_export **exp) +void lustre_deregister_lwp_item(struct obd_export **exp) { - struct osp_register_item *ori, *tmp; + struct lwp_register_item *lri, *tmp; - mutex_lock(&osp_register_list_lock); - cfs_list_for_each_entry_safe(ori, tmp, &osp_register_list, ori_list) { - if (exp == ori->ori_exp) { + mutex_lock(&lwp_register_list_lock); + cfs_list_for_each_entry_safe(lri, tmp, &lwp_register_list, lri_list) { + if (exp == lri->lri_exp) { if (*exp) class_export_put(*exp); - cfs_list_del(&ori->ori_list); - OBD_FREE_PTR(ori); + cfs_list_del(&lri->lri_list); + OBD_FREE_PTR(lri); break; } } - mutex_unlock(&osp_register_list_lock); + mutex_unlock(&lwp_register_list_lock); } -EXPORT_SYMBOL(lustre_deregister_osp_item); +EXPORT_SYMBOL(lustre_deregister_lwp_item); -static void lustre_notify_osp_list(struct obd_export *exp) +static void lustre_notify_lwp_list(struct obd_export *exp) { - struct osp_register_item *ori, *tmp; + struct lwp_register_item *lri, *tmp; LASSERT(exp != NULL); - mutex_lock(&osp_register_list_lock); - cfs_list_for_each_entry_safe(ori, tmp, &osp_register_list, ori_list) { - if (strcmp(exp->exp_obd->obd_name, ori->ori_name)) + mutex_lock(&lwp_register_list_lock); + cfs_list_for_each_entry_safe(lri, tmp, &lwp_register_list, lri_list) { + if (strcmp(exp->exp_obd->obd_name, lri->lri_name)) continue; - if (*ori->ori_exp != NULL) + if (*lri->lri_exp != NULL) continue; - *ori->ori_exp = class_export_get(exp); - if (ori->ori_cb_func != NULL) - ori->ori_cb_func(ori->ori_cb_data); + *lri->lri_exp = class_export_get(exp); + if (lri->lri_cb_func != NULL) + lri->lri_cb_func(lri->lri_cb_data); } - mutex_unlock(&osp_register_list_lock); + mutex_unlock(&lwp_register_list_lock); } -static int lustre_osp_connect(struct obd_device *osp) +static int lustre_lwp_connect(struct obd_device *lwp) { struct lu_env env; struct lu_context session_ctx; @@ -1039,7 +1038,7 @@ static int lustre_osp_connect(struct obd_device *osp) ENTRY; /* log has been fully processed, let clients connect */ - rc = lu_env_init(&env, osp->obd_lu_dev->ld_type->ldt_ctx_tags); + rc = lu_env_init(&env, lwp->obd_lu_dev->ld_type->ldt_ctx_tags); if (rc != 0) RETURN(rc); @@ -1062,19 +1061,19 @@ static int lustre_osp_connect(struct obd_device *osp) if (uuid == NULL) GOTO(out, rc = -ENOMEM); - if (strlen(osp->obd_name) > sizeof(uuid->uuid)) { - CERROR("%s: Too long osp name %s, max_size is %d\n", - osp->obd_name, osp->obd_name, (int)sizeof(uuid->uuid)); + if (strlen(lwp->obd_name) > sizeof(uuid->uuid)) { + CERROR("%s: Too long lwp name %s, max_size is %d\n", + lwp->obd_name, lwp->obd_name, (int)sizeof(uuid->uuid)); GOTO(out, rc = -EINVAL); } - /* Use osp name as the uuid, so we find the export by - * osp name later */ - memcpy(uuid->uuid, osp->obd_name, strlen(osp->obd_name)); - rc = obd_connect(&env, &exp, osp, uuid, data, NULL); + + /* Use lwp name as the uuid, so we find the export by lwp name later */ + memcpy(uuid->uuid, lwp->obd_name, strlen(lwp->obd_name)); + rc = obd_connect(&env, &exp, lwp, uuid, data, NULL); if (rc != 0) - CERROR("%s: connect failed: rc = %d\n", osp->obd_name, rc); + CERROR("%s: connect failed: rc = %d\n", lwp->obd_name, rc); else - lustre_notify_osp_list(exp); + lustre_notify_lwp_list(exp); out: if (data != NULL) @@ -1090,19 +1089,15 @@ out: } /** - * osp-on-ost is used by slaves (Non-MDT0 targets) to manage the connection + * lwp is used by slaves (Non-MDT0 targets) to manage the connection * to MDT0. - * - * The OSTs will communicate with MDT0 by the connection established by the - * osp-on-ost to get quota and fid sequence. - * **/ -static int lustre_osp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) +static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) { struct obd_connect_data *data = NULL; struct obd_device *obd; - char *ospname = NULL; - char *ospuuid = NULL; + char *lwpname = NULL; + char *lwpuuid = NULL; int rc; ENTRY; @@ -1113,101 +1108,143 @@ static int lustre_osp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) GOTO(out, rc); } - OBD_ALLOC(ospname, MTI_NAME_MAXLEN); - if (ospname == NULL) + OBD_ALLOC(lwpname, MTI_NAME_MAXLEN); + if (lwpname == NULL) GOTO(out, rc = -ENOMEM); - rc = tgt_name2ospname(lsi->lsi_svname, ospname); + rc = tgt_name2lwpname(lsi->lsi_svname, lwpname); if (rc != 0) { - CERROR("%s change ospname error: rc %d\n", + CERROR("%s: failed to generate lwp name. %d\n", lsi->lsi_svname, rc); GOTO(out, rc); } - OBD_ALLOC(ospuuid, MTI_NAME_MAXLEN); - if (ospuuid == NULL) + OBD_ALLOC(lwpuuid, MTI_NAME_MAXLEN); + if (lwpuuid == NULL) GOTO(out, rc = -ENOMEM); - sprintf(ospuuid, "%s_UUID", ospname); - rc = lustre_start_simple(ospname, LUSTRE_OSP_NAME, - ospuuid, lustre_cfg_string(lcfg, 1), + sprintf(lwpuuid, "%s_UUID", lwpname); + rc = lustre_start_simple(lwpname, LUSTRE_LWP_NAME, + lwpuuid, lustre_cfg_string(lcfg, 1), 0, 0, 0); if (rc) { - CERROR("%s: setup up failed: rc %d\n", ospname, rc); + CERROR("%s: setup up failed: rc %d\n", lwpname, rc); GOTO(out, rc); } - obd = class_name2obd(ospname); + obd = class_name2obd(lwpname); LASSERT(obd != NULL); - rc = lustre_osp_connect(obd); + rc = lustre_lwp_connect(obd); if (rc != 0) - CERROR("%s: connect failed: rc = %d\n", ospname, rc); + CERROR("%s: connect failed: rc = %d\n", lwpname, rc); out: if (data != NULL) OBD_FREE_PTR(data); - if (ospname != NULL) - OBD_FREE(ospname, MTI_NAME_MAXLEN); - if (ospuuid != NULL) - OBD_FREE(ospuuid, MTI_NAME_MAXLEN); + if (lwpname != NULL) + OBD_FREE(lwpname, MTI_NAME_MAXLEN); + if (lwpuuid != NULL) + OBD_FREE(lwpuuid, MTI_NAME_MAXLEN); RETURN(rc); } -static int lustre_osp_add_conn(struct lustre_cfg *cfg, +/* the caller is responsible for memory free */ +static struct obd_device *lustre_find_lwp(struct lustre_sb_info *lsi, + char **lwpname, char **logname) +{ + struct obd_device *lwp; + int rc = 0; + ENTRY; + + LASSERT(lwpname != NULL); + LASSERT(IS_OST(lsi) || IS_MDT(lsi)); + + OBD_ALLOC(*lwpname, MTI_NAME_MAXLEN); + if (*lwpname == NULL) + RETURN(ERR_PTR(-ENOMEM)); + + if (logname != NULL) { + OBD_ALLOC(*logname, MTI_NAME_MAXLEN); + if (*logname == NULL) + GOTO(out, rc = -ENOMEM); + rc = server_name2fsname(lsi->lsi_svname, *lwpname, NULL); + if (rc != 0) { + CERROR("%s: failed to get fsname from svname. %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc = -EINVAL); + } + sprintf(*logname, "%s-client", *lwpname); + } + + rc = tgt_name2lwpname(lsi->lsi_svname, *lwpname); + if (rc != 0) { + CERROR("%s: failed to generate lwp name. %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc = -EINVAL); + } + + lwp = class_name2obd(*lwpname); + +out: + if (rc != 0) { + if (*lwpname != NULL) { + OBD_FREE(*lwpname, MTI_NAME_MAXLEN); + *lwpname = NULL; + } + if (logname != NULL && *logname != NULL) { + OBD_FREE(*logname, MTI_NAME_MAXLEN); + *logname = NULL; + } + lwp = ERR_PTR(rc); + } + + RETURN(lwp != NULL ? lwp : ERR_PTR(-ENOENT)); +} + +static int lustre_lwp_add_conn(struct lustre_cfg *cfg, struct lustre_sb_info *lsi) { struct lustre_cfg_bufs *bufs = NULL; struct lustre_cfg *lcfg = NULL; - char *ospname = NULL; - struct obd_device *osp; + char *lwpname = NULL; + struct obd_device *lwp; int rc; ENTRY; - OBD_ALLOC(ospname, MTI_NAME_MAXLEN); - if (ospname == NULL) - GOTO(out, rc = -ENOMEM); - - rc = tgt_name2ospname(lsi->lsi_svname, ospname); - if (rc != 0) { - CERROR("%s change ospname error: rc %d\n", - lsi->lsi_svname, rc); - GOTO(out, rc); + lwp = lustre_find_lwp(lsi, &lwpname, NULL); + if (IS_ERR(lwp)) { + CERROR("%s: can't find lwp device.\n", lsi->lsi_svname); + GOTO(out, rc = PTR_ERR(lwp)); } + LASSERT(lwpname != NULL); OBD_ALLOC_PTR(bufs); if (bufs == NULL) GOTO(out, rc = -ENOMEM); - lustre_cfg_bufs_reset(bufs, ospname); + lustre_cfg_bufs_reset(bufs, lwpname); lustre_cfg_bufs_set_string(bufs, 1, lustre_cfg_string(cfg, 1)); lcfg = lustre_cfg_new(LCFG_ADD_CONN, bufs); - osp = class_name2obd(ospname); - if (osp == NULL) { - CERROR("Can not find %s\n", ospname); - GOTO(out, rc = -EINVAL); - } - - rc = class_add_conn(osp, lcfg); + rc = class_add_conn(lwp, lcfg); if (rc) - CERROR("%s: can't add conn: rc = %d\n", ospname, rc); + CERROR("%s: can't add conn: rc = %d\n", lwpname, rc); out: if (bufs != NULL) OBD_FREE_PTR(bufs); if (lcfg != NULL) lustre_cfg_free(lcfg); - if (ospname != NULL) - OBD_FREE(ospname, MTI_NAME_MAXLEN); - + if (lwpname != NULL) + OBD_FREE(lwpname, MTI_NAME_MAXLEN); RETURN(rc); } /** - * Retrieve MDT nids from the client log, then start the osp-on-ost device. + * Retrieve MDT nids from the client log, then start the lwp device. * there are only two scenarios which would include mdt nid. * 1. * marker 5 (flags=0x01, v2.1.54.0) lustre-MDT0000 'add mdc' xxx- @@ -1224,7 +1261,7 @@ out: * add_conn 0:lustre-MDT0000-mdc 1:192.168.122.2@tcp * marker 7 (flags=0x02, v2.1.54.0) lustre-MDT0000 'add failnid' xxxx- **/ -static int client_osp_config_process(const struct lu_env *env, +static int client_lwp_config_process(const struct lu_env *env, struct llog_handle *handle, struct llog_rec_hdr *rec, void *data) { @@ -1286,9 +1323,9 @@ static int client_osp_config_process(const struct lu_env *env, } case LCFG_ADD_UUID: { if (clli->cfg_flags == CFG_F_MARKER) { - rc = lustre_osp_setup(lcfg, lsi); + rc = lustre_lwp_setup(lcfg, lsi); /* XXX: process only the first nid as - * we don't need another instance of osp */ + * we don't need another instance of lwp */ clli->cfg_flags |= CFG_F_SKIP; } else if (clli->cfg_flags == (CFG_F_MARKER | CFG_F_SKIP)) { rc = class_add_uuid(lustre_cfg_string(lcfg, 1), @@ -1301,7 +1338,7 @@ static int client_osp_config_process(const struct lu_env *env, } case LCFG_ADD_CONN: { if (is_mdc_for_mdt0(lustre_cfg_string(lcfg, 0))) - rc = lustre_osp_add_conn(lcfg, lsi); + rc = lustre_lwp_add_conn(lcfg, lsi); break; } default: @@ -1311,11 +1348,11 @@ out: RETURN(rc); } -static int lustre_disconnect_osp(struct super_block *sb) +static int lustre_disconnect_lwp(struct super_block *sb) { struct lustre_sb_info *lsi = s2lsi(sb); - struct obd_device *osp; - char *ospname = NULL; + struct obd_device *lwp; + char *lwpname = NULL; char *logname = NULL; struct lustre_cfg *lcfg = NULL; struct lustre_cfg_bufs *bufs = NULL; @@ -1323,34 +1360,12 @@ static int lustre_disconnect_osp(struct super_block *sb) int rc; ENTRY; - LASSERT(IS_OST(lsi) || IS_MDT(lsi)); - if (IS_MDT(lsi)) { - int index; - - /* Only disconnect MDT0-osp-MDT0 here, other osp on MDT - * will be disconnect during MDT stack cleanup. - * FIXME: remove later when quota on DNE is finished */ - rc = server_name2index(lsi->lsi_svname, &index, NULL); - if (rc < 0) - RETURN(rc); - if (index != 0) - RETURN(0); - } - OBD_ALLOC(logname, MTI_NAME_MAXLEN); - if (logname == NULL) - RETURN(-ENOMEM); - - OBD_ALLOC(ospname, MTI_NAME_MAXLEN); - if (ospname == NULL) - GOTO(out, rc = -ENOMEM); + lwp = lustre_find_lwp(lsi, &lwpname, &logname); + if (IS_ERR(lwp) && PTR_ERR(lwp) != -ENOENT) + GOTO(out, rc = PTR_ERR(lwp)); - rc = server_name2fsname(lsi->lsi_svname, ospname, NULL); - if (rc != 0) { - CERROR("%s: get fsname error: %d\n", - lsi->lsi_svname, rc); - GOTO(out, rc); - } - sprintf(logname, "%s-client", ospname); + LASSERT(lwpname != NULL); + LASSERT(logname != NULL); OBD_ALLOC_PTR(cfg); if (cfg == NULL) @@ -1360,36 +1375,29 @@ static int lustre_disconnect_osp(struct super_block *sb) cfg->cfg_instance = sb; rc = lustre_end_log(sb, logname, cfg); if (rc != 0) { - CERROR("Can't end config log %s\n", ospname); + CERROR("%s: Can't end config log %s.\n", lwpname, logname); GOTO(out, rc); } - rc = tgt_name2ospname(lsi->lsi_svname, ospname); - if (rc != 0) { - CERROR("%s: get osp name error: %d\n", - lsi->lsi_svname, rc); - GOTO(out, rc); + if (PTR_ERR(lwp) == -ENOENT) { + CDEBUG(D_CONFIG, "%s: lwp device wasn't started.\n", + lsi->lsi_svname); + GOTO(out, rc = 0); } - osp = class_name2obd(ospname); - if (osp == NULL) { - CERROR("Can't find osp-on-ost %s\n", ospname); - GOTO(out, rc = -ENOENT); - } - OBD_ALLOC_PTR(bufs); if (bufs == NULL) GOTO(out, rc = -ENOMEM); - lustre_cfg_bufs_reset(bufs, osp->obd_name); + lustre_cfg_bufs_reset(bufs, lwp->obd_name); lustre_cfg_bufs_set_string(bufs, 1, NULL); lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs); if (!lcfg) GOTO(out, rc = -ENOMEM); /* Disconnect import first. NULL is passed for the '@env', since - * it will not be used for the 'osp-on-ost'. (see osp_shutdown()) */ - rc = osp->obd_lu_dev->ld_ops->ldo_process_config(NULL, osp->obd_lu_dev, + * it will not be used. */ + rc = lwp->obd_lu_dev->ld_ops->ldo_process_config(NULL, lwp->obd_lu_dev, lcfg); out: if (lcfg) @@ -1398,58 +1406,81 @@ out: OBD_FREE_PTR(bufs); if (cfg) OBD_FREE_PTR(cfg); - if (ospname) - OBD_FREE(ospname, MTI_NAME_MAXLEN); + if (lwpname) + OBD_FREE(lwpname, MTI_NAME_MAXLEN); if (logname) OBD_FREE(logname, MTI_NAME_MAXLEN); RETURN(rc); } /** - * Stop the osp(fsname-MDT0000-osp-OSTxxxx) or (fsname-MDT0000-osp-MDT0000) for an OST target. + * Stop the lwp for an OST/MDT target. **/ -static int lustre_stop_osp(struct super_block *sb) +static int lustre_stop_lwp(struct super_block *sb) { struct lustre_sb_info *lsi = s2lsi(sb); - struct obd_device *osp = NULL; - char *ospname = NULL; + struct obd_device *lwp = NULL; + char *lwpname = NULL; int rc = 0; ENTRY; - LASSERT(IS_OST(lsi) || IS_MDT(lsi)); - if (IS_MDT(lsi)) { - int index; - - /* Only disconnect MDT0-osp-MDT0 here, other osp on MDT - * will be disconnect during MDT stack cleanup. - * FIXME: remove later when quota on DNE is finished */ - rc = server_name2index(lsi->lsi_svname, &index, NULL); - if (rc < 0) - RETURN(rc); - if (index != 0) - RETURN(0); + lwp = lustre_find_lwp(lsi, &lwpname, NULL); + if (IS_ERR(lwp)) { + CDEBUG(PTR_ERR(lwp) == -ENOENT ? D_CONFIG : D_ERROR, + "%s: lwp wasn't started.\n", lsi->lsi_svname); + GOTO(out, rc = 0); } - OBD_ALLOC(ospname, MTI_NAME_MAXLEN); - rc = tgt_name2ospname(lsi->lsi_svname, ospname); - if (rc != 0) { - CERROR("%s get fsname error: rc %d\n", - lsi->lsi_svname, rc); - GOTO(cleanup, rc); - } + lwp->obd_force = 1; + rc = class_manual_cleanup(lwp); - osp = class_name2obd(ospname); - if (osp == NULL) { - CERROR("Can not find osp-on-ost %s\n", ospname); - GOTO(cleanup, rc = -ENOENT); - } +out: + if (lwpname != NULL) + OBD_FREE(lwpname, MTI_NAME_MAXLEN); + RETURN(rc); +} - osp->obd_force = 1; - rc = class_manual_cleanup(osp); +/** + * Start the lwp(fsname-MDT0000-lwp-OSTxxxx) for an OST or MDT target, + * which would be used to establish connection from OST to MDT0. + **/ +static int lustre_start_lwp(struct super_block *sb) +{ + struct lustre_sb_info *lsi = s2lsi(sb); + struct config_llog_instance *cfg = NULL; + struct obd_device *lwp; + char *lwpname = NULL; + char *logname = NULL; + int rc; + ENTRY; -cleanup: - if (ospname != NULL) - OBD_FREE(ospname, MTI_NAME_MAXLEN); + lwp = lustre_find_lwp(lsi, &lwpname, &logname); + + /* the lwp device already stared */ + if (lwp && !IS_ERR(lwp)) + GOTO(out, rc = 0); + + if (PTR_ERR(lwp) != -ENOENT) + GOTO(out, rc = PTR_ERR(lwp)); + + LASSERT(lwpname != NULL); + LASSERT(logname != NULL); + + OBD_ALLOC_PTR(cfg); + if (cfg == NULL) + GOTO(out, rc = -ENOMEM); + + cfg->cfg_callback = client_lwp_config_process; + cfg->cfg_instance = sb; + + rc = lustre_process_log(sb, logname, cfg); +out: + if (lwpname != NULL) + OBD_FREE(lwpname, MTI_NAME_MAXLEN); + if (logname != NULL) + OBD_FREE(logname, MTI_NAME_MAXLEN); + if (cfg != NULL) + OBD_FREE_PTR(cfg); RETURN(rc); } @@ -1732,66 +1763,6 @@ out: } -/** - * Start the osp(fsname-MDT0000-osp-OSTxxxx) for an OST target, - * which would be used to communicate with MDT0 for quota and FID. - **/ -static int lustre_start_osp(struct super_block *sb) -{ - struct lustre_sb_info *lsi = s2lsi(sb); - struct config_llog_instance *cfg = NULL; - struct obd_device *osp; - char *ospname = NULL; - char *logname = NULL; - char *tgt; - int rc; - ENTRY; - - LASSERT(IS_OST(lsi) || IS_MDT(lsi)); - OBD_ALLOC(ospname, MTI_NAME_MAXLEN); - OBD_ALLOC(logname, MTI_NAME_MAXLEN); - if (ospname == NULL || logname == NULL) - GOTO(cleanup, rc = -ENOMEM); - - rc = server_name2fsname(lsi->lsi_svname, ospname, &tgt); - if (rc != 0) { - CERROR("%s change fsname error: rc %d\n", - lsi->lsi_svname, rc); - GOTO(cleanup, rc); - } - sprintf(logname, "%s-client", ospname); - - rc = tgt_name2ospname(lsi->lsi_svname, ospname); - if (rc != 0) { - CERROR("%s change ospname error: rc %d\n", - lsi->lsi_svname, rc); - GOTO(cleanup, rc); - } - - osp = class_name2obd(ospname); - if (osp != NULL) - GOTO(cleanup, rc = 0); - - OBD_ALLOC_PTR(cfg); - if (cfg == NULL) - GOTO(cleanup, rc = -ENOMEM); - - cfg->cfg_callback = client_osp_config_process; - cfg->cfg_instance = sb; - - rc = lustre_process_log(sb, logname, cfg); - -cleanup: - if (ospname != NULL) - OBD_FREE(ospname, MTI_NAME_MAXLEN); - if (logname != NULL) - OBD_FREE(logname, MTI_NAME_MAXLEN); - if (cfg != NULL) - OBD_FREE_PTR(cfg); - - RETURN(rc); -} - /** Start server targets: MDTs and OSTs */ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt) @@ -1882,9 +1853,9 @@ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt) } if (IS_OST(lsi) || IS_MDT(lsi)) { - rc = lustre_start_osp(sb); + rc = lustre_start_lwp(sb); if (rc) { - CERROR("%s: failed to start OSP: %d\n", + CERROR("%s: failed to start LWP: %d\n", lsi->lsi_svname, rc); GOTO(out_mgc, rc); } @@ -2127,13 +2098,13 @@ static void server_put_super(struct super_block *sb) if (IS_MDT(lsi) && (lsi->lsi_lmd->lmd_flags & LMD_FLG_NOSVC)) snprintf(tmpname, tmpname_sz, "MGS"); - /* disconnect the osp-on-ost first to drain off the inflight request */ + /* disconnect the lwp first to drain off the inflight request */ if (IS_OST(lsi) || IS_MDT(lsi)) { int rc; - rc = lustre_disconnect_osp(sb); + rc = lustre_disconnect_lwp(sb); if (rc && rc != ETIMEDOUT) - CERROR("%s: failed to disconnect osp-on-ost (rc=%d)!\n", + CERROR("%s: failed to disconnect lwp. (rc=%d)\n", tmpname, rc); } @@ -2178,8 +2149,8 @@ static void server_put_super(struct super_block *sb) } if (IS_OST(lsi) || IS_MDT(lsi)) { - if (lustre_stop_osp(sb) < 0) - CERROR("%s: Fail to stop osp-on-ost!\n", tmpname); + if (lustre_stop_lwp(sb) < 0) + CERROR("%s: failed to stop lwp!\n", tmpname); } /* Clean the mgc and sb */ diff --git a/lustre/ofd/ofd_fs.c b/lustre/ofd/ofd_fs.c index 7690ebe..b1c7c98 100644 --- a/lustre/ofd/ofd_fs.c +++ b/lustre/ofd/ofd_fs.c @@ -194,12 +194,12 @@ static void ofd_deregister_seq_exp(struct ofd_device *ofd) struct seq_server_site *ss = &ofd->ofd_seq_site; if (ss->ss_client_seq != NULL) { - lustre_deregister_osp_item(&ss->ss_client_seq->lcs_exp); + lustre_deregister_lwp_item(&ss->ss_client_seq->lcs_exp); ss->ss_client_seq->lcs_exp = NULL; } if (ss->ss_server_fld != NULL) { - lustre_deregister_osp_item(&ss->ss_server_fld->lsf_control_exp); + lustre_deregister_lwp_item(&ss->ss_server_fld->lsf_control_exp); ss->ss_server_fld->lsf_control_exp = NULL; } } @@ -353,33 +353,33 @@ static int ofd_fld_init(const struct lu_env *env, const char *uuid, static int ofd_register_seq_exp(struct ofd_device *ofd) { struct seq_server_site *ss = &ofd->ofd_seq_site; - char *osp_name = NULL; + char *lwp_name = NULL; int rc; - OBD_ALLOC(osp_name, MAX_OBD_NAME); - if (osp_name == NULL) + OBD_ALLOC(lwp_name, MAX_OBD_NAME); + if (lwp_name == NULL) GOTO(out_free, rc = -ENOMEM); - rc = tgt_name2ospname(ofd_name(ofd), osp_name); + rc = tgt_name2lwpname(ofd_name(ofd), lwp_name); if (rc != 0) GOTO(out_free, rc); - rc = lustre_register_osp_item(osp_name, &ss->ss_client_seq->lcs_exp, + rc = lustre_register_lwp_item(lwp_name, &ss->ss_client_seq->lcs_exp, NULL, NULL); if (rc != 0) GOTO(out_free, rc); - rc = lustre_register_osp_item(osp_name, + rc = lustre_register_lwp_item(lwp_name, &ss->ss_server_fld->lsf_control_exp, NULL, NULL); if (rc != 0) { - lustre_deregister_osp_item(&ss->ss_client_seq->lcs_exp); + lustre_deregister_lwp_item(&ss->ss_client_seq->lcs_exp); ss->ss_client_seq->lcs_exp = NULL; GOTO(out_free, rc); } out_free: - if (osp_name != NULL) - OBD_FREE(osp_name, MAX_OBD_NAME); + if (lwp_name != NULL) + OBD_FREE(lwp_name, MAX_OBD_NAME); return rc; } diff --git a/lustre/osp/Makefile.in b/lustre/osp/Makefile.in index e504d0f..4176d21 100644 --- a/lustre/osp/Makefile.in +++ b/lustre/osp/Makefile.in @@ -1,6 +1,6 @@ -MODULES := osp -osp-objs := osp_dev.o osp_object.o osp_precreate.o osp_sync.o lproc_osp.o -osp-objs += osp_ost.o +MODULES = osp +osp-objs = osp_dev.o osp_object.o osp_precreate.o osp_sync.o lproc_osp.o +osp-objs += lwp_dev.o EXTRA_DIST = $(osp-objs:.o=.c) osp_internal.h diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c new file mode 100644 index 0000000..ed5dba9 --- /dev/null +++ b/lustre/osp/lwp_dev.c @@ -0,0 +1,450 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA + * + * GPL HEADER END + */ +/* + * Copyright (c) 2012 Intel Corporation + * Use is subject to license terms. + * + * lustre/osp/lwp_dev.c + * + * Light Weight Proxy, which is just for managing the connection established + * from OSTs/MDTs to MDT0. + * + * Author: + * Author: + */ +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif +#define DEBUG_SUBSYSTEM S_OST + +#include +#include +#include + +struct lwp_device { + struct lu_device lpd_dev; + struct obd_device *lpd_obd; + struct obd_uuid lpd_cluuid; + struct obd_export *lpd_exp; + int lpd_connects; +}; + +static inline struct lwp_device *lu2lwp_dev(struct lu_device *d) +{ + return container_of0(d, struct lwp_device, lpd_dev); +} + +static inline struct lu_device *lwp2lu_dev(struct lwp_device *d) +{ + return &d->lpd_dev; +} + +static int lwp_name2fsname(char *lwpname, char *fsname) +{ + char *ptr; + + LASSERT(lwpname != NULL); + LASSERT(fsname != NULL); + + sprintf(fsname, "-%s-", LUSTRE_LWP_NAME); + + ptr = strstr(lwpname, fsname); + if (ptr == NULL) + return -EINVAL; + + while (*(--ptr) != '-') { + if (ptr == lwpname) + return -EINVAL; + } + + strncpy(fsname, lwpname, ptr - lwpname); + fsname[ptr - lwpname] = '\0'; + + return 0; +} + +static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, + char *nidstring) +{ + struct lustre_cfg_bufs *bufs = NULL; + struct lustre_cfg *lcfg = NULL; + char *lwpname = lwp->lpd_obd->obd_name; + char *fsname = NULL; + char *server_uuid = NULL; + class_uuid_t uuid; + struct obd_import *imp; + int rc; + ENTRY; + + OBD_ALLOC_PTR(bufs); + if (bufs == NULL) + RETURN(-ENOMEM); + + OBD_ALLOC(fsname, strlen(lwpname)); + if (fsname == NULL) + GOTO(out, rc = -ENOMEM); + + rc = lwp_name2fsname(lwpname, fsname); + if (rc) { + CERROR("%s: failed to get fsname from lwpname. %d\n", + lwpname, rc); + GOTO(out, rc); + } + + OBD_ALLOC(server_uuid, strlen(fsname) + 15); + if (server_uuid == NULL) + GOTO(out, rc = -ENOMEM); + + sprintf(server_uuid, "%s-MDT0000_UUID", fsname); + lustre_cfg_bufs_reset(bufs, lwpname); + lustre_cfg_bufs_set_string(bufs, 1, server_uuid); + lustre_cfg_bufs_set_string(bufs, 2, nidstring); + lcfg = lustre_cfg_new(LCFG_SETUP, bufs); + if (lcfg == NULL) + GOTO(out, rc = -ENOMEM); + + rc = client_obd_setup(lwp->lpd_obd, lcfg); + if (rc != 0) { + CERROR("%s: client obd setup error: rc = %d\n", + lwp->lpd_obd->obd_name, rc); + GOTO(out, rc); + } + + imp = lwp->lpd_obd->u.cli.cl_import; + rc = ptlrpc_init_import(imp); + if (rc) + GOTO(out, rc); + + ll_generate_random_uuid(uuid); + class_uuid_unparse(uuid, &lwp->lpd_cluuid); +out: + if (bufs != NULL) + OBD_FREE_PTR(bufs); + if (server_uuid != NULL) + OBD_FREE(server_uuid, strlen(fsname) + 15); + if (fsname != NULL) + OBD_FREE(fsname, strlen(lwpname)); + if (lcfg != NULL) + lustre_cfg_free(lcfg); + if (rc) + client_obd_cleanup(lwp->lpd_obd); + RETURN(rc); +} + +int lwp_disconnect(struct lwp_device *d) +{ + struct obd_import *imp; + int rc = 0; + + imp = d->lpd_obd->u.cli.cl_import; + + /* Mark import deactivated now, so we don't try to reconnect if any + * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't + * fully deactivate the import, or that would drop all requests. */ + LASSERT(imp != NULL); + spin_lock(&imp->imp_lock); + imp->imp_deactive = 1; + spin_unlock(&imp->imp_lock); + + ptlrpc_deactivate_import(imp); + + /* Some non-replayable imports (MDS's OSCs) are pinged, so just + * delete it regardless. (It's safe to delete an import that was + * never added.) */ + (void)ptlrpc_pinger_del_import(imp); + + rc = ptlrpc_disconnect_import(imp, 0); + if (rc && rc != -ETIMEDOUT) + CERROR("%s: can't disconnect: rc = %d\n", + d->lpd_obd->obd_name, rc); + + ptlrpc_invalidate_import(imp); + + RETURN(rc); +} + +static int lwp_process_config(const struct lu_env *env, + struct lu_device *dev, struct lustre_cfg *lcfg) +{ + struct lwp_device *d = lu2lwp_dev(dev); + int rc; + ENTRY; + + switch (lcfg->lcfg_command) { + case LCFG_CLEANUP: + rc = lwp_disconnect(d); + break; + case LCFG_PARAM: + rc = -ENOSYS; + break; + default: + CERROR("%s: unknown command %u\n", + (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command); + rc = 0; + break; + } + + RETURN(rc); +} + +const struct lu_device_operations lwp_lu_ops = { + .ldo_process_config = lwp_process_config, +}; + +static struct lprocfs_vars lprocfs_lwp_module_vars[] = { + { "num_refs", lprocfs_rd_numrefs, 0, 0 }, + { 0 } +}; + +static struct lprocfs_vars lprocfs_lwp_obd_vars[] = { + { 0 } +}; + +void lprocfs_lwp_init_vars(struct lprocfs_static_vars *lvars) +{ + lvars->module_vars = lprocfs_lwp_module_vars; + lvars->obd_vars = lprocfs_lwp_obd_vars; +} + +int lwp_init0(const struct lu_env *env, struct lwp_device *lwp, + struct lu_device_type *ldt, struct lustre_cfg *cfg) +{ + struct lprocfs_static_vars lvars = { 0 }; + int rc; + ENTRY; + + lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0)); + if (lwp->lpd_obd == NULL) { + CERROR("Cannot find obd with name %s\n", + lustre_cfg_string(cfg, 0)); + RETURN(-ENODEV); + } + + lwp->lpd_dev.ld_ops = &lwp_lu_ops; + lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev; + + rc = ptlrpcd_addref(); + if (rc) { + CERROR("%s: ptlrpcd addref error: rc =%d\n", + lwp->lpd_obd->obd_name, rc); + RETURN(rc); + } + + rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1)); + if (rc) { + CERROR("%s: setup lwp failed. %d\n", + lwp->lpd_obd->obd_name, rc); + ptlrpcd_decref(); + RETURN(rc); + } + + lprocfs_lwp_init_vars(&lvars); + if (lprocfs_obd_setup(lwp->lpd_obd, lvars.obd_vars) == 0) + ptlrpc_lprocfs_register_obd(lwp->lpd_obd); + + RETURN(0); +} + +static struct lu_device *lwp_device_free(const struct lu_env *env, + struct lu_device *lu) +{ + struct lwp_device *m = lu2lwp_dev(lu); + ENTRY; + + if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) { + LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); + lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer); + } + lu_device_fini(&m->lpd_dev); + OBD_FREE_PTR(m); + RETURN(NULL); +} + +static struct lu_device *lwp_device_alloc(const struct lu_env *env, + struct lu_device_type *t, + struct lustre_cfg *lcfg) +{ + struct lwp_device *lwp; + struct lu_device *l; + + OBD_ALLOC_PTR(lwp); + if (lwp == NULL) { + l = ERR_PTR(-ENOMEM); + } else { + int rc; + + l = lwp2lu_dev(lwp); + lu_device_init(&lwp->lpd_dev, t); + rc = lwp_init0(env, lwp, t, lcfg); + if (rc != 0) { + lwp_device_free(env, l); + l = ERR_PTR(rc); + } + } + return l; +} + + +static struct lu_device *lwp_device_fini(const struct lu_env *env, + struct lu_device *d) +{ + struct lwp_device *m = lu2lwp_dev(d); + struct obd_import *imp; + int rc; + ENTRY; + + if (m->lpd_exp != NULL) + class_disconnect(m->lpd_exp); + + imp = m->lpd_obd->u.cli.cl_import; + + if (imp->imp_rq_pool) { + ptlrpc_free_rq_pool(imp->imp_rq_pool); + imp->imp_rq_pool = NULL; + } + + obd_cleanup_client_import(m->lpd_obd); + + LASSERT(m->lpd_obd); + ptlrpc_lprocfs_unregister_obd(m->lpd_obd); + lprocfs_obd_cleanup(m->lpd_obd); + + rc = client_obd_cleanup(m->lpd_obd); + LASSERTF(rc == 0, "error %d\n", rc); + + ptlrpcd_decref(); + + RETURN(NULL); +} + +static struct lu_device_type_operations lwp_device_type_ops = { + .ldto_device_alloc = lwp_device_alloc, + .ldto_device_free = lwp_device_free, + .ldto_device_fini = lwp_device_fini +}; + +struct lu_device_type lwp_device_type = { + .ldt_tags = LU_DEVICE_DT, + .ldt_name = LUSTRE_LWP_NAME, + .ldt_ops = &lwp_device_type_ops, + .ldt_ctx_tags = LCT_MD_THREAD +}; + +static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp, + struct obd_device *obd, struct obd_uuid *cluuid, + struct obd_connect_data *data, void *localdata) +{ + struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev); + struct obd_connect_data *ocd; + struct obd_import *imp; + struct lustre_handle conn; + int rc; + + ENTRY; + + CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects); + + rc = class_connect(&conn, obd, cluuid); + if (rc) + RETURN(rc); + + *exp = class_conn2export(&conn); + lwp->lpd_exp = *exp; + + /* Why should there ever be more than 1 connect? */ + lwp->lpd_connects++; + LASSERT(lwp->lpd_connects == 1); + + imp = lwp->lpd_obd->u.cli.cl_import; + imp->imp_dlm_handle = conn; + + LASSERT(data != NULL); + ocd = &imp->imp_connect_data; + *ocd = *data; + + LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT); + + ocd->ocd_version = LUSTRE_VERSION_CODE; + imp->imp_connect_flags_orig = ocd->ocd_connect_flags; + + rc = ptlrpc_connect_import(imp); + if (rc) { + CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc); + GOTO(out, rc); + } + + ptlrpc_pinger_add_import(imp); + +out: + RETURN(rc); +} + +static int lwp_obd_disconnect(struct obd_export *exp) +{ + struct obd_device *obd = exp->exp_obd; + struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev); + int rc; + ENTRY; + + /* Only disconnect the underlying layers on the final disconnect. */ + LASSERT(lwp->lpd_connects == 1); + lwp->lpd_connects--; + + rc = class_disconnect(exp); + if (rc) + CERROR("%s: class disconnect error: rc = %d\n", + obd->obd_name, rc); + + RETURN(rc); +} + +static int lwp_import_event(struct obd_device *obd, struct obd_import *imp, + enum obd_import_event event) +{ + switch (event) { + case IMP_EVENT_DISCON: + case IMP_EVENT_INACTIVE: + case IMP_EVENT_ACTIVE: + break; + case IMP_EVENT_INVALIDATE: + if (obd->obd_namespace == NULL) + break; + ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY); + break; + case IMP_EVENT_OCD: + break; + default: + CERROR("%s: unsupported import event: %#x\n", + obd->obd_name, event); + } + return 0; +} + +struct obd_ops lwp_obd_device_ops = { + .o_owner = THIS_MODULE, + .o_add_conn = client_import_add_conn, + .o_del_conn = client_import_del_conn, + .o_connect = lwp_obd_connect, + .o_disconnect = lwp_obd_disconnect, + .o_import_event = lwp_import_event, +}; diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c index 823af65..6ec6216 100644 --- a/lustre/osp/osp_dev.c +++ b/lustre/osp/osp_dev.c @@ -303,7 +303,7 @@ static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d) } } -int osp_disconnect(struct osp_device *d) +static int osp_disconnect(struct osp_device *d) { struct obd_import *imp; int rc = 0; @@ -342,11 +342,6 @@ static int osp_shutdown(const struct lu_env *env, struct osp_device *d) int rc = 0; ENTRY; - if (is_osp_for_connection(d->opd_obd->obd_name)) { - rc = osp_disconnect(d); - RETURN(rc); - } - LASSERT(env); /* release last_used file */ if (!d->opd_connect_mdt) @@ -378,8 +373,7 @@ static int osp_process_config(const struct lu_env *env, switch (lcfg->lcfg_command) { case LCFG_CLEANUP: - if (!is_osp_for_connection(d->opd_obd->obd_name)) - lu_dev_del_linkage(dev->ld_site, dev); + lu_dev_del_linkage(dev->ld_site, dev); rc = osp_shutdown(env, d); break; case LCFG_PARAM: @@ -776,10 +770,7 @@ static struct lu_device *osp_device_alloc(const struct lu_env *env, l = osp2lu_dev(m); dt_device_init(&m->opd_dt_dev, t); - if (is_osp_for_connection(lustre_cfg_string(lcfg, 0))) - rc = osp_init_for_ost(env, m, t, lcfg); - else - rc = osp_init0(env, m, t, lcfg); + rc = osp_init0(env, m, t, lcfg); if (rc != 0) { osp_device_free(env, l); l = ERR_PTR(rc); @@ -800,9 +791,6 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, if (m->opd_storage_exp) obd_disconnect(m->opd_storage_exp); - if (is_osp_for_connection(m->opd_obd->obd_name)) - osp_fini_for_ost(m); - imp = m->opd_obd->u.cli.cl_import; if (imp->imp_rq_pool) { @@ -866,8 +854,6 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, RETURN(rc); *exp = class_conn2export(&conn); - osp->opd_exp = *exp; - /* Why should there ever be more than 1 connect? */ osp->opd_connects++; LASSERT(osp->opd_connects == 1); @@ -881,8 +867,6 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX); ocd = &imp->imp_connect_data; *ocd = *data; - if (is_osp_for_connection(osp->opd_obd->obd_name)) - ocd->ocd_connect_flags |= OBD_CONNECT_LIGHTWEIGHT; imp->imp_connect_flags_orig = ocd->ocd_connect_flags; @@ -898,15 +882,14 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, ptlrpc_pinger_add_import(imp); - if (osp->opd_connect_mdt && data->ocd_index == 0 && - !is_osp_for_connection(obd->obd_name)) { + if (osp->opd_connect_mdt && data->ocd_index == 0) { + /* set seq controller export for MDC0 if exists */ struct seq_server_site *ss; ss = lu_site2seq(osp2lu_dev(osp)->ld_site); ss->ss_control_exp = class_export_get(*exp); ss->ss_server_fld->lsf_control_exp = *exp; } - out: RETURN(rc); } @@ -934,8 +917,7 @@ static int osp_obd_disconnect(struct obd_export *exp) } /* destroy the device */ - if (!is_osp_for_connection(obd->obd_name)) - class_manual_cleanup(obd); + class_manual_cleanup(obd); RETURN(rc); } @@ -1200,6 +1182,16 @@ static int __init osp_mod_init(void) return rc; } + lprocfs_lwp_init_vars(&lvars); + + rc = class_register_type(&lwp_obd_device_ops, NULL, lvars.module_vars, + LUSTRE_LWP_NAME, &lwp_device_type); + if (rc != 0) { + class_unregister_type(LUSTRE_OSP_NAME); + lu_kmem_fini(osp_caches); + return rc; + } + /* Note: add_rec/delcare_add_rec will be only used by catalogs */ osp_mds_ost_orig_logops = llog_osd_ops; osp_mds_ost_orig_logops.lop_add = llog_cat_add_rec; @@ -1220,6 +1212,7 @@ static void __exit osp_mod_exit(void) { lprocfs_try_remove_proc_entry("osc", proc_lustre_root); + class_unregister_type(LUSTRE_LWP_NAME); class_unregister_type(LUSTRE_OSP_NAME); lu_kmem_fini(osp_caches); } diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 6b95a78..81e3d60 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -406,10 +406,9 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d); int osp_sync_fini(struct osp_device *d); void __osp_sync_check_for_work(struct osp_device *d); -/* osp_ost.c */ -int osp_init_for_ost(const struct lu_env *env, struct osp_device *m, - struct lu_device_type *ldt, struct lustre_cfg *cfg); -int osp_disconnect(struct osp_device *d); -int osp_fini_for_ost(struct osp_device *osp); +/* lwp_dev.c */ +void lprocfs_lwp_init_vars(struct lprocfs_static_vars *lvars); +extern struct obd_ops lwp_obd_device_ops; +extern struct lu_device_type lwp_device_type; #endif diff --git a/lustre/osp/osp_ost.c b/lustre/osp/osp_ost.c deleted file mode 100644 index a177e12..0000000 --- a/lustre/osp/osp_ost.c +++ /dev/null @@ -1,183 +0,0 @@ -/* - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 021110-1307, USA - * - * GPL HEADER END - */ -/* - * Copyright (c) 2012, Intel Corporation. - * Use is subject to license terms. - * - * lustre/osp/osp_ost.c - * - * OSP on OST for communicating with MDT0 - * - * Author: - */ -#ifndef EXPORT_SYMTAB -# define EXPORT_SYMTAB -#endif -#define DEBUG_SUBSYSTEM S_OST - -#include -#include -#include - -#include "osp_internal.h" - -static int osp_name2fsname(char *ospname, char *fsname) -{ - char *ptr; - - LASSERT(ospname != NULL); - LASSERT(fsname != NULL); - if (!is_osp_for_connection(ospname)) - return -EINVAL; - - sprintf(fsname, "-%s-", LUSTRE_OSP_NAME); - - ptr = strstr(ospname, fsname); - if (ptr == NULL) - return -EINVAL; - - while (*(--ptr) != '-') { - if (ptr == ospname) - return -EINVAL; - } - - strncpy(fsname, ospname, ptr - ospname); - fsname[ptr - ospname] = '\0'; - - return 0; -} - -static int osp_setup_for_ost(const struct lu_env *env, struct osp_device *osp, - char *nidstring) -{ - struct lustre_cfg_bufs *bufs = NULL; - struct lustre_cfg *lcfg = NULL; - char *ospname = osp->opd_obd->obd_name; - char *fsname = NULL; - char *server_uuid = NULL; - class_uuid_t uuid; - struct obd_import *imp; - int rc; - ENTRY; - - OBD_ALLOC_PTR(bufs); - if (bufs == NULL) - RETURN(-ENOMEM); - - OBD_ALLOC(fsname, strlen(ospname)); - if (fsname == NULL) - GOTO(out, rc = -ENOMEM); - - rc = osp_name2fsname(ospname, fsname); - if (rc) { - CERROR("%s: name change error: rc %d\n", ospname, rc); - GOTO(out, rc); - } - - OBD_ALLOC(server_uuid, strlen(fsname) + 15); - if (server_uuid == NULL) - GOTO(out, rc = -ENOMEM); - - sprintf(server_uuid, "%s-MDT0000_UUID", fsname); - lustre_cfg_bufs_reset(bufs, ospname); - lustre_cfg_bufs_set_string(bufs, 1, server_uuid); - lustre_cfg_bufs_set_string(bufs, 2, nidstring); - lcfg = lustre_cfg_new(LCFG_SETUP, bufs); - if (lcfg == NULL) - GOTO(out, rc = -ENOMEM); - - osp->opd_connect_mdt = 1; - rc = client_obd_setup(osp->opd_obd, lcfg); - if (rc != 0) { - CERROR("%s: client obd setup error: rc = %d\n", - osp->opd_obd->obd_name, rc); - GOTO(out, rc); - } - - imp = osp->opd_obd->u.cli.cl_import; - rc = ptlrpc_init_import(imp); - if (rc) - GOTO(out, rc); - - ll_generate_random_uuid(uuid); - class_uuid_unparse(uuid, &osp->opd_cluuid); -out: - if (bufs != NULL) - OBD_FREE_PTR(bufs); - if (server_uuid != NULL) - OBD_FREE(server_uuid, strlen(fsname) + 15); - if (fsname != NULL) - OBD_FREE(fsname, strlen(ospname)); - if (lcfg != NULL) - lustre_cfg_free(lcfg); - if (rc) - client_obd_cleanup(osp->opd_obd); - RETURN(rc); -} - -int osp_fini_for_ost(struct osp_device *osp) -{ - if (osp->opd_exp != NULL) - class_disconnect(osp->opd_exp); - return 0; -} - -int osp_init_for_ost(const struct lu_env *env, struct osp_device *osp, - struct lu_device_type *ldt, struct lustre_cfg *cfg) -{ - struct lprocfs_static_vars lvars = { 0 }; - int rc; - ENTRY; - - osp->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0)); - if (osp->opd_obd == NULL) { - CERROR("Cannot find obd with name %s\n", - lustre_cfg_string(cfg, 0)); - RETURN(-ENODEV); - } - - osp->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops; - osp->opd_dt_dev.dd_ops = &osp_dt_ops; - osp->opd_obd->obd_lu_dev = &osp->opd_dt_dev.dd_lu_dev; - - rc = ptlrpcd_addref(); - if (rc) { - CERROR("%s: ptlrpcd addref error: rc =%d\n", - osp->opd_obd->obd_name, rc); - RETURN(rc); - } - - rc = osp_setup_for_ost(env, osp, lustre_cfg_string(cfg, 1)); - if (rc) { - CERROR("%s: osp_setup_for_ost error: rc =%d\n", - osp->opd_obd->obd_name, rc); - ptlrpcd_decref(); - RETURN(rc); - } - - lprocfs_osp_init_vars(&lvars); - if (lprocfs_obd_setup(osp->opd_obd, lvars.obd_vars) == 0) - ptlrpc_lprocfs_register_obd(osp->opd_obd); - - RETURN(0); -} diff --git a/lustre/quota/qsd_lib.c b/lustre/quota/qsd_lib.c index adb2f86f..0d0e518 100644 --- a/lustre/quota/qsd_lib.c +++ b/lustre/quota/qsd_lib.c @@ -487,7 +487,7 @@ void qsd_fini(const struct lu_env *env, struct qsd_instance *qsd) /* deregister connection to the quota master */ qsd->qsd_exp_valid = false; - lustre_deregister_osp_item(&qsd->qsd_exp); + lustre_deregister_lwp_item(&qsd->qsd_exp); /* release per-filesystem information */ if (qsd->qsd_fsinfo != NULL) { @@ -709,7 +709,7 @@ int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd) } /* generate osp name */ - rc = tgt_name2ospname((char *)qsd->qsd_svname, qti->qti_buf); + rc = tgt_name2lwpname((char *)qsd->qsd_svname, qti->qti_buf); if (rc) { CERROR("%s: failed to generate ospname (%d)\n", qsd->qsd_svname, rc); @@ -718,7 +718,7 @@ int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd) /* the connection callback will start the reintegration * procedure if quota is enabled */ - rc = lustre_register_osp_item(qti->qti_buf, &qsd->qsd_exp, + rc = lustre_register_lwp_item(qti->qti_buf, &qsd->qsd_exp, qsd_conn_callback, (void *)qsd); if (rc) { CERROR("%s: fail to get connection to master (%d)\n", diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index daa6f27..68d6eda 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -1469,8 +1469,8 @@ run_test 12b "Inode quota rebalancing" test_13(){ local TESTFILE=$DIR/$tdir/$tfile - # the name of osp on ost1 name is MDT0000-osp-OST0000 - local procf="ldlm.namespaces.*MDT0000-osp-OST0000.lru_size" + # the name of lwp on ost1 name is MDT0000-lwp-OST0000 + local procf="ldlm.namespaces.*MDT0000-lwp-OST0000.lru_size" setup_quota_test trap cleanup_quota_test EXIT