From 10e9bc1b7d37b4c7283310712e316d95c5c9c0ba Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Fri, 21 Sep 2012 04:54:54 -0400 Subject: [PATCH] LU-1842 osp: osp-on-ost device Setup osp-on-ost device for each OST to maintain the connection to MDT0, quota slave & ofd will send quota or fid sequence request over this connection. We also setup one such device on MDT0 for now, it'll be used by the quota slave on MDT0, once the DNE code landed, this device (on MDT0) could be replaced by normal osp device. Signed-off-by: Wang Di Signed-off-by: Niu Yawei Change-Id: I2229cec1fa206dcce65a4cceae032566f4c20901 Reviewed-on: http://review.whamcloud.com/4068 Reviewed-by: Johann Lombardi Tested-by: Hudson Tested-by: Maloo --- lustre/include/lustre_log.h | 2 - lustre/include/obd.h | 27 ++ lustre/include/obd_class.h | 27 ++ lustre/ldlm/ldlm_lib.c | 8 +- lustre/llite/llite_lib.c | 2 +- lustre/mgc/mgc_request.c | 6 + lustre/obdclass/lustre_peer.c | 1 + lustre/obdclass/obd_config.c | 20 +- lustre/obdclass/obd_mount.c | 787 +++++++++++++++++++++++++++++++++++++---- lustre/osp/Makefile.in | 1 + lustre/osp/osp_dev.c | 79 ++++- lustre/osp/osp_internal.h | 8 + lustre/osp/osp_ost.c | 175 +++++++++ lustre/tests/test-framework.sh | 1 + 14 files changed, 1049 insertions(+), 95 deletions(-) create mode 100644 lustre/osp/osp_ost.c diff --git a/lustre/include/lustre_log.h b/lustre/include/lustre_log.h index 8b5b98b..b62255f 100644 --- a/lustre/include/lustre_log.h +++ b/lustre/include/lustre_log.h @@ -116,8 +116,6 @@ static inline void fid_to_logid(struct lu_fid *fid, struct llog_logid *id) struct llog_handle; /* llog.c - general API */ -typedef int (*llog_cb_t)(const struct lu_env *env, struct llog_handle *lgh, - struct llog_rec_hdr *rec, void *data); int llog_init_handle(const struct lu_env *env, struct llog_handle *handle, int flags, struct obd_uuid *uuid); int llog_process(const struct lu_env *env, struct llog_handle *loghandle, diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 3b9fdf6..48ec59e 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -856,6 +856,33 @@ struct niobuf_local { /* Don't conflict with on-wire flags OBD_BRW_WRITE, etc */ #define N_LOCAL_TEMP_PAGE 0x10000000 +static inline int is_osp_on_ost(char *name) +{ + char *ptr; + + ptr = strrchr(name, '-'); + if (ptr == NULL) { + CERROR("%s is not a obdname\n", name); + return 0; + } + + if (strncmp(ptr + 1, "OST", 3) != 0 && strncmp(ptr + 1, "MDT", 3) != 0) + return 0; + + /* match the "-osp" */ + if (ptr - name < strlen(LUSTRE_OSP_NAME) + 1) + return 0; + + ptr -= (strlen(LUSTRE_OSP_NAME) + 1); + if (*ptr != '-') + return 0; + + if (strncmp(ptr + 1, LUSTRE_OSP_NAME, strlen(LUSTRE_OSP_NAME)) != 0) + return 0; + + return 1; +} + struct obd_trans_info { __u64 oti_transno; __u64 oti_xid; diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index f5522ef..c22caa8 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -124,6 +124,10 @@ int kuc_ispayload(void *p); void *kuc_alloc(int payload_len, int transport, int type); void kuc_free(void *p, int payload_len); +struct llog_handle; +struct llog_rec_hdr; +typedef int (*llog_cb_t)(const struct lu_env *, struct llog_handle *, + struct llog_rec_hdr *, void *); /* obd_config.c */ struct lustre_cfg *lustre_cfg_rename(struct lustre_cfg *cfg, const char *new_name); @@ -139,6 +143,11 @@ struct obd_device *class_incref(struct obd_device *obd, void class_decref(struct obd_device *obd, const char *scope, const void *source); void dump_exports(struct obd_device *obd, int locks); +int class_config_llog_handler(const struct lu_env *env, + struct llog_handle *handle, + struct llog_rec_hdr *rec, void *data); +int class_add_conn(struct obd_device *obd, struct lustre_cfg *lcfg); +int class_add_uuid(const char *uuid, __u64 nid); /*obdecho*/ #ifdef LPROCFS @@ -162,6 +171,7 @@ struct config_llog_instance { void *cfg_instance; struct super_block *cfg_sb; struct obd_uuid cfg_uuid; + llog_cb_t cfg_callback; int cfg_last_idx; /* for partial llog processing */ int cfg_flags; }; @@ -2262,12 +2272,29 @@ static inline void fid2obdo(struct lu_fid *fid, struct obdo *oa) /* something here */ } +typedef int (*register_osp_cb)(void *data); + +struct osp_register_item { + struct obd_export **ori_exp; + register_osp_cb ori_cb_func; + void *ori_cb_data; + cfs_list_t ori_list; + char ori_name[MTI_NAME_MAXLEN]; +}; + /* I'm as embarrassed about this as you are. * * // XXX do not look into _superhack with remaining eye * // XXX if this were any uglier, I'd get my own show on MTV */ extern int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c); +/* obd_mount.c */ +int server_name2fsname(char *svname, char *fsname, char **endptr); +int lustre_register_osp_item(char *ospname, struct obd_export **exp, + register_osp_cb cb_func, void *cb_data); +void lustre_deregister_osp_item(struct obd_export **exp); +int tgt_name2ospname(char *ost_name, char *osp_name); + /* sysctl.c */ extern void obd_sysctl_init (void); extern void obd_sysctl_clean (void); diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index d14a6dc..288003c 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -266,7 +266,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) /* In a more perfect world, we would hang a ptlrpc_client off of * obd_type and just use the values from there. */ - if (!strcmp(name, LUSTRE_OSC_NAME)) { + if (!strcmp(name, LUSTRE_OSC_NAME) || + (!strcmp(name, LUSTRE_OSP_NAME) && + !is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) { rq_portal = OST_REQUEST_PORTAL; rp_portal = OSC_REPLY_PORTAL; connect_op = OST_CONNECT; @@ -274,7 +276,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) cli->cl_sp_to = LUSTRE_SP_OST; ns_type = LDLM_NS_TYPE_OSC; - } else if (!strcmp(name, LUSTRE_MDC_NAME)) { + } else if (!strcmp(name, LUSTRE_MDC_NAME) || + (!strcmp(name, LUSTRE_OSP_NAME) && + is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) { rq_portal = MDS_REQUEST_PORTAL; rp_portal = MDC_REPLY_PORTAL; connect_op = MDS_CONNECT; diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 7305317..b30165a 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -997,7 +997,7 @@ int ll_fill_super(struct super_block *sb, struct vfsmount *mnt) Use the address of the super itself.*/ cfg->cfg_instance = sb; cfg->cfg_uuid = lsi->lsi_llsbi->ll_sb_uuid; - + cfg->cfg_callback = class_config_llog_handler; /* set up client obds */ err = lustre_process_log(sb, profilenm, cfg); if (err < 0) { diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index b1553ba..34d19fb 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -214,6 +214,8 @@ struct config_llog_data *do_config_log_add(struct obd_device *obd, strcpy(cld->cld_logname, logname); if (cfg) cld->cld_cfg = *cfg; + else + cld->cld_cfg.cfg_callback = class_config_llog_handler; cfs_mutex_init(&cld->cld_lock); cld->cld_cfg.cfg_last_idx = 0; cld->cld_cfg.cfg_flags = 0; @@ -262,6 +264,10 @@ static struct config_llog_data *config_recover_log_add(struct obd_device *obd, if (IS_OST(lsi)) return NULL; + /* for osp-on-ost, see lustre_start_osp() */ + if (IS_MDT(lsi) && lcfg.cfg_instance) + return NULL; + /* we have to use different llog for clients and mdts for cmd * where only clients are notified if one of cmd server restarts */ LASSERT(strlen(fsname) < sizeof(logname) / 2); diff --git a/lustre/obdclass/lustre_peer.c b/lustre/obdclass/lustre_peer.c index 20e14dc..0d53c7e 100644 --- a/lustre/obdclass/lustre_peer.c +++ b/lustre/obdclass/lustre_peer.c @@ -145,6 +145,7 @@ int class_add_uuid(const char *uuid, __u64 nid) } return 0; } +EXPORT_SYMBOL(class_add_uuid); /* Delete the nids for one uuid if specified, otherwise delete all */ int class_del_uuid(const char *uuid) diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index f2344e2..02c4669 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -800,6 +800,7 @@ int class_add_conn(struct obd_device *obd, struct lustre_cfg *lcfg) RETURN(rc); } +EXPORT_SYMBOL(class_add_conn); /** Remove a failover nid location. */ @@ -1334,9 +1335,9 @@ extern int lustre_check_exclusion(struct super_block *sb, char *svname); * records, change uuids, etc), then class_process_config() resulting * net records. */ -static int class_config_llog_handler(const struct lu_env *env, - struct llog_handle *handle, - struct llog_rec_hdr *rec, void *data) +int class_config_llog_handler(const struct lu_env *env, + struct llog_handle *handle, + struct llog_rec_hdr *rec, void *data) { struct config_llog_instance *clli = data; int cfg_len = rec->lrh_len; @@ -1547,14 +1548,15 @@ out: } RETURN(rc); } +EXPORT_SYMBOL(class_config_llog_handler); int class_config_parse_llog(const struct lu_env *env, struct llog_ctxt *ctxt, char *name, struct config_llog_instance *cfg) { struct llog_process_cat_data cd = {0, 0}; struct llog_handle *llh; + llog_cb_t callback; int rc; - ENTRY; CDEBUG(D_INFO, "looking up llog %s\n", name); @@ -1567,11 +1569,17 @@ int class_config_parse_llog(const struct lu_env *env, struct llog_ctxt *ctxt, GOTO(parse_out, rc); /* continue processing from where we last stopped to end-of-log */ - if (cfg) + if (cfg) { cd.lpcd_first_idx = cfg->cfg_last_idx; + callback = cfg->cfg_callback; + LASSERT(callback != NULL); + } else { + callback = class_config_llog_handler; + } + cd.lpcd_last_idx = 0; - rc = llog_process(env, llh, class_config_llog_handler, cfg, &cd); + rc = llog_process(env, llh, callback, cfg, &cd); CDEBUG(D_CONFIG, "Processed log %s gen %d-%d (rc=%d)\n", name, cd.lpcd_first_idx + 1, cd.lpcd_last_idx, rc); diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 2ca91e9..4c82860 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -822,6 +822,609 @@ static int server_mgc_clear_fs(struct obd_device *mgc) RETURN(rc); } +/** Get the fsname ("lustre") from the server name ("lustre-OST003F"). + * @param [in] svname server name including type and index + * @param [out] fsname Buffer to copy filesystem name prefix into. + * Must have at least 'strlen(fsname) + 1' chars. + * @param [out] endptr if endptr isn't NULL it is set to end of fsname + * rc < 0 on error + */ +int server_name2fsname(char *svname, char *fsname, char **endptr) +{ + char *dash = strrchr(svname, '-'); + if (!dash) { + dash = strrchr(svname, ':'); + if (!dash) + return -EINVAL; + } + + /* interpret -MDTXXXXX-mdc as mdt, the better way is to pass + * in the fsname, then determine the server index */ + if (!strcmp(LUSTRE_MDC_NAME, dash + 1)) { + dash--; + for (; dash > svname && *dash != '-' && *dash != ':'; dash--) + ; + if (dash == svname) + return -EINVAL; + } + + if (fsname != NULL) { + strncpy(fsname, svname, dash - svname); + fsname[dash - svname] = '\0'; + } + + if (endptr != NULL) + *endptr = dash; + + return 0; +} +EXPORT_SYMBOL(server_name2fsname); + +static int is_mdc_device(char *devname) +{ + char *ptr; + ptr = strrchr(devname, '-'); + if (ptr != NULL && strcmp(ptr, "-mdc") == 0) + return 1; + return 0; +} + +static int inline tgt_is_mdt0(char *tgtname) +{ + __u32 idx; + int type; + + type = server_name2index(tgtname, &idx, NULL); + if (type != LDD_F_SV_TYPE_MDT) + return 0; + + return (idx == 0) ? 1 :0; +} + +static int inline is_mdc_for_mdt0(char *devname) +{ + char *ptr; + + if (!is_mdc_device(devname)) + return 0; + + ptr = strrchr(devname, '-'); + if (ptr == NULL) + return 0; + + *ptr = 0; + if (tgt_is_mdt0(devname)) { + *ptr = '-'; + return 1; + } + *ptr = '-'; + return 0; +} + +/** + * Convert OST/MDT name(fsname-OSTxxxx) to an osp name + * (fsname-MDT0000-osp-OSTxxxx), which will be used to + * communicate with MDT0 for this target. + **/ +int tgt_name2ospname(char *svname, char *ospname) +{ + char *fsname; + char *tgt; + int rc; + ENTRY; + + OBD_ALLOC(fsname, MTI_NAME_MAXLEN); + if (fsname == NULL) + RETURN(-ENOMEM); + + rc = server_name2fsname(svname, fsname, &tgt); + if (rc != 0) { + CERROR("%s change fsname error: rc %d\n", svname, rc); + GOTO(cleanup, rc); + } + + if (*tgt != '-' && *tgt != ':') { + CERROR("%s wrong svname name!\n", svname); + GOTO(cleanup, rc = -EINVAL); + } + + tgt++; + if (strncmp(tgt, "OST", 3) != 0 && strncmp(tgt, "MDT", 3) != 0) { + CERROR("%s is not an OST or MDT target!\n", svname); + GOTO(cleanup, rc = -EINVAL); + } + sprintf(ospname, "%s-MDT0000-%s-%s", fsname, LUSTRE_OSP_NAME, tgt); +cleanup: + if (fsname != NULL) + OBD_FREE(fsname, MTI_NAME_MAXLEN); + RETURN(rc); +} +EXPORT_SYMBOL(tgt_name2ospname); + +static CFS_LIST_HEAD(osp_register_list); +CFS_DEFINE_MUTEX(osp_register_list_lock); + +int lustre_register_osp_item(char *ospname, struct obd_export **exp, + register_osp_cb cb_func, void *cb_data) +{ + struct obd_device *osp; + struct osp_register_item *ori; + ENTRY; + + LASSERTF(strlen(ospname) < MTI_NAME_MAXLEN, "ospname is too long %s\n", + ospname); + LASSERT(exp != NULL && *exp == NULL); + + OBD_ALLOC_PTR(ori); + if (ori == NULL) + RETURN(-ENOMEM); + + cfs_mutex_lock(&osp_register_list_lock); + + osp = class_name2obd(ospname); + if (osp != NULL && osp->obd_set_up == 1) { + struct obd_uuid *uuid; + + OBD_ALLOC_PTR(uuid); + if (uuid == NULL) { + cfs_mutex_unlock(&osp_register_list_lock); + RETURN(-ENOMEM); + } + memcpy(uuid->uuid, ospname, strlen(ospname)); + *exp = cfs_hash_lookup(osp->obd_uuid_hash, uuid); + OBD_FREE_PTR(uuid); + } + + memcpy(ori->ori_name, ospname, strlen(ospname)); + ori->ori_exp = exp; + ori->ori_cb_func = cb_func; + ori->ori_cb_data = cb_data; + CFS_INIT_LIST_HEAD(&ori->ori_list); + cfs_list_add(&ori->ori_list, &osp_register_list); + + if (*exp != NULL && cb_func != NULL) + cb_func(cb_data); + + cfs_mutex_unlock(&osp_register_list_lock); + RETURN(0); +} +EXPORT_SYMBOL(lustre_register_osp_item); + +void lustre_deregister_osp_item(struct obd_export **exp) +{ + struct osp_register_item *ori, *tmp; + + cfs_mutex_lock(&osp_register_list_lock); + cfs_list_for_each_entry_safe(ori, tmp, &osp_register_list, ori_list) { + if (exp == ori->ori_exp) { + if (*exp) + class_export_put(*exp); + cfs_list_del(&ori->ori_list); + OBD_FREE_PTR(ori); + break; + } + } + cfs_mutex_unlock(&osp_register_list_lock); +} +EXPORT_SYMBOL(lustre_deregister_osp_item); + +static void lustre_notify_osp_list(struct obd_export *exp) +{ + struct osp_register_item *ori, *tmp; + LASSERT(exp != NULL); + + cfs_mutex_lock(&osp_register_list_lock); + cfs_list_for_each_entry_safe(ori, tmp, &osp_register_list, ori_list) { + if (strcmp(exp->exp_obd->obd_name, ori->ori_name)) + continue; + if (*ori->ori_exp != NULL) + continue; + *ori->ori_exp = class_export_get(exp); + if (ori->ori_cb_func != NULL) + ori->ori_cb_func(ori->ori_cb_data); + } + cfs_mutex_unlock(&osp_register_list_lock); +} + +static int lustre_osp_connect(struct obd_device *osp) +{ + struct lu_env env; + struct lu_context session_ctx; + struct obd_export *exp; + struct obd_uuid *uuid = NULL; + struct obd_connect_data *data = NULL; + int rc; + ENTRY; + + /* log has been fully processed, let clients connect */ + rc = lu_env_init(&env, osp->obd_lu_dev->ld_type->ldt_ctx_tags); + if (rc != 0) + RETURN(rc); + + lu_context_init(&session_ctx, LCT_SESSION); + session_ctx.lc_thread = NULL; + lu_context_enter(&session_ctx); + env.le_ses = &session_ctx; + + OBD_ALLOC_PTR(data); + if (data == NULL) + GOTO(out, rc = -ENOMEM); + + data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_INDEX; + data->ocd_version = LUSTRE_VERSION_CODE; + data->ocd_ibits_known = MDS_INODELOCK_UPDATE; + data->ocd_connect_flags |= OBD_CONNECT_ACL | OBD_CONNECT_IBITS | + OBD_CONNECT_MDS_MDS | OBD_CONNECT_FID | + OBD_CONNECT_AT | OBD_CONNECT_FULL20 | + OBD_CONNECT_LIGHTWEIGHT; + OBD_ALLOC_PTR(uuid); + if (uuid == NULL) + GOTO(out, rc = -ENOMEM); + + if (strlen(osp->obd_name) > sizeof(uuid->uuid)) { + CERROR("%s: Too long osp name %s, max_size is %d\n", + osp->obd_name, osp->obd_name, (int)sizeof(uuid->uuid)); + GOTO(out, rc = -EINVAL); + } + /* Use osp name as the uuid, so we find the export by + * osp name later */ + memcpy(uuid->uuid, osp->obd_name, strlen(osp->obd_name)); + rc = obd_connect(&env, &exp, osp, uuid, data, NULL); + if (rc != 0) + CERROR("%s: connect failed: rc = %d\n", osp->obd_name, rc); + else + lustre_notify_osp_list(exp); + +out: + if (data != NULL) + OBD_FREE_PTR(data); + if (uuid != NULL) + OBD_FREE_PTR(uuid); + + lu_env_fini(&env); + lu_context_exit(&session_ctx); + lu_context_fini(&session_ctx); + + RETURN(rc); +} + +/** + * osp-on-ost is used by slaves (Non-MDT0 targets) to manage the connection + * to MDT0. + * + * The OSTs will communicate with MDT0 by the connection established by the + * osp-on-ost to get quota and fid sequence. + * + **/ +static int lustre_osp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi) +{ + struct obd_connect_data *data = NULL; + struct obd_device *obd; + char *ospname = NULL; + char *ospuuid = NULL; + int rc; + ENTRY; + + rc = class_add_uuid(lustre_cfg_string(lcfg, 1), + lcfg->lcfg_nid); + if (rc) { + CERROR("%s: Can't add uuid: rc =%d\n", lsi->lsi_svname, rc); + GOTO(out, rc); + } + + OBD_ALLOC(ospname, MTI_NAME_MAXLEN); + if (ospname == NULL) + GOTO(out, rc = -ENOMEM); + + rc = tgt_name2ospname(lsi->lsi_svname, ospname); + if (rc != 0) { + CERROR("%s change ospname error: rc %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc); + } + + OBD_ALLOC(ospuuid, MTI_NAME_MAXLEN); + if (ospuuid == NULL) + GOTO(out, rc = -ENOMEM); + + sprintf(ospuuid, "%s_UUID", ospname); + rc = lustre_start_simple(ospname, LUSTRE_OSP_NAME, + ospuuid, lustre_cfg_string(lcfg, 1), + 0, 0, 0); + if (rc) { + CERROR("%s: setup up failed: rc %d\n", ospname, rc); + GOTO(out, rc); + } + + obd = class_name2obd(ospname); + LASSERT(obd != NULL); + + rc = lustre_osp_connect(obd); + if (rc != 0) + CERROR("%s: connect failed: rc = %d\n", ospname, rc); +out: + if (data != NULL) + OBD_FREE_PTR(data); + if (ospname != NULL) + OBD_FREE(ospname, MTI_NAME_MAXLEN); + if (ospuuid != NULL) + OBD_FREE(ospuuid, MTI_NAME_MAXLEN); + + RETURN(rc); +} + +static int lustre_osp_add_conn(struct lustre_cfg *cfg, + struct lustre_sb_info *lsi) +{ + struct lustre_cfg_bufs *bufs = NULL; + struct lustre_cfg *lcfg = NULL; + char *ospname = NULL; + struct obd_device *osp; + int rc; + ENTRY; + + OBD_ALLOC(ospname, MTI_NAME_MAXLEN); + if (ospname == NULL) + GOTO(out, rc = -ENOMEM); + + rc = tgt_name2ospname(lsi->lsi_svname, ospname); + if (rc != 0) { + CERROR("%s change ospname error: rc %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc); + } + + OBD_ALLOC_PTR(bufs); + if (bufs == NULL) + RETURN(-ENOMEM); + + lustre_cfg_bufs_reset(bufs, ospname); + lustre_cfg_bufs_set_string(bufs, 1, + lustre_cfg_string(cfg, 1)); + + lcfg = lustre_cfg_new(LCFG_ADD_CONN, bufs); + + osp = class_name2obd(ospname); + if (osp == NULL) { + CERROR("Can not find %s\n", ospname); + GOTO(out, rc = -EINVAL); + } + + rc = class_add_conn(osp, lcfg); + if (rc) + CERROR("%s: can't add conn: rc = %d\n", ospname, rc); + +out: + if (bufs != NULL) + OBD_FREE_PTR(bufs); + if (lcfg != NULL) + lustre_cfg_free(lcfg); + if (ospname != NULL) + OBD_FREE(ospname, MTI_NAME_MAXLEN); + + RETURN(rc); +} + +/** + * Retrieve MDT nids from the client log, then start the osp-on-ost device. + * there are only two scenarios which would include mdt nid. + * 1. + * marker 5 (flags=0x01, v2.1.54.0) lustre-MDT0000 'add mdc' xxx- + * add_uuid nid=192.168.122.162@tcp(0x20000c0a87aa2) 0: 1:192.168.122.162@tcp + * attach 0:lustre-MDT0000-mdc 1:mdc 2:lustre-clilmv_UUID + * setup 0:lustre-MDT0000-mdc 1:lustre-MDT0000_UUID 2:192.168.122.162@tcp + * add_uuid nid=192.168.172.1@tcp(0x20000c0a8ac01) 0: 1:192.168.172.1@tcp + * add_conn 0:lustre-MDT0000-mdc 1:192.168.172.1@tcp + * modify_mdc_tgts add 0:lustre-clilmv 1:lustre-MDT0000_UUID xxxx + * marker 5 (flags=0x02, v2.1.54.0) lustre-MDT0000 'add mdc' xxxx- + * 2. + * marker 7 (flags=0x01, v2.1.54.0) lustre-MDT0000 'add failnid' xxxx- + * add_uuid nid=192.168.122.2@tcp(0x20000c0a87a02) 0: 1:192.168.122.2@tcp + * add_conn 0:lustre-MDT0000-mdc 1:192.168.122.2@tcp + * marker 7 (flags=0x02, v2.1.54.0) lustre-MDT0000 'add failnid' xxxx- +**/ +static int client_osp_config_process(const struct lu_env *env, + struct llog_handle *handle, + struct llog_rec_hdr *rec, void *data) +{ + struct config_llog_instance *clli = data; + int cfg_len = rec->lrh_len; + char *cfg_buf = (char *) (rec + 1); + struct lustre_cfg *lcfg = NULL; + struct lustre_sb_info *lsi; + int rc = 0, swab = 0; + ENTRY; + + if (rec->lrh_type != OBD_CFG_REC) { + CERROR("Unknown llog record type %#x encountered\n", + rec->lrh_type); + RETURN(-EINVAL); + } + + LASSERT(clli->cfg_sb != NULL); + lsi = s2lsi(clli->cfg_sb); + + lcfg = (struct lustre_cfg *)cfg_buf; + if (lcfg->lcfg_version == __swab32(LUSTRE_CFG_VERSION)) { + lustre_swab_lustre_cfg(lcfg); + swab = 1; + } + + rc = lustre_cfg_sanity_check(cfg_buf, cfg_len); + if (rc) + GOTO(out, rc); + + switch (lcfg->lcfg_command) { + case LCFG_MARKER: { + struct cfg_marker *marker = lustre_cfg_buf(lcfg, 1); + + lustre_swab_cfg_marker(marker, swab, + LUSTRE_CFG_BUFLEN(lcfg, 1)); + if (marker->cm_flags & CM_SKIP || + marker->cm_flags & CM_EXCLUDE) + GOTO(out, rc = 0); + + if (!tgt_is_mdt0(marker->cm_tgtname)) + GOTO(out, rc = 0); + + if(!strncmp(marker->cm_comment, "add mdc", 7) || + !strncmp(marker->cm_comment, "add failnid", 11)) { + if (marker->cm_flags & CM_START) { + clli->cfg_flags = CFG_F_MARKER; + /* This hack is to differentiate the + * ADD_UUID is come from "add mdc" record + * or from "add failnid" record. */ + if (!strncmp(marker->cm_comment, + "add failnid", 11)) + clli->cfg_flags |= CFG_F_SKIP; + } else if (marker->cm_flags & CM_END) { + clli->cfg_flags = 0; + } + } + break; + } + case LCFG_ADD_UUID: { + if (clli->cfg_flags == CFG_F_MARKER) { + rc = lustre_osp_setup(lcfg, lsi); + } else if (clli->cfg_flags == (CFG_F_MARKER | CFG_F_SKIP)) { + rc = class_add_uuid(lustre_cfg_string(lcfg, 1), + lcfg->lcfg_nid); + if (rc) + CERROR("%s: Fail to add uuid, rc:%d\n", + lsi->lsi_svname, rc); + } + break; + } + case LCFG_ADD_CONN: { + if (is_mdc_for_mdt0(lustre_cfg_string(lcfg, 0))) + rc = lustre_osp_add_conn(lcfg, lsi); + break; + } + default: + break; + } +out: + RETURN(rc); +} + +static int lustre_disconnect_osp(struct super_block *sb) +{ + struct lustre_sb_info *lsi = s2lsi(sb); + struct obd_device *osp; + char *ospname = NULL; + char *logname = NULL; + struct lustre_cfg *lcfg = NULL; + struct lustre_cfg_bufs *bufs = NULL; + struct config_llog_instance *cfg = NULL; + int rc; + ENTRY; + + LASSERT(IS_OST(lsi) || IS_MDT(lsi)); + OBD_ALLOC(logname, MTI_NAME_MAXLEN); + if (logname == NULL) + RETURN(-ENOMEM); + + OBD_ALLOC(ospname, MTI_NAME_MAXLEN); + if (ospname == NULL) + GOTO(out, rc = -ENOMEM); + + rc = server_name2fsname(lsi->lsi_svname, ospname, NULL); + if (rc != 0) { + CERROR("%s: get fsname error: %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc); + } + sprintf(logname, "%s-client", ospname); + + OBD_ALLOC_PTR(cfg); + if (cfg == NULL) + GOTO(out, rc = -ENOMEM); + + /* end log first */ + cfg->cfg_instance = sb; + rc = lustre_end_log(sb, logname, cfg); + if (rc != 0) { + CERROR("Can't end config log %s\n", ospname); + GOTO(out, rc); + } + + rc = tgt_name2ospname(lsi->lsi_svname, ospname); + if (rc != 0) { + CERROR("%s: get osp name error: %d\n", + lsi->lsi_svname, rc); + GOTO(out, rc); + } + + osp = class_name2obd(ospname); + if (osp == NULL) { + CERROR("Can't find osp-on-ost %s\n", ospname); + GOTO(out, rc = -ENOENT); + } + + OBD_ALLOC_PTR(bufs); + if (bufs == NULL) + GOTO(out, rc = -ENOMEM); + + lustre_cfg_bufs_reset(bufs, osp->obd_name); + lustre_cfg_bufs_set_string(bufs, 1, NULL); + lcfg = lustre_cfg_new(LCFG_CLEANUP, bufs); + if (!lcfg) + GOTO(out, rc = -ENOMEM); + + /* Disconnect import first. NULL is passed for the '@env', since + * it will not be used for the 'osp-on-ost'. (see osp_shutdown()) */ + rc = osp->obd_lu_dev->ld_ops->ldo_process_config(NULL, osp->obd_lu_dev, + lcfg); +out: + if (lcfg) + lustre_cfg_free(lcfg); + if (bufs) + OBD_FREE_PTR(bufs); + if (cfg) + OBD_FREE_PTR(cfg); + if (ospname) + OBD_FREE(ospname, MTI_NAME_MAXLEN); + if (logname) + OBD_FREE(logname, MTI_NAME_MAXLEN); + RETURN(rc); +} + +/** + * Stop the osp(fsname-MDT0000-osp-OSTxxxx) for an OST target. + **/ +static int lustre_stop_osp(struct super_block *sb) +{ + struct lustre_sb_info *lsi = s2lsi(sb); + struct obd_device *osp = NULL; + char *ospname = NULL; + int rc = 0; + ENTRY; + + LASSERT(IS_OST(lsi) || IS_MDT(lsi)); + OBD_ALLOC(ospname, MTI_NAME_MAXLEN); + + rc = tgt_name2ospname(lsi->lsi_svname, ospname); + if (rc != 0) { + CERROR("%s get fsname error: rc %d\n", + lsi->lsi_svname, rc); + GOTO(cleanup, rc); + } + + osp = class_name2obd(ospname); + if (osp == NULL) { + CERROR("Can not find osp-on-ost %s\n", ospname); + GOTO(cleanup, rc = -ENOENT); + } + + osp->obd_force = 1; + rc = class_manual_cleanup(osp); + +cleanup: + if (ospname != NULL) + OBD_FREE(ospname, MTI_NAME_MAXLEN); + RETURN(rc); +} + CFS_DEFINE_MUTEX(server_start_lock); /* Stop MDS/OSS if nobody is using them */ @@ -872,34 +1475,6 @@ int server_mti_print(char *title, struct mgs_target_info *mti) return(0); } -/** Get the fsname ("lustre") from the server name ("lustre-OST003F"). - * @param [in] svname server name including type and index - * @param [out] fsname Buffer to copy filesystem name prefix into. - * Must have at least 'strlen(fsname) + 1' chars. - * @param [out] endptr if endptr isn't NULL it is set to end of fsname - * rc < 0 on error - */ -static int server_name2fsname(char *svname, char *fsname, char **endptr) -{ - char *p; - - p = strstr(svname, "-OST"); - if (p == NULL) - p = strstr(svname, "-MDT"); - if (p == NULL) - return -1; - - if (fsname) { - strncpy(fsname, svname, p - svname); - fsname[p - svname] = '\0'; - } - - if (endptr != NULL) - *endptr = p; - - return 0; -} - /** * Get service name (svname) from string * rc < 0 on error @@ -1129,6 +1704,66 @@ out: } +/** + * Start the osp(fsname-MDT0000-osp-OSTxxxx) for an OST target, + * which would be used to communicate with MDT0 for quota and FID. + **/ +static int lustre_start_osp(struct super_block *sb) +{ + struct lustre_sb_info *lsi = s2lsi(sb); + struct config_llog_instance *cfg = NULL; + struct obd_device *osp; + char *ospname = NULL; + char *logname = NULL; + char *tgt; + int rc; + ENTRY; + + LASSERT(IS_OST(lsi) || IS_MDT(lsi)); + OBD_ALLOC(ospname, MTI_NAME_MAXLEN); + OBD_ALLOC(logname, MTI_NAME_MAXLEN); + if (ospname == NULL || logname == NULL) + GOTO(cleanup, rc = -ENOMEM); + + rc = server_name2fsname(lsi->lsi_svname, ospname, &tgt); + if (rc != 0) { + CERROR("%s change fsname error: rc %d\n", + lsi->lsi_svname, rc); + GOTO(cleanup, rc); + } + sprintf(logname, "%s-client", ospname); + + rc = tgt_name2ospname(lsi->lsi_svname, ospname); + if (rc != 0) { + CERROR("%s change ospname error: rc %d\n", + lsi->lsi_svname, rc); + GOTO(cleanup, rc); + } + + osp = class_name2obd(ospname); + if (osp != NULL) + GOTO(cleanup, rc = 0); + + OBD_ALLOC_PTR(cfg); + if (cfg == NULL) + GOTO(cleanup, rc = -ENOMEM); + + cfg->cfg_callback = client_osp_config_process; + cfg->cfg_instance = sb; + + rc = lustre_process_log(sb, logname, cfg); + +cleanup: + if (ospname != NULL) + OBD_FREE(ospname, MTI_NAME_MAXLEN); + if (logname != NULL) + OBD_FREE(logname, MTI_NAME_MAXLEN); + if (cfg != NULL) + OBD_FREE_PTR(cfg); + + RETURN(rc); +} + /** Start server targets: MDTs and OSTs */ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt) @@ -1205,6 +1840,7 @@ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt) /* Start targets using the llog named for the target */ memset(&cfg, 0, sizeof(cfg)); + cfg.cfg_callback = class_config_llog_handler; rc = lustre_process_log(sb, lsi->lsi_svname, &cfg); if (rc) { CERROR("failed to start server %s: %d\n", @@ -1215,55 +1851,61 @@ static int server_start_targets(struct super_block *sb, struct vfsmount *mnt) GOTO(out_mgc, rc); } -out_mgc: - /* Release the mgc fs for others to use */ - if (lsi->lsi_srv_mnt) - server_mgc_clear_fs(lsi->lsi_mgc); + obd = class_name2obd(lsi->lsi_svname); + if (!obd) { + CERROR("no server named %s was started\n", lsi->lsi_svname); + GOTO(out_mgc, rc = -ENXIO); + } - if (!rc) { - obd = class_name2obd(lsi->lsi_svname); - if (!obd) { - CERROR("no server named %s was started\n", - lsi->lsi_svname); - RETURN(-ENXIO); - } + if (IS_OST(lsi) || IS_MDT(lsi)) { + rc = lustre_start_osp(sb); + if (rc) { + CERROR("%s: failed to start OSP: %d\n", + lsi->lsi_svname, rc); + GOTO(out_mgc, rc); + } + } - server_notify_target(sb, obd); + server_notify_target(sb, obd); - /* calculate recovery timeout, do it after lustre_process_log */ - server_calc_timeout(lsi, obd); + /* calculate recovery timeout, do it after lustre_process_log */ + server_calc_timeout(lsi, obd); - /* log has been fully processed */ - obd_notify(obd, NULL, OBD_NOTIFY_CONFIG, (void *)CONFIG_LOG); + /* log has been fully processed */ + obd_notify(obd, NULL, OBD_NOTIFY_CONFIG, (void *)CONFIG_LOG); - /* log has been fully processed, let clients connect */ - dev = obd->obd_lu_dev; - if (dev && dev->ld_ops->ldo_prepare) { - rc = lu_env_init(&env, dev->ld_type->ldt_ctx_tags); - if (rc == 0) { - struct lu_context session_ctx; + /* log has been fully processed, let clients connect */ + dev = obd->obd_lu_dev; + if (dev && dev->ld_ops->ldo_prepare) { + rc = lu_env_init(&env, dev->ld_type->ldt_ctx_tags); + if (rc == 0) { + struct lu_context session_ctx; - lu_context_init(&session_ctx, LCT_SESSION); - session_ctx.lc_thread = NULL; - lu_context_enter(&session_ctx); - env.le_ses = &session_ctx; + lu_context_init(&session_ctx, LCT_SESSION); + session_ctx.lc_thread = NULL; + lu_context_enter(&session_ctx); + env.le_ses = &session_ctx; - dev->ld_ops->ldo_prepare(&env, NULL, dev); + dev->ld_ops->ldo_prepare(&env, NULL, dev); - lu_env_fini(&env); - lu_context_exit(&session_ctx); - lu_context_fini(&session_ctx); - } + lu_env_fini(&env); + lu_context_exit(&session_ctx); + lu_context_fini(&session_ctx); } + } - /* abort recovery only on the complete stack: - * many devices can be involved */ - if ((lsi->lsi_lmd->lmd_flags & LMD_FLG_ABORT_RECOV) && - (OBP(obd, iocontrol))) { - obd_iocontrol(OBD_IOC_ABORT_RECOVERY, - obd->obd_self_export, 0, NULL, NULL); - } - } + /* abort recovery only on the complete stack: + * many devices can be involved */ + if ((lsi->lsi_lmd->lmd_flags & LMD_FLG_ABORT_RECOV) && + (OBP(obd, iocontrol))) { + obd_iocontrol(OBD_IOC_ABORT_RECOVERY, obd->obd_self_export, 0, + NULL, NULL); + } + +out_mgc: + /* Release the mgc fs for others to use */ + if (lsi->lsi_srv_mnt) + server_mgc_clear_fs(lsi->lsi_mgc); RETURN(rc); } @@ -1459,6 +2101,12 @@ static void server_put_super(struct super_block *sb) if (IS_MDT(lsi) && (lsi->lsi_lmd->lmd_flags & LMD_FLG_NOSVC)) snprintf(tmpname, tmpname_sz, "MGS"); + /* disconnect the osp-on-ost first to drain off the inflight request */ + if (IS_OST(lsi) || IS_MDT(lsi)) { + if (lustre_disconnect_osp(sb) < 0) + CERROR("%s: Fail to disconnect osp-on-ost!\n", tmpname); + } + /* Stop the target */ if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOSVC) && (IS_MDT(lsi) || IS_OST(lsi))) { @@ -1499,6 +2147,11 @@ static void server_put_super(struct super_block *sb) server_stop_mgs(sb); } + if (IS_OST(lsi) || IS_MDT(lsi)) { + if (lustre_stop_osp(sb) < 0) + CERROR("%s: Fail to stop osp-on-ost!\n", tmpname); + } + /* Clean the mgc and sb */ lustre_common_put_super(sb); diff --git a/lustre/osp/Makefile.in b/lustre/osp/Makefile.in index 788d5a4..e504d0f 100644 --- a/lustre/osp/Makefile.in +++ b/lustre/osp/Makefile.in @@ -1,5 +1,6 @@ MODULES := osp osp-objs := osp_dev.o osp_object.o osp_precreate.o osp_sync.o lproc_osp.o +osp-objs += osp_ost.o EXTRA_DIST = $(osp-objs:.o=.c) osp_internal.h diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c index 9c533a9..cf9ee4f 100644 --- a/lustre/osp/osp_dev.c +++ b/lustre/osp/osp_dev.c @@ -183,24 +183,23 @@ out: static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d) { - lu_object_put(env, &d->opd_last_used_file->do_lu); - d->opd_last_used_file = NULL; + if (d->opd_last_used_file != NULL) { + lu_object_put(env, &d->opd_last_used_file->do_lu); + d->opd_last_used_file = NULL; + } } -static int osp_shutdown(const struct lu_env *env, struct osp_device *d) +int osp_disconnect(struct osp_device *d) { - struct obd_import *imp; - int rc = 0; - ENTRY; - - /* release last_used file */ - osp_last_used_fini(env, d); + struct obd_import *imp; + int rc = 0; imp = d->opd_obd->u.cli.cl_import; /* Mark import deactivated now, so we don't try to reconnect if any * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't * fully deactivate the import, or that would drop all requests. */ + LASSERT(imp != NULL); cfs_spin_lock(&imp->imp_lock); imp->imp_deactive = 1; cfs_spin_unlock(&imp->imp_lock); @@ -219,6 +218,25 @@ static int osp_shutdown(const struct lu_env *env, struct osp_device *d) ptlrpc_invalidate_import(imp); + RETURN(rc); +} + +static int osp_shutdown(const struct lu_env *env, struct osp_device *d) +{ + int rc = 0; + ENTRY; + + if (is_osp_on_ost(d->opd_obd->obd_name)) { + rc = osp_disconnect(d); + RETURN(rc); + } + + LASSERT(env); + /* release last_used file */ + osp_last_used_fini(env, d); + + rc = osp_disconnect(d); + /* stop precreate thread */ osp_precreate_fini(d); @@ -239,7 +257,8 @@ static int osp_process_config(const struct lu_env *env, switch (lcfg->lcfg_command) { case LCFG_CLEANUP: - lu_dev_del_linkage(dev->ld_site, dev); + if (!is_osp_on_ost(d->opd_obd->obd_name)) + lu_dev_del_linkage(dev->ld_site, dev); rc = osp_shutdown(env, d); break; case LCFG_PARAM: @@ -343,7 +362,7 @@ static int osp_sync(const struct lu_env *env, struct dt_device *dev) RETURN(0); } -static const struct dt_device_operations osp_dt_ops = { +const struct dt_device_operations osp_dt_ops = { .dt_statfs = osp_statfs, .dt_sync = osp_sync, }; @@ -585,7 +604,10 @@ static struct lu_device *osp_device_alloc(const struct lu_env *env, l = osp2lu_dev(m); dt_device_init(&m->opd_dt_dev, t); - rc = osp_init0(env, m, t, lcfg); + if (is_osp_on_ost(lustre_cfg_string(lcfg, 0))) + rc = osp_init_for_ost(env, m, t, lcfg); + else + rc = osp_init0(env, m, t, lcfg); if (rc != 0) { osp_device_free(env, l); l = ERR_PTR(rc); @@ -603,8 +625,11 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, ENTRY; - LASSERT(m->opd_storage_exp); - obd_disconnect(m->opd_storage_exp); + if (m->opd_storage_exp) + obd_disconnect(m->opd_storage_exp); + + if (is_osp_on_ost(m->opd_obd->obd_name)) + osp_fini_for_ost(m); imp = m->opd_obd->u.cli.cl_import; @@ -661,6 +686,8 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, RETURN(rc); *exp = class_conn2export(&conn); + if (is_osp_on_ost(obd->obd_name)) + osp->opd_exp = *exp; /* Why should there ever be more than 1 connect? */ osp->opd_connects++; @@ -680,7 +707,8 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, OBD_CONNECT_OSS_CAPA | OBD_CONNECT_REQPORTAL | OBD_CONNECT_SKIP_ORPHAN | - OBD_CONNECT_VERSION; + OBD_CONNECT_VERSION | + OBD_CONNECT_FID; ocd->ocd_version = LUSTRE_VERSION_CODE; LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX); ocd->ocd_index = data->ocd_index; @@ -707,7 +735,6 @@ static int osp_obd_disconnect(struct obd_export *exp) struct obd_device *obd = exp->exp_obd; struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev); int rc; - ENTRY; /* Only disconnect the underlying layers on the final disconnect. */ @@ -715,9 +742,14 @@ static int osp_obd_disconnect(struct obd_export *exp) osp->opd_connects--; rc = class_disconnect(exp); + if (rc) { + CERROR("%s: class disconnect error: rc = %d\n", + obd->obd_name, rc); + RETURN(rc); + } /* destroy the device */ - if (rc == 0) + if (!is_osp_on_ost(obd->obd_name)) class_manual_cleanup(obd); RETURN(rc); @@ -798,12 +830,16 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, case IMP_EVENT_DISCON: d->opd_got_disconnected = 1; d->opd_imp_connected = 0; + if (is_osp_on_ost(d->opd_obd->obd_name)) + break; osp_pre_update_status(d, -ENODEV); cfs_waitq_signal(&d->opd_pre_waitq); CDEBUG(D_HA, "got disconnected\n"); break; case IMP_EVENT_INACTIVE: d->opd_imp_active = 0; + if (is_osp_on_ost(d->opd_obd->obd_name)) + break; osp_pre_update_status(d, -ENODEV); cfs_waitq_signal(&d->opd_pre_waitq); CDEBUG(D_HA, "got inactive\n"); @@ -814,10 +850,19 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, d->opd_new_connection = 1; d->opd_imp_connected = 1; d->opd_imp_seen_connected = 1; + if (is_osp_on_ost(d->opd_obd->obd_name)) + break; cfs_waitq_signal(&d->opd_pre_waitq); __osp_sync_check_for_work(d); CDEBUG(D_HA, "got connected\n"); break; + case IMP_EVENT_INVALIDATE: + if (obd->obd_namespace == NULL) + break; + ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY); + break; + case IMP_EVENT_OCD: + break; default: CERROR("%s: unsupported import event: %#x\n", obd->obd_name, event); diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 7a82bca..5f4b77c 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -42,6 +42,7 @@ #define _OSP_INTERNAL_H #include +#include #include #include @@ -182,6 +183,7 @@ struct osp_object { }; extern struct lu_object_operations osp_lu_obj_ops; +extern const struct dt_device_operations osp_dt_ops; struct osp_thread_info { struct lu_buf osi_lb; @@ -314,4 +316,10 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d); int osp_sync_fini(struct osp_device *d); void __osp_sync_check_for_work(struct osp_device *d); +/* osp_ost.c */ +int osp_init_for_ost(const struct lu_env *env, struct osp_device *m, + struct lu_device_type *ldt, struct lustre_cfg *cfg); +int osp_disconnect(struct osp_device *d); +int osp_fini_for_ost(struct osp_device *osp); + #endif diff --git a/lustre/osp/osp_ost.c b/lustre/osp/osp_ost.c new file mode 100644 index 0000000..2fb6a55 --- /dev/null +++ b/lustre/osp/osp_ost.c @@ -0,0 +1,175 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA + * + * GPL HEADER END + */ +/* + * Copyright (c) 2012 Intel, Inc. + * Use is subject to license terms. + * + * lustre/osp/osp_ost.c + * + * OSP on OST for communicating with MDT0 + * + * Author: + */ +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif +#define DEBUG_SUBSYSTEM S_OST + +#include +#include +#include + +#include "osp_internal.h" + +static int osp_name2fsname(char *ospname, char *fsname) +{ + char *ptr; + + LASSERT(ospname != NULL); + LASSERT(fsname != NULL); + if (!is_osp_on_ost(ospname)) + return -EINVAL; + + sprintf(fsname, "-%s-", LUSTRE_OSP_NAME); + + ptr = strstr(ospname, fsname); + if (ptr) { + strncpy(fsname, ospname, ptr - ospname); + fsname[ptr - ospname] = '\0'; + } + return 0; +} + +static int osp_setup_for_ost(const struct lu_env *env, struct osp_device *osp, + char *nidstring) +{ + struct lustre_cfg_bufs *bufs = NULL; + struct lustre_cfg *lcfg = NULL; + char *ospname = osp->opd_obd->obd_name; + char *fsname = NULL; + char *server_uuid = NULL; + class_uuid_t uuid; + struct obd_import *imp; + int rc; + ENTRY; + + OBD_ALLOC_PTR(bufs); + if (bufs == NULL) + RETURN(-ENOMEM); + + OBD_ALLOC(fsname, strlen(ospname)); + if (fsname == NULL) + GOTO(out, rc = -ENOMEM); + + rc = osp_name2fsname(ospname, fsname); + if (rc) { + CERROR("%s: name change error: rc %d\n", ospname, rc); + GOTO(out, rc); + } + + OBD_ALLOC(server_uuid, strlen(fsname) + 15); + if (server_uuid == NULL) + GOTO(out, rc = -ENOMEM); + + sprintf(server_uuid, "%s-MDT0000_UUID", fsname); + lustre_cfg_bufs_reset(bufs, ospname); + lustre_cfg_bufs_set_string(bufs, 1, server_uuid); + lustre_cfg_bufs_set_string(bufs, 2, nidstring); + lcfg = lustre_cfg_new(LCFG_SETUP, bufs); + if (lcfg == NULL) + GOTO(out, rc = -ENOMEM); + + rc = client_obd_setup(osp->opd_obd, lcfg); + if (rc != 0) { + CERROR("%s: client obd setup error: rc = %d\n", + osp->opd_obd->obd_name, rc); + GOTO(out, rc); + } + + imp = osp->opd_obd->u.cli.cl_import; + rc = ptlrpc_init_import(imp); + if (rc) + GOTO(out, rc); + + ll_generate_random_uuid(uuid); + class_uuid_unparse(uuid, &osp->opd_cluuid); +out: + if (bufs != NULL) + OBD_FREE_PTR(bufs); + if (server_uuid != NULL) + OBD_FREE(server_uuid, strlen(fsname) + 15); + if (fsname != NULL) + OBD_FREE(fsname, strlen(ospname)); + if (lcfg != NULL) + lustre_cfg_free(lcfg); + if (rc) + client_obd_cleanup(osp->opd_obd); + RETURN(rc); +} + +int osp_fini_for_ost(struct osp_device *osp) +{ + if (osp->opd_exp != NULL) + class_disconnect(osp->opd_exp); + return 0; +} + +int osp_init_for_ost(const struct lu_env *env, struct osp_device *osp, + struct lu_device_type *ldt, struct lustre_cfg *cfg) +{ + struct lprocfs_static_vars lvars = { 0 }; + int rc; + ENTRY; + + osp->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0)); + if (osp->opd_obd == NULL) { + CERROR("Cannot find obd with name %s\n", + lustre_cfg_string(cfg, 0)); + RETURN(-ENODEV); + } + + osp->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops; + osp->opd_dt_dev.dd_ops = &osp_dt_ops; + osp->opd_obd->obd_lu_dev = &osp->opd_dt_dev.dd_lu_dev; + + rc = ptlrpcd_addref(); + if (rc) { + CERROR("%s: ptlrpcd addref error: rc =%d\n", + osp->opd_obd->obd_name, rc); + RETURN(rc); + } + + rc = osp_setup_for_ost(env, osp, lustre_cfg_string(cfg, 1)); + if (rc) { + CERROR("%s: osp_setup_for_ost error: rc =%d\n", + osp->opd_obd->obd_name, rc); + ptlrpcd_decref(); + RETURN(rc); + } + + lprocfs_osp_init_vars(&lvars); + if (lprocfs_obd_setup(osp->opd_obd, lvars.obd_vars) == 0) + ptlrpc_lprocfs_register_obd(osp->opd_obd); + + RETURN(0); +} diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index f013315..9afb3ea 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -480,6 +480,7 @@ load_modules_local() { else load_module obdfilter/obdfilter fi + load_module osp/osp fi -- 1.8.3.1