From: ericm Date: Mon, 24 Nov 2008 22:53:46 +0000 (+0000) Subject: branch: HEAD X-Git-Tag: v1_9_120~80 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=9edebe083264464d2692da446f68483e3e45a80d branch: HEAD rewrite part of sptlrpc configuration system to be able to interoperate with 1.8. b=15363 r=nathan r=fanyong --- diff --git a/lustre/include/lustre_sec.h b/lustre/include/lustre_sec.h index 24d2a41..f391e35 100644 --- a/lustre/include/lustre_sec.h +++ b/lustre/include/lustre_sec.h @@ -192,10 +192,14 @@ enum lustre_sec_part { LUSTRE_SP_CLI = 0, LUSTRE_SP_MDT, LUSTRE_SP_OST, + LUSTRE_SP_MGC, LUSTRE_SP_MGS, LUSTRE_SP_ANY = 0xFF }; +const char *sptlrpc_part2name(enum lustre_sec_part sp); +enum lustre_sec_part sptlrpc_target_sec_part(struct obd_device *obd); + struct sptlrpc_rule { __u32 sr_netid; /* LNET network ID */ __u8 sr_from; /* sec_part */ @@ -210,18 +214,6 @@ struct sptlrpc_rule_set { struct sptlrpc_rule *srs_rules; }; -#define SPTLRPC_CONF_LOG_MAX (64) - -struct sptlrpc_conf_log { - __u32 scl_max; /* maximum rules # */ - __u32 scl_nrule; /* rules # */ - __u8 scl_part; /* which part am i */ - __u8 scl_pad0; - __u16 scl_pad1; - __u32 scl_pad2; - struct sptlrpc_rule scl_rules[SPTLRPC_CONF_LOG_MAX]; -}; - static inline void sptlrpc_rule_set_init(struct sptlrpc_rule_set *set) { memset(set, 0, sizeof(*set)); @@ -232,31 +224,21 @@ int sptlrpc_rule_set_expand(struct sptlrpc_rule_set *set, int expand); int sptlrpc_rule_set_merge(struct sptlrpc_rule_set *set, struct sptlrpc_rule *rule, int expand); -int sptlrpc_rule_set_from_log(struct sptlrpc_rule_set *rset, - struct sptlrpc_conf_log *log); -void sptlrpc_rule_set_choose(struct sptlrpc_rule_set *rset, - enum lustre_sec_part from, - lnet_nid_t nid, - struct sptlrpc_flavor *flavor); void sptlrpc_rule_set_dump(struct sptlrpc_rule_set *set); -struct sptlrpc_conf_log *sptlrpc_conf_log_alloc(void); -void sptlrpc_conf_log_free(struct sptlrpc_conf_log *log); -int sptlrpc_conf_log_populate(struct sptlrpc_rule_set *gen, - struct sptlrpc_rule_set *tgt, - enum lustre_sec_part from, - enum lustre_sec_part to, - unsigned int fl_udesc, - struct sptlrpc_conf_log *log); -struct sptlrpc_conf_log *sptlrpc_conf_log_extract(struct lustre_cfg *lcfg); -void sptlrpc_conf_log_cleanup(struct sptlrpc_conf_log *log); -void sptlrpc_conf_log_dump(struct sptlrpc_conf_log *log); - -const char *sptlrpc_part2name(enum lustre_sec_part part); -enum lustre_sec_part sptlrpc_target_sec_part(struct obd_device *obd); - -int sptlrpc_cliobd_process_config(struct obd_device *obd, - struct lustre_cfg *lcfg); +int sptlrpc_process_config(struct lustre_cfg *lcfg); +void sptlrpc_conf_log_start(const char *logname); +void sptlrpc_conf_log_stop(const char *logname); +void sptlrpc_conf_log_update_begin(const char *logname); +void sptlrpc_conf_log_update_end(const char *logname); +void sptlrpc_conf_client_adapt(struct obd_device *obd); +int sptlrpc_conf_target_get_rules(struct obd_device *obd, + struct sptlrpc_rule_set *rset, + int initial); +void sptlrpc_target_choose_flavor(struct sptlrpc_rule_set *rset, + enum lustre_sec_part from, + lnet_nid_t nid, + struct sptlrpc_flavor *flavor); /* The maximum length of security payload. 1024 is enough for Kerberos 5, * and should be enough for other future mechanisms but not sure. diff --git a/lustre/include/obd.h b/lustre/include/obd.h index d3fc0e8..86f8659 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -389,9 +389,8 @@ struct client_obd { int cl_max_mds_easize; int cl_max_mds_cookiesize; - /* security configuration */ - struct sptlrpc_rule_set cl_sptlrpc_rset; - enum lustre_sec_part cl_sec_part; + enum lustre_sec_part cl_sp_me; + enum lustre_sec_part cl_sp_to; //struct llog_canceld_ctxt *cl_llcd; /* it's included by obd_llog_ctxt */ void *cl_llcd_offset; @@ -693,6 +692,7 @@ struct lov_obd { lustre_hash_t *lov_pools_hash_body; /* used for key access */ struct list_head lov_pool_list; /* used for sequential access */ cfs_proc_dir_entry_t *lov_pool_proc_entry; + enum lustre_sec_part lov_sp_me; }; struct lmv_tgt_desc { @@ -1108,6 +1108,7 @@ enum obd_cleanup_stage { #define KEY_BLOCKSIZE "blocksize" #define KEY_BLOCKSIZE_BITS "blocksize_bits" #define KEY_FIEMAP "FIEMAP" +#define KEY_SPTLRPC_CONF "sptlrpc_conf" /* XXX unused ?*/ #define KEY_INTERMDS "inter_mds" #define KEY_ASYNC "async" diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 385fbd9..1bc75e1 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -90,6 +90,8 @@ struct obd_device * class_devices_in_group(struct obd_uuid *grp_uuid, int *next); struct obd_device * class_num2obd(int num); +int class_notify_sptlrpc_conf(const char *fsname, int namelen); + char *obd_export_nid2str(struct obd_export *exp); int obd_export_evict_by_nid(struct obd_device *obd, const char *nid); @@ -132,6 +134,7 @@ static inline void lprocfs_echo_init_vars(struct lprocfs_static_vars *lvars) /* Passed as data param to class_config_parse_llog */ struct config_llog_instance { char * cfg_instance; + char * cfg_obdname; struct super_block *cfg_sb; struct obd_uuid cfg_uuid; int cfg_last_idx; /* for partial llog processing */ @@ -149,9 +152,11 @@ struct config_llog_data { struct config_llog_instance cld_cfg; struct list_head cld_list_chain; atomic_t cld_refcount; + struct config_llog_data *cld_sptlrpc;/* depended sptlrpc log */ struct obd_export *cld_mgcexp; unsigned int cld_stopping:1, /* we were told to stop watching */ - cld_lostlock:1; /* lock not requeued */ + cld_lostlock:1, /* lock not requeued */ + cld_is_sptlrpc:1; }; struct lustre_profile { diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 3cd7bda..7f65388 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -221,14 +221,20 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) rq_portal = OST_REQUEST_PORTAL; rp_portal = OSC_REPLY_PORTAL; connect_op = OST_CONNECT; + cli->cl_sp_me = LUSTRE_SP_CLI; + cli->cl_sp_to = LUSTRE_SP_OST; } else if (!strcmp(name, LUSTRE_MDC_NAME)) { rq_portal = MDS_REQUEST_PORTAL; rp_portal = MDC_REPLY_PORTAL; connect_op = MDS_CONNECT; + cli->cl_sp_me = LUSTRE_SP_CLI; + cli->cl_sp_to = LUSTRE_SP_MDT; } else if (!strcmp(name, LUSTRE_MGC_NAME)) { rq_portal = MGS_REQUEST_PORTAL; rp_portal = MGC_REPLY_PORTAL; connect_op = MGS_CONNECT; + cli->cl_sp_me = LUSTRE_SP_MGC; + cli->cl_sp_to = LUSTRE_SP_MGS; } else { CERROR("unknown client OBD type \"%s\", can't setup\n", name); @@ -257,8 +263,6 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) init_rwsem(&cli->cl_sem); sema_init(&cli->cl_mgc_sem, 1); - sptlrpc_rule_set_init(&cli->cl_sptlrpc_rset); - cli->cl_sec_part = LUSTRE_SP_ANY; cli->cl_conn_count = 0; memcpy(server_uuid.uuid, lustre_cfg_buf(lcfg, 2), min_t(unsigned int, LUSTRE_CFG_BUFLEN(lcfg, 2), @@ -374,7 +378,6 @@ err: int client_obd_cleanup(struct obd_device *obddev) { ENTRY; - sptlrpc_rule_set_free(&obddev->u.cli.cl_sptlrpc_rset); ldlm_put_ref(); RETURN(0); } diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index ca0f68c..eb32c62 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -144,6 +144,9 @@ int lov_connect_obd(struct obd_device *obd, __u32 index, int activate, RETURN(-EINVAL); } + /* override the sp_me from lov */ + tgt_obd->u.cli.cl_sp_me = lov->lov_sp_me; + if (data && (data->ocd_connect_flags & OBD_CONNECT_INDEX)) data->ocd_index = index; @@ -801,6 +804,7 @@ int lov_setup(struct obd_device *obd, struct lustre_cfg *lcfg) atomic_set(&lov->lov_refcount, 0); CFS_INIT_LIST_HEAD(&lov->lov_qos.lq_oss_list); init_rwsem(&lov->lov_qos.lq_rw_sem); + lov->lov_sp_me = LUSTRE_SP_CLI; lov->lov_qos.lq_dirty = 1; lov->lov_qos.lq_rr.lqr_dirty = 1; lov->lov_qos.lq_reset = 1; diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 0ca79b9..ff4cc21 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1175,6 +1175,10 @@ int mdc_set_info_async(struct obd_export *exp, rc = do_set_info_async(exp, keylen, key, vallen, val, set); RETURN(rc); } + if (KEY_IS(KEY_SPTLRPC_CONF)) { + sptlrpc_conf_client_adapt(exp->exp_obd); + RETURN(0); + } if (KEY_IS(KEY_FLUSH_CTX)) { sptlrpc_import_flush_my_ctx(imp); RETURN(0); @@ -1687,9 +1691,6 @@ static int mdc_process_config(struct obd_device *obd, obd_count len, void *buf) lprocfs_mdc_init_vars(&lvars); switch (lcfg->lcfg_command) { - case LCFG_SPTLRPC_CONF: - rc = sptlrpc_cliobd_process_config(obd, lcfg); - break; default: rc = class_process_proc_param(PARAM_MDC, lvars.obd_vars, lcfg, obd); diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 0b4120b..4c1ede1 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -633,6 +633,8 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) GOTO(err_exit, rc); } + mds->mds_osc_obd->u.lov.lov_sp_me = LUSTRE_SP_MDT; + OBD_ALLOC(data, sizeof(*data)); if (data == NULL) RETURN(-ENOMEM); diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 6e582c1..061e22c 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -72,6 +72,7 @@ #endif #include #include +#include mdl_mode_t mdt_mdl_lock_modes[] = { [LCK_MINMODE] = MDL_MINMODE, @@ -4097,6 +4098,54 @@ out: return rc; } +/** + * setup CONFIG_ORIG context, used to access local config log. + * this may need to be rewrite as part of llog rewrite for lu-api. + */ +static int mdt_obd_llog_setup(struct obd_device *obd, + struct lustre_sb_info *lsi) +{ + int rc; + + LASSERT(obd->obd_fsops == NULL); + + obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd)); + if (IS_ERR(obd->obd_fsops)) + return (int) PTR_ERR(obd->obd_fsops); + + rc = fsfilt_setup(obd, lsi->lsi_srv_mnt->mnt_sb); + if (rc) { + fsfilt_put_ops(obd->obd_fsops); + return rc; + } + + OBD_SET_CTXT_MAGIC(&obd->obd_lvfs_ctxt); + obd->obd_lvfs_ctxt.pwdmnt = lsi->lsi_srv_mnt; + obd->obd_lvfs_ctxt.pwd = lsi->lsi_srv_mnt->mnt_root; + obd->obd_lvfs_ctxt.fs = get_ds(); + + rc = llog_setup(obd, &obd->obd_olg, LLOG_CONFIG_ORIG_CTXT, obd, + 0, NULL, &llog_lvfs_ops); + if (rc) { + CERROR("llog setup failed: %d\n", rc); + fsfilt_put_ops(obd->obd_fsops); + } + + return rc; +} + +static void mdt_obd_llog_cleanup(struct obd_device *obd) +{ + struct llog_ctxt *ctxt; + + ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); + if (ctxt) + llog_cleanup(ctxt); + + if (obd->obd_fsops) + fsfilt_put_ops(obd->obd_fsops); +} + static void mdt_fini(const struct lu_env *env, struct mdt_device *m) { struct md_device *next = m->mdt_child; @@ -4131,6 +4180,7 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) target_recovery_fini(obd); mdt_stop_ptlrpc_service(m); + mdt_obd_llog_cleanup(obd); obd_zombie_barrier(); #ifdef HAVE_QUOTA_SUPPORT next->md_ops->mdo_quota.mqo_cleanup(env, next); @@ -4176,6 +4226,30 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) EXIT; } +static int mdt_adapt_sptlrpc_conf(struct obd_device *obd, int initial) +{ + struct mdt_device *m = mdt_dev(obd->obd_lu_dev); + struct sptlrpc_rule_set tmp_rset; + int rc; + + sptlrpc_rule_set_init(&tmp_rset); + rc = sptlrpc_conf_target_get_rules(obd, &tmp_rset, initial); + if (rc) { + CERROR("mdt %s: failed get sptlrpc rules: %d\n", + obd->obd_name, rc); + return rc; + } + + sptlrpc_target_update_exp_flavor(obd, &tmp_rset); + + write_lock(&m->mdt_sptlrpc_lock); + sptlrpc_rule_set_free(&m->mdt_sptlrpc_rset); + m->mdt_sptlrpc_rset = tmp_rset; + write_unlock(&m->mdt_sptlrpc_lock); + + return 0; +} + static void fsoptions_to_mdt_flags(struct mdt_device *m, char *options) { char *p = options; @@ -4404,11 +4478,17 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, if (rc) GOTO(err_capa, rc); + rc = mdt_obd_llog_setup(obd, lsi); + if (rc) + GOTO(err_fs_cleanup, rc); + + mdt_adapt_sptlrpc_conf(obd, 1); + #ifdef HAVE_QUOTA_SUPPORT next = m->mdt_child; rc = next->md_ops->mdo_quota.mqo_setup(env, next, lmi->lmi_mnt); if (rc) - GOTO(err_fs_cleanup, rc); + GOTO(err_llog_cleanup, rc); #endif server_put_mount_2(dev, lmi->lmi_mnt); @@ -4446,8 +4526,10 @@ err_recovery: target_recovery_fini(obd); #ifdef HAVE_QUOTA_SUPPORT next->md_ops->mdo_quota.mqo_cleanup(env, next); -err_fs_cleanup: +err_llog_cleanup: #endif + mdt_obd_llog_cleanup(obd); +err_fs_cleanup: mdt_fs_cleanup(env, m); err_capa: cfs_timer_disarm(&m->mdt_ck_timer); @@ -4490,34 +4572,6 @@ static int mdt_process_config(const struct lu_env *env, ENTRY; switch (cfg->lcfg_command) { - case LCFG_SPTLRPC_CONF: { - struct sptlrpc_conf_log *log; - struct sptlrpc_rule_set tmp_rset; - - log = sptlrpc_conf_log_extract(cfg); - if (IS_ERR(log)) { - rc = PTR_ERR(log); - break; - } - - sptlrpc_rule_set_init(&tmp_rset); - - rc = sptlrpc_rule_set_from_log(&tmp_rset, log); - if (rc) { - CERROR("mdt %p: failed get sptlrpc rules: %d\n", m, rc); - break; - } - - write_lock(&m->mdt_sptlrpc_lock); - sptlrpc_rule_set_free(&m->mdt_sptlrpc_rset); - m->mdt_sptlrpc_rset = tmp_rset; - write_unlock(&m->mdt_sptlrpc_lock); - - sptlrpc_target_update_exp_flavor( - md2lu_dev(&m->mdt_md_dev)->ld_obd, &tmp_rset); - - break; - } case LCFG_PARAM: { struct lprocfs_static_vars lvars; struct obd_device *obd = d->ld_obd; @@ -4634,6 +4688,25 @@ static const struct lu_object_operations mdt_obj_ops = { .loo_object_free = mdt_object_free }; +static int mdt_obd_set_info_async(struct obd_export *exp, + __u32 keylen, void *key, + __u32 vallen, void *val, + struct ptlrpc_request_set *set) +{ + struct obd_device *obd = exp->exp_obd; + int rc; + ENTRY; + + LASSERT(obd); + + if (KEY_IS(KEY_SPTLRPC_CONF)) { + rc = mdt_adapt_sptlrpc_conf(obd, 0); + RETURN(rc); + } + + RETURN(0); +} + /* mds_connect_internal */ static int mdt_connect_internal(struct obd_export *exp, struct mdt_device *mdt, @@ -4681,6 +4754,49 @@ static int mdt_connect_internal(struct obd_export *exp, return 0; } +static int mdt_connect_check_sptlrpc(struct mdt_device *mdt, + struct obd_export *exp, + struct ptlrpc_request *req) +{ + struct sptlrpc_flavor flvr; + int rc = 0; + + if (exp->exp_flvr.sf_rpc == SPTLRPC_FLVR_INVALID) { + read_lock(&mdt->mdt_sptlrpc_lock); + sptlrpc_target_choose_flavor(&mdt->mdt_sptlrpc_rset, + req->rq_sp_from, + req->rq_peer.nid, + &flvr); + read_unlock(&mdt->mdt_sptlrpc_lock); + + spin_lock(&exp->exp_lock); + + exp->exp_sp_peer = req->rq_sp_from; + exp->exp_flvr = flvr; + + if (exp->exp_flvr.sf_rpc != req->rq_flvr.sf_rpc) { + CERROR("unauthorized rpc flavor %x from %s, " + "expect %x\n", req->rq_flvr.sf_rpc, + libcfs_nid2str(req->rq_peer.nid), + exp->exp_flvr.sf_rpc); + rc = -EACCES; + } + + spin_unlock(&exp->exp_lock); + } else { + if (exp->exp_sp_peer != req->rq_sp_from) { + CERROR("RPC source %s doesn't match %s\n", + sptlrpc_part2name(req->rq_sp_from), + sptlrpc_part2name(exp->exp_sp_peer)); + rc = -EACCES; + } else { + rc = sptlrpc_target_export_check(exp, req); + } + } + + return rc; +} + /* mds_connect copy */ static int mdt_obd_connect(const struct lu_env *env, struct lustre_handle *conn, struct obd_device *obd, @@ -4711,25 +4827,9 @@ static int mdt_obd_connect(const struct lu_env *env, exp = class_conn2export(conn); LASSERT(exp != NULL); - CDEBUG(D_SEC, "from %s\n", sptlrpc_part2name(req->rq_sp_from)); - - spin_lock(&exp->exp_lock); - exp->exp_sp_peer = req->rq_sp_from; - - read_lock(&mdt->mdt_sptlrpc_lock); - sptlrpc_rule_set_choose(&mdt->mdt_sptlrpc_rset, exp->exp_sp_peer, - req->rq_peer.nid, &exp->exp_flvr); - read_unlock(&mdt->mdt_sptlrpc_lock); - - if (exp->exp_flvr.sf_rpc != req->rq_flvr.sf_rpc) { - CERROR("invalid rpc flavor %x, expect %x, from %s\n", - req->rq_flvr.sf_rpc, exp->exp_flvr.sf_rpc, - libcfs_nid2str(req->rq_peer.nid)); - exp->exp_flvr.sf_rpc = SPTLRPC_FLVR_INVALID; - spin_unlock(&exp->exp_lock); - RETURN(-EACCES); - } - spin_unlock(&exp->exp_lock); + rc = mdt_connect_check_sptlrpc(mdt, exp, req); + if (rc) + GOTO(out, rc); rc = mdt_connect_internal(exp, mdt, data); if (rc == 0) { @@ -4753,6 +4853,7 @@ static int mdt_obd_connect(const struct lu_env *env, rc = -ENOMEM; } +out: if (rc != 0) class_disconnect(exp); else @@ -4780,28 +4881,9 @@ static int mdt_obd_reconnect(const struct lu_env *env, req = info->mti_pill->rc_req; mdt = mdt_dev(obd->obd_lu_dev); - CDEBUG(D_SEC, "from %s\n", sptlrpc_part2name(req->rq_sp_from)); - - spin_lock(&exp->exp_lock); - if (exp->exp_flvr.sf_rpc == SPTLRPC_FLVR_INVALID) { - exp->exp_sp_peer = req->rq_sp_from; - - read_lock(&mdt->mdt_sptlrpc_lock); - sptlrpc_rule_set_choose(&mdt->mdt_sptlrpc_rset, - exp->exp_sp_peer, - req->rq_peer.nid, &exp->exp_flvr); - read_unlock(&mdt->mdt_sptlrpc_lock); - - if (exp->exp_flvr.sf_rpc != req->rq_flvr.sf_rpc) { - CERROR("invalid rpc flavor %x, expect %x, from %s\n", - req->rq_flvr.sf_rpc, exp->exp_flvr.sf_rpc, - libcfs_nid2str(req->rq_peer.nid)); - exp->exp_flvr.sf_rpc = SPTLRPC_FLVR_INVALID; - spin_unlock(&exp->exp_lock); - RETURN(-EACCES); - } - } - spin_unlock(&exp->exp_lock); + rc = mdt_connect_check_sptlrpc(mdt, exp, req); + if (rc) + RETURN(rc); rc = mdt_connect_internal(exp, mdt_dev(obd->obd_lu_dev), data); if (rc == 0) @@ -5112,6 +5194,7 @@ int mdt_obd_postrecov(struct obd_device *obd) static struct obd_ops mdt_obd_device_ops = { .o_owner = THIS_MODULE, + .o_set_info_async = mdt_obd_set_info_async, .o_connect = mdt_obd_connect, .o_reconnect = mdt_obd_reconnect, .o_disconnect = mdt_obd_disconnect, diff --git a/lustre/mgc/mgc_internal.h b/lustre/mgc/mgc_internal.h index d76bc67..cb74025 100644 --- a/lustre/mgc/mgc_internal.h +++ b/lustre/mgc/mgc_internal.h @@ -53,4 +53,6 @@ static void lprocfs_mgc_init_vars(struct lprocfs_static_vars *lvars) } #endif /* LPROCFS */ +int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld); + #endif /* _MGC_INTERNAL_H */ diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index 37cfda5..2ab2ef3 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -126,28 +126,42 @@ static int config_log_get(struct config_llog_data *cld) static void config_log_put(struct config_llog_data *cld) { ENTRY; + CDEBUG(D_INFO, "log %s refs %d\n", cld->cld_logname, atomic_read(&cld->cld_refcount)); - if (atomic_dec_and_test(&cld->cld_refcount)) { - CDEBUG(D_MGC, "dropping config log %s\n", cld->cld_logname); - class_export_put(cld->cld_mgcexp); - spin_lock(&config_list_lock); + LASSERT(atomic_read(&cld->cld_refcount) > 0); + + /* spinlock to make sure no item with 0 refcount in the list */ + spin_lock(&config_list_lock); + if (unlikely(atomic_dec_and_test(&cld->cld_refcount))) { list_del(&cld->cld_list_chain); spin_unlock(&config_list_lock); + + CDEBUG(D_MGC, "dropping config log %s\n", cld->cld_logname); + + if (cld->cld_sptlrpc) + config_log_put(cld->cld_sptlrpc); + if (cld->cld_is_sptlrpc) + sptlrpc_conf_log_stop(cld->cld_logname); + + class_export_put(cld->cld_mgcexp); OBD_FREE(cld->cld_logname, strlen(cld->cld_logname) + 1); if (cld->cld_cfg.cfg_instance != NULL) OBD_FREE(cld->cld_cfg.cfg_instance, strlen(cld->cld_cfg.cfg_instance) + 1); OBD_FREE(cld, sizeof(*cld)); + } else { + spin_unlock(&config_list_lock); } + EXIT; } /* Find a config log by name */ -static struct config_llog_data *config_log_find(char *logname, - struct config_llog_instance *cfg) +static +struct config_llog_data *config_log_find(char *logname, + struct config_llog_instance *cfg) { - struct list_head *tmp; struct config_llog_data *cld; char *logid = logname; int match_instance = 0; @@ -163,8 +177,7 @@ static struct config_llog_data *config_log_find(char *logname, } spin_lock(&config_list_lock); - list_for_each(tmp, &config_llog_list) { - cld = list_entry(tmp, struct config_llog_data, cld_list_chain); + list_for_each_entry(cld, &config_llog_list, cld_list_chain) { if (match_instance && cld->cld_cfg.cfg_instance && strcmp(logid, cld->cld_cfg.cfg_instance) == 0) goto out_found; @@ -179,56 +192,123 @@ static struct config_llog_data *config_log_find(char *logname, out_found: atomic_inc(&cld->cld_refcount); spin_unlock(&config_list_lock); + LASSERT(cld->cld_stopping == 0 || cld->cld_is_sptlrpc == 0); RETURN(cld); } -/* Add this log to our list of active logs. - We have one active log per "mount" - client instance or servername. - Each instance may be at a different point in the log. */ -static int config_log_add(char *logname, struct config_llog_instance *cfg, - struct super_block *sb) +static +struct config_llog_data *do_config_log_add(struct obd_device *obd, + char *logname, + unsigned int is_sptlrpc, + struct config_llog_instance *cfg, + struct super_block *sb) { struct config_llog_data *cld; - struct lustre_sb_info *lsi = s2lsi(sb); - int rc; + int rc; ENTRY; - CDEBUG(D_MGC, "adding config log %s:%s\n", logname, cfg->cfg_instance); + CDEBUG(D_MGC, "do adding config log %s:%s\n", logname, + cfg ? cfg->cfg_instance : "NULL"); OBD_ALLOC(cld, sizeof(*cld)); if (!cld) - RETURN(-ENOMEM); + RETURN(ERR_PTR(-ENOMEM)); OBD_ALLOC(cld->cld_logname, strlen(logname) + 1); if (!cld->cld_logname) { OBD_FREE(cld, sizeof(*cld)); - RETURN(-ENOMEM); + RETURN(ERR_PTR(-ENOMEM)); } strcpy(cld->cld_logname, logname); - cld->cld_cfg = *cfg; + if (cfg) + cld->cld_cfg = *cfg; cld->cld_cfg.cfg_last_idx = 0; cld->cld_cfg.cfg_flags = 0; cld->cld_cfg.cfg_sb = sb; + cld->cld_is_sptlrpc = is_sptlrpc; atomic_set(&cld->cld_refcount, 1); /* Keep the mgc around until we are done */ - cld->cld_mgcexp = class_export_get(lsi->lsi_mgc->obd_self_export); + cld->cld_mgcexp = class_export_get(obd->obd_self_export); - if (cfg->cfg_instance != NULL) { + if (cfg && cfg->cfg_instance != NULL) { OBD_ALLOC(cld->cld_cfg.cfg_instance, strlen(cfg->cfg_instance) + 1); strcpy(cld->cld_cfg.cfg_instance, cfg->cfg_instance); } + + if (is_sptlrpc) { + sptlrpc_conf_log_start(logname); + cld->cld_cfg.cfg_obdname = obd->obd_name; + } + rc = mgc_logname2resid(logname, &cld->cld_resid); + spin_lock(&config_list_lock); list_add(&cld->cld_list_chain, &config_llog_list); spin_unlock(&config_list_lock); if (rc) { config_log_put(cld); - RETURN(rc); + RETURN(ERR_PTR(rc)); } - RETURN(rc); + if (is_sptlrpc) { + rc = mgc_process_log(obd, cld); + if (rc) + CERROR("failed processing sptlrpc log: %d\n", rc); + } + + RETURN(cld); +} + +/** + * Add this log to our list of active logs. + * We have one active log per "mount" - client instance or servername. + * Each instance may be at a different point in the log. + */ +static int config_log_add(struct obd_device *obd, char *logname, + struct config_llog_instance *cfg, + struct super_block *sb) +{ + struct config_llog_data *cld, *sptlrpc_cld; + char seclogname[20]; + char *ptr; + ENTRY; + + CDEBUG(D_MGC, "adding config log %s:%s\n", logname, cfg->cfg_instance); + + /* + * for each regular log, the depended sptlrpc log name is + * -sptlrpc. multiple regular logs may share one sptlrpc log. + */ + ptr = strrchr(logname, '-'); + if (ptr == NULL || ptr - logname > 8) { + CERROR("logname %s is too long\n", logname); + RETURN(-EINVAL); + } + + memcpy(seclogname, logname, ptr - logname); + strcpy(seclogname + (ptr - logname), "-sptlrpc"); + + sptlrpc_cld = config_log_find(seclogname, NULL); + if (IS_ERR(sptlrpc_cld)) { + sptlrpc_cld = do_config_log_add(obd, seclogname, 1, NULL, NULL); + if (IS_ERR(sptlrpc_cld)) { + CERROR("can't create sptlrpc log: %s\n", seclogname); + RETURN(PTR_ERR(sptlrpc_cld)); + } + } + + cld = do_config_log_add(obd, logname, 0, cfg, sb); + if (IS_ERR(cld)) { + CERROR("can't create log: %s\n", logname); + config_log_put(sptlrpc_cld); + RETURN(PTR_ERR(cld)); + } + + cld->cld_sptlrpc = sptlrpc_cld; + + RETURN(0); } DECLARE_MUTEX(llog_process_lock); @@ -236,7 +316,7 @@ DECLARE_MUTEX(llog_process_lock); /* Stop watching for updates on this log. */ static int config_log_end(char *logname, struct config_llog_instance *cfg) { - struct config_llog_data *cld; + struct config_llog_data *cld, *cld_sptlrpc; int rc = 0; ENTRY; @@ -250,6 +330,14 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) cld->cld_stopping = 1; up(&llog_process_lock); + spin_lock(&config_list_lock); + cld_sptlrpc = cld->cld_sptlrpc; + cld->cld_sptlrpc = NULL; + spin_unlock(&config_list_lock); + + if (cld_sptlrpc) + config_log_put(cld_sptlrpc); + /* drop the start ref */ config_log_put(cld); CDEBUG(D_MGC, "end config log %s (%d)\n", logname ? logname : "client", @@ -265,14 +353,29 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) static int rq_state = 0; static cfs_waitq_t rq_waitq; -static int mgc_process_log(struct obd_device *mgc, - struct config_llog_data *cld); static int mgc_requeue_add(struct config_llog_data *cld, int later); +static void do_requeue(struct config_llog_data *cld) +{ + LASSERT(atomic_read(&cld->cld_refcount) > 0); + + if (cld->cld_mgcexp->exp_obd->u.cli.cl_conn_count != 0) { + CDEBUG(D_MGC, "updating log %s\n", cld->cld_logname); + mgc_process_log(cld->cld_mgcexp->exp_obd, cld); + } else { + CDEBUG(D_MGC, "disconnecting, won't update log %s\n", + cld->cld_logname); + } + + /* Whether we enqueued again or not in mgc_process_log, we're done + * with the ref from the old enqueue */ + config_log_put(cld); +} + static int mgc_requeue_thread(void *data) { struct l_wait_info lwi_now, lwi_later; - struct config_llog_data *cld, *n; + struct config_llog_data *cld, *cld_next, *cld_prev; char name[] = "ll_cfg_requeue"; int rc = 0; ENTRY; @@ -298,22 +401,52 @@ static int mgc_requeue_thread(void *data) NULL, NULL); l_wait_event(rq_waitq, rq_state & RQ_STOP, &lwi_now); + /* + * iterate & processing through the list. for each cld, process + * its depending sptlrpc cld firstly (if any) and then itself. + * + * it's guaranteed any item in the list must have + * reference > 0; and if cld_lostlock is set, at + * least one reference is taken by the previous enqueue. + * + * Note: releasing a cld might lead to itself and its depended + * sptlrpc cld be unlinked from the list. to safely iterate + * we need to take a reference on next cld before processing. + */ + cld_prev = NULL; + spin_lock(&config_list_lock); - list_for_each_entry_safe(cld, n, &config_llog_list, + list_for_each_entry_safe(cld, cld_next, &config_llog_list, cld_list_chain) { - spin_unlock(&config_list_lock); + if (cld->cld_list_chain.next != &config_llog_list) + atomic_inc(&cld_next->cld_refcount); + if (cld->cld_lostlock) { - CDEBUG(D_MGC, "updating log %s\n", - cld->cld_logname); + if (cld->cld_sptlrpc && + cld->cld_sptlrpc->cld_lostlock) { + cld->cld_sptlrpc->cld_lostlock = 0; + + spin_unlock(&config_list_lock); + do_requeue(cld->cld_sptlrpc); + spin_lock(&config_list_lock); + LASSERT(cld->cld_lostlock); + } + cld->cld_lostlock = 0; - rc = mgc_process_log(cld->cld_mgcexp->exp_obd, - cld); - /* Whether we enqueued again or not in - mgc_process_log, we're done with the ref - from the old enqueue */ - config_log_put(cld); + + spin_unlock(&config_list_lock); + do_requeue(cld); + spin_lock(&config_list_lock); } - spin_lock(&config_list_lock); + + + if (cld_prev) { + spin_unlock(&config_list_lock); + config_log_put(cld_prev); + spin_lock(&config_list_lock); + } + + cld_prev = cld_next; } spin_unlock(&config_list_lock); @@ -339,11 +472,13 @@ static int mgc_requeue_add(struct config_llog_data *cld, int later) CDEBUG(D_INFO, "log %s: requeue (l=%d r=%d sp=%d st=%x)\n", cld->cld_logname, later, atomic_read(&cld->cld_refcount), cld->cld_stopping, rq_state); + LASSERT(atomic_read(&cld->cld_refcount) > 0); /* Hold lock for rq_state */ spin_lock(&config_list_lock); if (cld->cld_stopping || (rq_state & RQ_STOP)) { + cld->cld_lostlock = 0; spin_unlock(&config_list_lock); config_log_put(cld); RETURN(0); @@ -599,7 +734,8 @@ static int mgc_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc, break; } /* Did we fail to get the lock? */ - if (lock->l_req_mode != lock->l_granted_mode) { + if (lock->l_req_mode != lock->l_granted_mode && + !cld->cld_is_sptlrpc) { CDEBUG(D_MGC, "log %s: original grant failed, will " "requeue later\n", cld->cld_logname); /* Try to re-enqueue later */ @@ -1081,22 +1217,18 @@ out: /* Get a config log from the MGS and process it. This func is called for both clients and servers. */ -static int mgc_process_log(struct obd_device *mgc, - struct config_llog_data *cld) +int mgc_process_log(struct obd_device *mgc, + struct config_llog_data *cld) { struct llog_ctxt *ctxt, *lctxt; struct lustre_handle lockh; struct client_obd *cli = &mgc->u.cli; struct lvfs_run_ctxt saved; - struct lustre_sb_info *lsi; + struct lustre_sb_info *lsi = NULL; int rc = 0, rcl, flags = 0, must_pop = 0; ENTRY; - if (!cld || !cld->cld_cfg.cfg_sb) { - /* This should never happen */ - CERROR("Missing cld, aborting log update\n"); - RETURN(-EINVAL); - } + LASSERT(cld); /* I don't want mutliple processes running process_log at once -- sounds like badness. It actually might be fine, as long as @@ -1111,7 +1243,8 @@ static int mgc_process_log(struct obd_device *mgc, OBD_FAIL_TIMEOUT(OBD_FAIL_MGC_PAUSE_PROCESS_LOG, 20); - lsi = s2lsi(cld->cld_cfg.cfg_sb); + if (cld->cld_cfg.cfg_sb) + lsi = s2lsi(cld->cld_cfg.cfg_sb); CDEBUG(D_MGC, "Process log %s:%s from %d\n", cld->cld_logname, cld->cld_cfg.cfg_instance, cld->cld_cfg.cfg_last_idx + 1); @@ -1132,6 +1265,13 @@ static int mgc_process_log(struct obd_device *mgc, lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT); + /* + * local copy of sptlrpc log is controlled elsewhere, don't try to + * read it up here. + */ + if (rcl && cld->cld_is_sptlrpc) + goto out_pop; + /* Copy the setup log locally if we can. Don't mess around if we're running an MGS though (logs are already local). */ if (lctxt && lsi && (lsi->lsi_flags & LSI_SERVER) && @@ -1160,6 +1300,9 @@ static int mgc_process_log(struct obd_device *mgc, ctxt = lctxt; } + if (cld->cld_is_sptlrpc) + sptlrpc_conf_log_update_begin(cld->cld_logname); + /* logname and instance info should be the same, so use our copy of the instance for the update. The cfg_last_idx will be updated here. */ @@ -1171,6 +1314,19 @@ out_pop: if (must_pop) pop_ctxt(&saved, &mgc->obd_lvfs_ctxt, NULL); + /* + * update settings on existing OBDs. doing it inside + * of llog_process_lock so no device is attaching/detaching + * in parallel. + * the logname must be -sptlrpc + */ + if (cld->cld_is_sptlrpc && rcl == 0) { + sptlrpc_conf_log_update_end(cld->cld_logname); + class_notify_sptlrpc_conf(cld->cld_logname, + strlen(cld->cld_logname) - + strlen("-sptlrpc")); + } + /* Now drop the lock so MGS can revoke it */ if (!rcl) { rcl = mgc_cancel(mgc->u.cli.cl_mgc_mgsexp, NULL, @@ -1215,6 +1371,10 @@ static int mgc_process_config(struct obd_device *obd, obd_count len, void *buf) CERROR("lov_del_obd unimplemented\n"); rc = -ENOSYS; break; + case LCFG_SPTLRPC_CONF: { + rc = sptlrpc_process_config(lcfg); + break; + } case LCFG_LOG_START: { struct config_llog_data *cld; struct config_llog_instance *cfg; @@ -1227,7 +1387,7 @@ static int mgc_process_config(struct obd_device *obd, obd_count len, void *buf) cfg->cfg_last_idx); /* We're only called through here on the initial mount */ - rc = config_log_add(logname, cfg, sb); + rc = config_log_add(obd, logname, cfg, sb); if (rc) break; cld = config_log_find(logname, cfg); diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index 99e698f..eeb98e9 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -87,6 +87,7 @@ struct fs_db { int mgs_init_fsdb_list(struct obd_device *obd); int mgs_cleanup_fsdb_list(struct obd_device *obd); +int mgs_get_fsdb_srpc_from_llog(struct obd_device *obd, struct fs_db *fsdb); int mgs_check_index(struct obd_device *obd, struct mgs_target_info *mti); int mgs_check_failnid(struct obd_device *obd, struct mgs_target_info *mti); int mgs_write_log_target(struct obd_device *obd, struct mgs_target_info *mti); diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 11d310d..6440f2f 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -65,13 +65,6 @@ #include #include "mgs_internal.h" -static int mgs_get_fsdb_srpc_from_llog(struct obd_device *obd, - struct fs_db *fsdb); -static int mgs_get_srpc_conf_log(struct fs_db *fsdb, const char *tgt, - enum lustre_sec_part from, - enum lustre_sec_part to, - struct sptlrpc_conf_log *log); - /********************** Class functions ********************/ /* Caller must list_del and OBD_FREE each dentry from the list */ @@ -770,25 +763,6 @@ static inline int record_setup(struct obd_device *obd, struct llog_handle *llh, return record_base(obd,llh,devname,0,LCFG_SETUP,s1,s2,s3,s4); } -static inline int record_sptlrpc_conf(struct obd_device *obd, - struct llog_handle *llh, - char *devname, - struct sptlrpc_conf_log *srpc_log) -{ - struct lustre_cfg_bufs bufs; - struct lustre_cfg *lcfg; - int rc; - - lustre_cfg_bufs_reset(&bufs, devname); - lustre_cfg_bufs_set(&bufs, 1, srpc_log, sizeof(*srpc_log)); - lcfg = lustre_cfg_new(LCFG_SPTLRPC_CONF, &bufs); - - rc = record_lcfg(obd, llh, lcfg); - - lustre_cfg_free(lcfg); - return rc; -} - static int record_lov_setup(struct obd_device *obd, struct llog_handle *llh, char *devname, struct lov_desc *desc) { @@ -1354,7 +1328,6 @@ static int mgs_write_log_mdc_to_lmv(struct obd_device *obd, struct fs_db *fsdb, char *logname, char *lmvname) { struct llog_handle *llh = NULL; - struct sptlrpc_conf_log *srpc_log; char *mdcname, *nodeuuid, *mdcuuid, *lmvuuid; char index[5]; int i, rc; @@ -1368,16 +1341,6 @@ static int mgs_write_log_mdc_to_lmv(struct obd_device *obd, struct fs_db *fsdb, CDEBUG(D_MGS, "adding mdc for %s to log %s:lmv(%s)\n", mti->mti_svname, logname, lmvname); - srpc_log = sptlrpc_conf_log_alloc(); - if (IS_ERR(srpc_log)) - RETURN(PTR_ERR(srpc_log)); - srpc_log->scl_part = LUSTRE_SP_CLI; - - rc = mgs_get_srpc_conf_log(fsdb, mti->mti_svname, - LUSTRE_SP_CLI, LUSTRE_SP_MDT, srpc_log); - if (rc) - goto out_srpc; - name_create(&nodeuuid, libcfs_nid2str(mti->mti_nids[0]), ""); name_create(&mdcname, mti->mti_svname, "-mdc"); name_create(&mdcuuid, mdcname, "_UUID"); @@ -1396,7 +1359,6 @@ static int mgs_write_log_mdc_to_lmv(struct obd_device *obd, struct fs_db *fsdb, rc = record_attach(obd, llh, mdcname, LUSTRE_MDC_NAME, lmvuuid); rc = record_setup(obd, llh, mdcname, mti->mti_uuid, nodeuuid, 0, 0); - rc = record_sptlrpc_conf(obd, llh, mdcname, srpc_log); rc = mgs_write_log_failnids(obd, mti, llh, mdcname); snprintf(index, sizeof(index), "%d", mti->mti_stripe_index); rc = record_mdc_add(obd, llh, lmvname, mdcuuid, mti->mti_uuid, @@ -1409,8 +1371,6 @@ static int mgs_write_log_mdc_to_lmv(struct obd_device *obd, struct fs_db *fsdb, name_destroy(&mdcuuid); name_destroy(&mdcname); name_destroy(&nodeuuid); -out_srpc: - sptlrpc_conf_log_free(srpc_log); RETURN(rc); } @@ -1419,7 +1379,6 @@ static int mgs_write_log_mdc_to_mdt(struct obd_device *obd, struct fs_db *fsdb, struct mgs_target_info *mti, char *logname) { struct llog_handle *llh = NULL; - struct sptlrpc_conf_log *srpc_log; char *nodeuuid, *mdcname, *mdcuuid, *mdtuuid; int idx = mti->mti_stripe_index; char index[9]; @@ -1433,16 +1392,6 @@ static int mgs_write_log_mdc_to_mdt(struct obd_device *obd, struct fs_db *fsdb, CDEBUG(D_MGS, "adding mdc index %d to %s\n", idx, logname); - srpc_log = sptlrpc_conf_log_alloc(); - if (IS_ERR(srpc_log)) - RETURN(PTR_ERR(srpc_log)); - srpc_log->scl_part = LUSTRE_SP_MDT; - - rc = mgs_get_srpc_conf_log(fsdb, mti->mti_svname, - LUSTRE_SP_MDT, LUSTRE_SP_MDT, srpc_log); - if (rc) - goto out_srpc; - name_create(&nodeuuid, libcfs_nid2str(mti->mti_nids[0]), ""); snprintf(index, sizeof(index), "-mdc%04x", idx); name_create(&mdcname, logname, index); @@ -1458,7 +1407,6 @@ static int mgs_write_log_mdc_to_mdt(struct obd_device *obd, struct fs_db *fsdb, } rc = record_attach(obd, llh, mdcname, LUSTRE_MDC_NAME, mdcuuid); rc = record_setup(obd, llh, mdcname, mti->mti_uuid, nodeuuid, 0, 0); - rc = record_sptlrpc_conf(obd, llh, mdcname, srpc_log); rc = mgs_write_log_failnids(obd, mti, llh, mdcname); snprintf(index, sizeof(index), "%d", idx); @@ -1471,8 +1419,6 @@ static int mgs_write_log_mdc_to_mdt(struct obd_device *obd, struct fs_db *fsdb, name_destroy(&mdcname); name_destroy(&nodeuuid); name_destroy(&mdtuuid); -out_srpc: - sptlrpc_conf_log_free(srpc_log); RETURN(rc); } @@ -1483,24 +1429,13 @@ static int mgs_write_log_mdt0(struct obd_device *obd, struct fs_db *fsdb, struct llog_handle *llh = NULL; char *uuid, *lovname; char mdt_index[5]; - struct sptlrpc_conf_log *srpc_log; char *ptr = mti->mti_params; int rc = 0, failout = 0; ENTRY; - srpc_log = sptlrpc_conf_log_alloc(); - if (IS_ERR(srpc_log)) - RETURN(PTR_ERR(srpc_log)); - srpc_log->scl_part = LUSTRE_SP_MDT; - - rc = mgs_get_srpc_conf_log(fsdb, mti->mti_svname, - LUSTRE_SP_ANY, LUSTRE_SP_MDT, srpc_log); - if (rc) - GOTO(out_srpc, rc); - OBD_ALLOC(uuid, sizeof(struct obd_uuid)); if (uuid == NULL) - GOTO(out_srpc, rc = -ENOMEM); + RETURN(-ENOMEM); if (class_find_param(ptr, PARAM_FAILMODE, &ptr) == 0) failout = (strncmp(ptr, "failout", 7) == 0); @@ -1523,14 +1458,11 @@ static int mgs_write_log_mdt0(struct obd_device *obd, struct fs_db *fsdb, rc = record_mount_opt(obd, llh, log, lovname, NULL); rc = record_setup(obd, llh, log, uuid, mdt_index, lovname, failout ? "n" : "f"); - rc = record_sptlrpc_conf(obd, llh, log, srpc_log); rc = record_marker(obd, llh, fsdb, CM_END, log, "add mdt"); rc = record_end_log(obd, &llh); out: name_destroy(&lovname); OBD_FREE(uuid, sizeof(struct obd_uuid)); -out_srpc: - sptlrpc_conf_log_free(srpc_log); RETURN(rc); } @@ -1665,7 +1597,6 @@ static int mgs_write_log_osc_to_lov(struct obd_device *obd, struct fs_db *fsdb, enum lustre_sec_part sec_part, int flags) { struct llog_handle *llh = NULL; - struct sptlrpc_conf_log *srpc_log; char *nodeuuid, *oscname, *oscuuid, *lovuuid, *svname; char index[5]; int i, rc; @@ -1674,16 +1605,6 @@ static int mgs_write_log_osc_to_lov(struct obd_device *obd, struct fs_db *fsdb, CDEBUG(D_INFO, "adding osc for %s to log %s\n", mti->mti_svname, logname); - srpc_log = sptlrpc_conf_log_alloc(); - if (IS_ERR(srpc_log)) - RETURN(PTR_ERR(srpc_log)); - srpc_log->scl_part = sec_part; - - rc = mgs_get_srpc_conf_log(fsdb, mti->mti_svname, - sec_part, LUSTRE_SP_OST, srpc_log); - if (rc) - goto out_srpc; - if (mgs_log_is_empty(obd, logname)) { /* The first item in the log must be the lov, so we have somewhere to add our osc. */ @@ -1720,7 +1641,6 @@ static int mgs_write_log_osc_to_lov(struct obd_device *obd, struct fs_db *fsdb, } rc = record_attach(obd, llh, oscname, LUSTRE_OSC_NAME, lovuuid); rc = record_setup(obd, llh, oscname, mti->mti_uuid, nodeuuid, 0, 0); - rc = record_sptlrpc_conf(obd, llh, oscname, srpc_log); rc = mgs_write_log_failnids(obd, mti, llh, oscname); snprintf(index, sizeof(index), "%d", mti->mti_stripe_index); rc = record_lov_add(obd, llh, lovname, mti->mti_uuid, index, "1"); @@ -1733,8 +1653,6 @@ out: name_destroy(&oscname); name_destroy(&svname); name_destroy(&nodeuuid); -out_srpc: - sptlrpc_conf_log_free(srpc_log); RETURN(rc); } @@ -1742,7 +1660,6 @@ static int mgs_write_log_ost(struct obd_device *obd, struct fs_db *fsdb, struct mgs_target_info *mti) { struct llog_handle *llh = NULL; - struct sptlrpc_conf_log *srpc_log; char *logname, *lovname; char mdt_index[9]; char *ptr = mti->mti_params; @@ -1764,16 +1681,6 @@ static int mgs_write_log_ost(struct obd_device *obd, struct fs_db *fsdb, RETURN(-EALREADY); } - srpc_log = sptlrpc_conf_log_alloc(); - if (IS_ERR(srpc_log)) - RETURN(PTR_ERR(srpc_log)); - srpc_log->scl_part = LUSTRE_SP_OST; - - rc = mgs_get_srpc_conf_log(fsdb, mti->mti_svname, - LUSTRE_SP_ANY, LUSTRE_SP_OST, srpc_log); - if (rc) - goto out_srpc; - /* attach obdfilter ost1 ost1_UUID setup /dev/loop2 ldiskfs f|n errors=remount-ro,user_xattr @@ -1793,7 +1700,6 @@ static int mgs_write_log_ost(struct obd_device *obd, struct fs_db *fsdb, rc = record_setup(obd, llh, mti->mti_svname, "dev"/*ignored*/, "type"/*ignored*/, failout ? "n" : "f", 0/*options*/); - rc = record_sptlrpc_conf(obd, llh, mti->mti_svname, srpc_log); rc = record_marker(obd, llh, fsdb, CM_END, mti->mti_svname, "add ost"); rc = record_end_log(obd, &llh); @@ -1835,8 +1741,6 @@ static int mgs_write_log_ost(struct obd_device *obd, struct fs_db *fsdb, mgs_write_log_osc_to_lov(obd, fsdb, mti, logname, "", fsdb->fsdb_clilov, LUSTRE_SP_CLI, 0); name_destroy(&logname); -out_srpc: - sptlrpc_conf_log_free(srpc_log); RETURN(rc); } @@ -1939,333 +1843,6 @@ static int mgs_wlp_lcfg(struct obd_device *obd, struct fs_db *fsdb, return rc; } -/* - * populate rules which applied to a target device - */ -static int mgs_get_srpc_conf_log(struct fs_db *fsdb, const char *tgt, - enum lustre_sec_part from, - enum lustre_sec_part to, - struct sptlrpc_conf_log *log) -{ - struct mgs_tgt_srpc_conf *tgtconf; - struct sptlrpc_rule_set *tgt_rset; - int found_tgt = 0, rc; - - for (tgtconf = fsdb->fsdb_srpc_tgt; tgtconf; - tgtconf = tgtconf->mtsc_next) { - if (!strcmp(tgt, tgtconf->mtsc_tgt)) { - found_tgt = 1; - break; - } - } - - if (found_tgt) - tgt_rset = &tgtconf->mtsc_rset; - else - tgt_rset = NULL; - - rc = sptlrpc_conf_log_populate(&fsdb->fsdb_srpc_gen, tgt_rset, - from, to, fsdb->fsdb_srpc_fl_udesc, log); - if (rc) - CERROR("failed to populate srpc log for %s: %d\n", tgt, rc); - - return rc; -} - -struct mgs_msl_data { - struct obd_device *mmd_obd; - struct fs_db *mmd_fsdb; - struct mgs_target_info *mmd_mti; - int mmd_skip; - int mmd_attached; - int mmd_server; - enum lustre_sec_part mmd_tgtpart; - char mmd_tgtname[MTI_NAME_MAXLEN]; -}; - -static void mgs_msl_data_cleanup(struct mgs_msl_data *mmd) -{ - mmd->mmd_attached = 0; - mmd->mmd_tgtname[0] = '\0'; -} - -static int mgs_msl_tgt_uuid2name(char *tgtname, char *tgtuuid) -{ - char *ptr; - - if (tgtuuid == NULL) { - CERROR("missing target UUID???\n"); - return -EINVAL; - } - - ptr = strstr(tgtuuid, "_UUID"); - if (ptr == NULL) { - CERROR("unrecognized UUID: %s\n", tgtuuid); - return -EINVAL; - } - - *ptr = '\0';; - strncpy(tgtname, tgtuuid, MTI_NAME_MAXLEN); - tgtname[MTI_NAME_MAXLEN - 1] = '\0'; - - return 0; -} - -static int mgs_modify_srpc_log_handler(struct llog_handle *llh, - struct llog_rec_hdr *rec, - void *data) -{ - struct mgs_msl_data *mmd = (struct mgs_msl_data *)data; - struct cfg_marker *marker; - struct lustre_cfg *lcfg = (struct lustre_cfg *)(rec + 1); - int cfg_len, rc; - ENTRY; - - if (rec->lrh_type != OBD_CFG_REC) { - CERROR("unhandled lrh_type: %#x\n", rec->lrh_type); - RETURN(-EINVAL); - } - - cfg_len = rec->lrh_len - sizeof(struct llog_rec_hdr) - - sizeof(struct llog_rec_tail); - - rc = lustre_cfg_sanity_check(lcfg, cfg_len); - if (rc) { - CERROR("Insane cfg\n"); - RETURN(rc); - } - - if (lcfg->lcfg_command == LCFG_MARKER) { - marker = lustre_cfg_buf(lcfg, 1); - - if (marker->cm_flags & CM_START && - marker->cm_flags & CM_SKIP) - mmd->mmd_skip = 1; - if (marker->cm_flags & CM_END) - mmd->mmd_skip = 0; - - RETURN(0); - } - - if (mmd->mmd_skip) - RETURN(0); - - switch (lcfg->lcfg_command) { - case LCFG_ATTACH: - mmd->mmd_attached = 1; - - if (!strcmp(lustre_cfg_string(lcfg, 1), LUSTRE_OST_NAME)) { - mmd->mmd_server = 1; - mmd->mmd_tgtpart = LUSTRE_SP_OST; - } else if (!strcmp(lustre_cfg_string(lcfg, 1), - LUSTRE_MDT_NAME)) { - mmd->mmd_server = 1; - mmd->mmd_tgtpart = LUSTRE_SP_MDT; - } else if (!strcmp(lustre_cfg_string(lcfg, 1), - LUSTRE_OSC_NAME)) { - mmd->mmd_server = 0; - mmd->mmd_tgtpart = LUSTRE_SP_OST; - } else if (!strcmp(lustre_cfg_string(lcfg, 1), - LUSTRE_MDC_NAME)) { - mmd->mmd_server = 0; - mmd->mmd_tgtpart = LUSTRE_SP_MDT; - } else { - mmd->mmd_attached = 0; - } - - if (mmd->mmd_attached && mmd->mmd_server) { - rc = mgs_msl_tgt_uuid2name(mmd->mmd_tgtname, - lustre_cfg_string(lcfg, 2)); - if (rc) { - mgs_msl_data_cleanup(mmd); - break; - } - } - - break; - case LCFG_SETUP: - if (!mmd->mmd_attached) - break; - - /* already got tgtname at LCFG_ATTACH */ - if (mmd->mmd_server) - break; - - rc = mgs_msl_tgt_uuid2name(mmd->mmd_tgtname, - lustre_cfg_string(lcfg, 1)); - if (rc) { - mgs_msl_data_cleanup(mmd); - break; - } - - break; - case LCFG_SPTLRPC_CONF: { - struct sptlrpc_conf_log *log; - enum lustre_sec_part from; - - if (!mmd->mmd_attached) - break; - - log = sptlrpc_conf_log_extract(lcfg); - if (log == NULL) { - CERROR("missing sptlrpc config log???\n"); - mgs_msl_data_cleanup(mmd); - break; - } - - if (mmd->mmd_server) - from = LUSTRE_SP_ANY; - else - from = log->scl_part; - - /* cleanup the old log */ - sptlrpc_conf_log_cleanup(log); - - /* populate new log */ - rc = mgs_get_srpc_conf_log(mmd->mmd_fsdb, mmd->mmd_tgtname, - from, mmd->mmd_tgtpart, log); - if (rc) { - mgs_msl_data_cleanup(mmd); - break; - } - - /* Overwrite the log */ - rec->lrh_len = cfg_len; - rc = llog_write_rec(llh, rec, NULL, 0, (void *)lcfg, - rec->lrh_index); - if (rc) - CERROR("overwrite sptlrpc conf log failed: %d\n", rc); - - /* append new one */ - rc = record_marker(mmd->mmd_obd, llh, mmd->mmd_fsdb, CM_START, - mmd->mmd_mti->mti_svname, "sptlrpc config"); - rc = record_sptlrpc_conf(mmd->mmd_obd, llh, - lustre_cfg_string(lcfg, 0), log); - rc = record_marker(mmd->mmd_obd, llh, mmd->mmd_fsdb, CM_END, - mmd->mmd_mti->mti_svname, "sptlrpc config"); - - mgs_msl_data_cleanup(mmd); - break; - } - default: - /* ignore all others */ - break; - } - - RETURN(rc); -} - -static int mgs_modify_srpc_log(struct obd_device *obd, - struct fs_db *fsdb, - struct mgs_target_info *mti, - char *logname) -{ - struct llog_handle *llh; - struct lvfs_run_ctxt saved; - struct llog_ctxt *ctxt; - struct mgs_msl_data *mmd; - int rc, rc2; - ENTRY; - - CDEBUG(D_MGS, "modify sptlrpc log for %s\n", logname); - - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - - ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); - LASSERT(ctxt != NULL); - rc = llog_create(ctxt, &llh, NULL, logname); - if (rc) - GOTO(out_pop, rc); - - rc = llog_init_handle(llh, LLOG_F_IS_PLAIN, NULL); - if (rc) - GOTO(out_close, rc); - - if (llog_get_size(llh) <= 1) - GOTO(out_close, rc = 0); - - OBD_ALLOC_PTR(mmd); - if (!mmd) - GOTO(out_close, rc = -ENOMEM); - - mmd->mmd_obd = obd; - mmd->mmd_fsdb = fsdb; - mmd->mmd_mti = mti; - - rc = llog_process(llh, mgs_modify_srpc_log_handler, (void *) mmd, NULL); - - OBD_FREE_PTR(mmd); - -out_close: - rc2 = llog_close(llh); - if (!rc) - rc = rc2; - -out_pop: - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - llog_ctxt_put(ctxt); - - if (rc) - CERROR("modify sptlrpc log %s failed %d\n", logname, rc); - RETURN(rc); -} - -/* - * for each of log, remove old conf at first - */ -static int mgs_modify_srpc_log_all(struct obd_device *obd, - struct fs_db *fsdb, - struct mgs_target_info *mti) -{ - char tgt_index[9]; - char *logname; - int i, rc = 0, rc2; - ENTRY; - - for (i = 0; i < INDEX_MAP_SIZE * 8; i++){ - if (test_bit(i, fsdb->fsdb_mdt_index_map)) { - sprintf(tgt_index,"-MDT%04x",i); - - name_create(&logname, mti->mti_fsname, tgt_index); - rc2 = mgs_modify(obd, fsdb, mti, logname, - mti->mti_fsname, "sptlrpc config", - CM_SKIP); - rc2 = mgs_modify_srpc_log(obd, fsdb, mti, logname); - name_destroy(&logname); - - if (rc2 && rc == 0) - rc = rc2; - } - } - - for (i = 0; i < INDEX_MAP_SIZE * 8; i++){ - if (test_bit(i, fsdb->fsdb_ost_index_map)) { - sprintf(tgt_index,"-OST%04x",i); - - name_create(&logname, mti->mti_fsname, tgt_index); - rc2 = mgs_modify(obd, fsdb, mti, logname, - mti->mti_fsname, "sptlrpc config", - CM_SKIP); - rc2 = mgs_modify_srpc_log(obd, fsdb, mti, logname); - name_destroy(&logname); - - if (rc2 && rc == 0) - rc = rc2; - } - } - - name_create(&logname, mti->mti_fsname, "-client"); - rc2 = mgs_modify(obd, fsdb, mti, logname, - mti->mti_fsname, "sptlrpc config", CM_SKIP); - rc2 = mgs_modify_srpc_log(obd, fsdb, mti, logname); - name_destroy(&logname); - - if (rc2 && rc == 0) - rc = rc2; - - RETURN(rc); -} - static int mgs_srpc_set_param_disk(struct obd_device *obd, struct fs_db *fsdb, struct mgs_target_info *mti, @@ -2293,7 +1870,7 @@ static int mgs_srpc_set_param_disk(struct obd_device *obd, /* prepare lcfg */ lustre_cfg_bufs_reset(&bufs, mti->mti_svname); lustre_cfg_bufs_set_string(&bufs, 1, param); - lcfg = lustre_cfg_new(0, &bufs); + lcfg = lustre_cfg_new(LCFG_SPTLRPC_CONF, &bufs); if (lcfg == NULL) GOTO(out_comment, rc = -ENOMEM); @@ -2433,14 +2010,6 @@ static int mgs_srpc_set_param_mem(struct fs_db *fsdb, rset = &fsdb->fsdb_srpc_gen; } - /* limit the maximum number of rules, but allow deletion in any case */ - if (rset->srs_nrule >= SPTLRPC_CONF_LOG_MAX / 2 && - rule.sr_flvr.sf_rpc != SPTLRPC_FLVR_INVALID) { - CERROR("too many (%d) rules already for %s\n", - rset->srs_nrule, svname); - RETURN(-E2BIG); - } - rc = sptlrpc_rule_set_merge(rset, &rule, 1); RETURN(rc); @@ -2451,8 +2020,8 @@ static int mgs_srpc_set_param(struct obd_device *obd, struct mgs_target_info *mti, char *param) { - char *copy; - int rc, copy_size; + char *copy; + int rc, copy_size; ENTRY; /* keep a copy of original param, which could be destroied @@ -2469,12 +2038,6 @@ static int mgs_srpc_set_param(struct obd_device *obd, /* previous steps guaranteed the syntax is correct */ rc = mgs_srpc_set_param_disk(obd, fsdb, mti, copy); - if (rc) - goto out_free; - - /* now apply the new rules to all existing config logs */ - rc = mgs_modify_srpc_log_all(obd, fsdb, mti); - out_free: OBD_FREE(copy, copy_size); RETURN(rc); @@ -2525,7 +2088,7 @@ static int mgs_srpc_read_handler(struct llog_handle *llh, if (msrd->msrd_skip) RETURN(0); - if (lcfg->lcfg_command != 0) { + if (lcfg->lcfg_command != LCFG_SPTLRPC_CONF) { CERROR("invalid command (%x)\n", lcfg->lcfg_command); RETURN(0); } @@ -2549,8 +2112,8 @@ static int mgs_srpc_read_handler(struct llog_handle *llh, RETURN(0); } -static int mgs_get_fsdb_srpc_from_llog(struct obd_device *obd, - struct fs_db *fsdb) +int mgs_get_fsdb_srpc_from_llog(struct obd_device *obd, + struct fs_db *fsdb) { struct llog_handle *llh = NULL; struct lvfs_run_ctxt saved; @@ -2560,14 +2123,14 @@ static int mgs_get_fsdb_srpc_from_llog(struct obd_device *obd, int rc; ENTRY; - ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); - LASSERT(ctxt != NULL); - /* construct log name */ rc = name_create(&logname, fsdb->fsdb_name, "-sptlrpc"); if (rc) RETURN(rc); + ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); + LASSERT(ctxt != NULL); + if (mgs_log_is_empty(obd, logname)) GOTO(out, rc = 0); @@ -2594,8 +2157,8 @@ out_close: out_pop: pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); out: - name_destroy(&logname); llog_ctxt_put(ctxt); + name_destroy(&logname); if (rc) CERROR("failed to read sptlrpc config database: %d\n", rc); diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 5adafa3..2de3465 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -538,6 +538,49 @@ struct obd_device * class_devices_in_group(struct obd_uuid *grp_uuid, int *next) return NULL; } +/** + * to notify sptlrpc log for @fsname has changed, let every relevant OBD + * adjust sptlrpc settings accordingly. + */ +int class_notify_sptlrpc_conf(const char *fsname, int namelen) +{ + struct obd_device *obd; + const char *type; + int i, rc = 0, rc2; + + LASSERT(namelen > 0); + + spin_lock(&obd_dev_lock); + for (i = 0; i < class_devno_max(); i++) { + obd = class_num2obd(i); + + if (obd == NULL || obd->obd_set_up == 0 || obd->obd_stopping) + continue; + + /* only notify mdc, osc, mdt, ost */ + type = obd->obd_type->typ_name; + if (strcmp(type, LUSTRE_MDC_NAME) != 0 && + strcmp(type, LUSTRE_OSC_NAME) != 0 && + strcmp(type, LUSTRE_MDT_NAME) != 0 && + strcmp(type, LUSTRE_OST_NAME) != 0) + continue; + + if (strncmp(obd->obd_name, fsname, namelen)) + continue; + + class_incref(obd, __FUNCTION__, obd); + spin_unlock(&obd_dev_lock); + rc2 = obd_set_info_async(obd->obd_self_export, + sizeof(KEY_SPTLRPC_CONF), + KEY_SPTLRPC_CONF, 0, NULL, NULL); + rc = rc ? rc : rc2; + class_decref(obd, __FUNCTION__, obd); + spin_lock(&obd_dev_lock); + } + spin_unlock(&obd_dev_lock); + return rc; +} +EXPORT_SYMBOL(class_notify_sptlrpc_conf); void obd_cleanup_caches(void) { diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index ef51c1e..1c636b4 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -1018,8 +1018,9 @@ static int class_config_llog_handler(struct llog_handle * handle, CDEBUG(D_CONFIG, "SKIP #%d\n", marker->cm_step); } else if ((marker->cm_flags & CM_EXCLUDE) || - lustre_check_exclusion(clli->cfg_sb, - marker->cm_tgtname)) { + (clli->cfg_sb && + lustre_check_exclusion(clli->cfg_sb, + marker->cm_tgtname))) { clli->cfg_flags |= CFG_F_EXCLUDE; CDEBUG(D_CONFIG, "EXCLUDE %d\n", marker->cm_step); @@ -1100,6 +1101,22 @@ static int class_config_llog_handler(struct llog_handle * handle, lustre_cfg_bufs_set_string(&bufs, 2, clli->cfg_uuid.uuid); } + /* + * sptlrpc config record, we expect 2 data segments: + * [0]: fs_name/target_name, + * [1]: rule string + * moving them to index [1] and [2], and insert MGC's + * obdname at index [0]. + */ + if (clli && clli->cfg_instance == NULL && + lcfg->lcfg_command == LCFG_SPTLRPC_CONF) { + lustre_cfg_bufs_set(&bufs, 2, bufs.lcfg_buf[1], + bufs.lcfg_buflen[1]); + lustre_cfg_bufs_set(&bufs, 1, bufs.lcfg_buf[0], + bufs.lcfg_buflen[0]); + lustre_cfg_bufs_set_string(&bufs, 0, + clli->cfg_obdname); + } lcfg_new = lustre_cfg_new(lcfg->lcfg_command, &bufs); diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 2c3d5c1..3daebea 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -1888,6 +1888,30 @@ static void filter_iobuf_pool_done(struct filter_obd *filter) EXIT; } +static int filter_adapt_sptlrpc_conf(struct obd_device *obd, int initial) +{ + struct filter_obd *filter = &obd->u.filter; + struct sptlrpc_rule_set tmp_rset; + int rc; + + sptlrpc_rule_set_init(&tmp_rset); + rc = sptlrpc_conf_target_get_rules(obd, &tmp_rset, initial); + if (rc) { + CERROR("obd %s: failed get sptlrpc rules: %d\n", + obd->obd_name, rc); + return rc; + } + + sptlrpc_target_update_exp_flavor(obd, &tmp_rset); + + write_lock(&filter->fo_sptlrpc_lock); + sptlrpc_rule_set_free(&filter->fo_sptlrpc_rset); + filter->fo_sptlrpc_rset = tmp_rset; + write_unlock(&filter->fo_sptlrpc_lock); + + return 0; +} + /* * pre-allocate pool of iobuf's to be used by filter_{prep,commit}rw_write(). */ @@ -2039,9 +2063,6 @@ int filter_common_setup(struct obd_device *obd, struct lustre_cfg* lcfg, CFS_INIT_LIST_HEAD(&filter->fo_llog_list); spin_lock_init(&filter->fo_llog_list_lock); - rwlock_init(&filter->fo_sptlrpc_lock); - sptlrpc_rule_set_init(&filter->fo_sptlrpc_rset); - filter->fo_fl_oss_capa = 1; CFS_INIT_LIST_HEAD(&filter->fo_capa_keys); filter->fo_capa_hash = init_capa_hash(); @@ -2066,6 +2087,11 @@ int filter_common_setup(struct obd_device *obd, struct lustre_cfg* lcfg, GOTO(err_post, rc); } + rwlock_init(&filter->fo_sptlrpc_lock); + sptlrpc_rule_set_init(&filter->fo_sptlrpc_rset); + /* do this after llog being initialized */ + filter_adapt_sptlrpc_conf(obd, 1); + rc = lquota_setup(filter_quota_interface_ref, obd); if (rc) GOTO(err_post, rc); @@ -2221,10 +2247,18 @@ static int filter_olg_fini(struct obd_llog_group *olg) rc = llog_cleanup(ctxt); ctxt = llog_group_get_ctxt(olg, LLOG_SIZE_ORIG_CTXT); - if (ctxt) + if (ctxt) { + rc2 = llog_cleanup(ctxt); + if (!rc) + rc = rc2; + } + + ctxt = llog_group_get_ctxt(olg, LLOG_CONFIG_ORIG_CTXT); + if (ctxt) { rc2 = llog_cleanup(ctxt); - if (!rc) - rc = rc2; + if (!rc) + rc = rc2; + } RETURN(rc); } @@ -2277,6 +2311,11 @@ filter_default_olg_init(struct obd_device *obd, struct obd_llog_group *olg, if (rc) GOTO(cleanup_lcm, rc); + rc = llog_setup(obd, olg, LLOG_CONFIG_ORIG_CTXT, tgt, 0, NULL, + &llog_lvfs_ops); + if (rc) + GOTO(cleanup_olg, rc); + ctxt = llog_group_get_ctxt(olg, LLOG_MDS_OST_REPL_CTXT); if (!ctxt) { CERROR("Can't get ctxt for %p:%x\n", olg, @@ -4275,6 +4314,11 @@ static int filter_set_info_async(struct obd_export *exp, __u32 keylen, RETURN(0); } + if (KEY_IS(KEY_SPTLRPC_CONF)) { + filter_adapt_sptlrpc_conf(obd, 0); + RETURN(0); + } + if (!KEY_IS(KEY_MDS_CONN)) RETURN(-EINVAL); @@ -4404,34 +4448,6 @@ static int filter_process_config(struct obd_device *obd, obd_count len, int rc = 0; switch (lcfg->lcfg_command) { - case LCFG_SPTLRPC_CONF: { - struct filter_obd *filter = &obd->u.filter; - struct sptlrpc_conf_log *log; - struct sptlrpc_rule_set tmp_rset; - - log = sptlrpc_conf_log_extract(lcfg); - if (IS_ERR(log)) { - rc = PTR_ERR(log); - break; - } - - sptlrpc_rule_set_init(&tmp_rset); - - rc = sptlrpc_rule_set_from_log(&tmp_rset, log); - if (rc) { - CERROR("obd %s: failed get sptlrpc rules: %d\n", - obd->obd_name, rc); - break; - } - - write_lock(&filter->fo_sptlrpc_lock); - sptlrpc_rule_set_free(&filter->fo_sptlrpc_rset); - filter->fo_sptlrpc_rset = tmp_rset; - write_unlock(&filter->fo_sptlrpc_lock); - - sptlrpc_target_update_exp_flavor(obd, &tmp_rset); - break; - } default: lprocfs_filter_init_vars(&lvars); diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index f36306f..9bf1afa 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3596,6 +3596,11 @@ static int osc_set_info_async(struct obd_export *exp, obd_count keylen, RETURN(0); } + if (KEY_IS(KEY_SPTLRPC_CONF)) { + sptlrpc_conf_client_adapt(obd); + RETURN(0); + } + if (KEY_IS(KEY_FLUSH_CTX)) { sptlrpc_import_flush_my_ctx(imp); RETURN(0); @@ -3967,9 +3972,6 @@ int osc_process_config_base(struct obd_device *obd, struct lustre_cfg *lcfg) lprocfs_osc_init_vars(&lvars); switch (lcfg->lcfg_command) { - case LCFG_SPTLRPC_CONF: - rc = sptlrpc_cliobd_process_config(obd, lcfg); - break; default: rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars, lcfg, obd); diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c index 3d56643..b49ca7a 100644 --- a/lustre/ost/ost_handler.c +++ b/lustre/ost/ost_handler.c @@ -1411,41 +1411,54 @@ static int ost_init_sec_level(struct ptlrpc_request *req) RETURN(rc); } -static int filter_export_check_flavor(struct filter_obd *filter, - struct obd_export *exp, - struct ptlrpc_request *req) +/* + * FIXME + * this should be done in filter_connect()/filter_reconnect(), but + * we can't obtain information like NID, which stored in incoming + * request, thus can't decide what flavor to use. so we do it here. + * + * This hack should be removed after the OST stack be rewritten, just + * like what we are doing in mdt_obd_connect()/mdt_obd_reconnect(). + */ +static int ost_connect_check_sptlrpc(struct ptlrpc_request *req) { - int rc = 0; - - /* FIXME - * this should be done in filter_connect()/filter_reconnect(), but - * we can't obtain information like NID, which stored in incoming - * request, thus can't decide what flavor to use. so we do it here. - * - * This hack should be removed after the OST stack be rewritten, just - * like what we are doing in mdt_obd_connect()/mdt_obd_reconnect(). - */ - if (exp->exp_flvr.sf_rpc != SPTLRPC_FLVR_INVALID) - return 0; - - CDEBUG(D_SEC, "from %s\n", sptlrpc_part2name(req->rq_sp_from)); - spin_lock(&exp->exp_lock); - exp->exp_sp_peer = req->rq_sp_from; - - read_lock(&filter->fo_sptlrpc_lock); - sptlrpc_rule_set_choose(&filter->fo_sptlrpc_rset, exp->exp_sp_peer, - req->rq_peer.nid, &exp->exp_flvr); - read_unlock(&filter->fo_sptlrpc_lock); - - if (exp->exp_flvr.sf_rpc != req->rq_flvr.sf_rpc) { - CERROR("invalid rpc flavor %x, expect %x, from %s\n", - req->rq_flvr.sf_rpc, exp->exp_flvr.sf_rpc, - libcfs_nid2str(req->rq_peer.nid)); - exp->exp_flvr.sf_rpc = SPTLRPC_FLVR_INVALID; - rc = -EACCES; - } + struct obd_export *exp = req->rq_export; + struct filter_obd *filter = &exp->exp_obd->u.filter; + struct sptlrpc_flavor flvr; + int rc = 0; + + if (exp->exp_flvr.sf_rpc == SPTLRPC_FLVR_INVALID) { + read_lock(&filter->fo_sptlrpc_lock); + sptlrpc_target_choose_flavor(&filter->fo_sptlrpc_rset, + req->rq_sp_from, + req->rq_peer.nid, + &flvr); + read_unlock(&filter->fo_sptlrpc_lock); + + spin_lock(&exp->exp_lock); + + exp->exp_sp_peer = req->rq_sp_from; + exp->exp_flvr = flvr; + + if (exp->exp_flvr.sf_rpc != req->rq_flvr.sf_rpc) { + CERROR("unauthorized rpc flavor %x from %s, " + "expect %x\n", req->rq_flvr.sf_rpc, + libcfs_nid2str(req->rq_peer.nid), + exp->exp_flvr.sf_rpc); + rc = -EACCES; + } - spin_unlock(&exp->exp_lock); + spin_unlock(&exp->exp_lock); + } else { + if (exp->exp_sp_peer != req->rq_sp_from) { + CERROR("RPC source %s doesn't match %s\n", + sptlrpc_part2name(req->rq_sp_from), + sptlrpc_part2name(exp->exp_sp_peer)); + rc = -EACCES; + } else { + rc = sptlrpc_target_export_check(exp, req); + } + } return rc; } @@ -1860,13 +1873,8 @@ int ost_handle(struct ptlrpc_request *req) RETURN(0); if (!rc) { rc = ost_init_sec_level(req); - if (!rc) { - struct obd_export *exp = req->rq_export; - - obd = exp->exp_obd; - rc = filter_export_check_flavor(&obd->u.filter, - exp, req); - } + if (!rc) + rc = ost_connect_check_sptlrpc(req); } break; } diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 49694a1..0913e85 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -1307,6 +1307,7 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set) if (status) { if (req->rq_err) { req->rq_status = status; + req->rq_wait_ctx = 0; force_timer_recalc = 1; } else { req->rq_wait_ctx = 1; @@ -1529,6 +1530,10 @@ int ptlrpc_expired_set(void *data) struct ptlrpc_request *req = list_entry(tmp, struct ptlrpc_request, rq_set_chain); + /* don't expire request waiting for context */ + if (req->rq_wait_ctx) + continue; + /* Request in-flight? */ if (!((req->rq_phase == RQ_PHASE_RPC && !req->rq_waiting && !req->rq_resend) || @@ -2181,9 +2186,11 @@ restart: */ spin_lock(&imp->imp_lock); list_del_init(&req->rq_list); + atomic_dec(&imp->imp_inflight); spin_unlock(&imp->imp_lock); - CERROR("Failed to refresh ctx of req %p: %d\n", req, rc); + CERROR("Failed to refresh ctx of req %p: %d\n", + req, rc); GOTO(out, rc); } /* simulating we got error during send rpc */ diff --git a/lustre/ptlrpc/gss/gss_cli_upcall.c b/lustre/ptlrpc/gss/gss_cli_upcall.c index 196fb9f..f3df6d2 100644 --- a/lustre/ptlrpc/gss/gss_cli_upcall.c +++ b/lustre/ptlrpc/gss/gss_cli_upcall.c @@ -248,8 +248,8 @@ int gss_do_ctx_init_rpc(__user char *buffer, unsigned long count) int rc; if (count != sizeof(param)) { - CERROR("ioctl size %lu, expect %lu, please check lgssd version\n", - count, (unsigned long) sizeof(param)); + CERROR("ioctl size %lu, expect %lu, please check lgss_keyring " + "version\n", count, (unsigned long) sizeof(param)); RETURN(-EINVAL); } if (copy_from_user(¶m, buffer, sizeof(param))) { @@ -275,11 +275,40 @@ int gss_do_ctx_init_rpc(__user char *buffer, unsigned long count) RETURN(-EINVAL); } + if (unlikely(!obd->obd_set_up)) { + CERROR("obd %s not setup\n", obdname); + RETURN(-EINVAL); + } + + spin_lock(&obd->obd_dev_lock); + if (obd->obd_stopping) { + CERROR("obd %s has stopped\n", obdname); + spin_unlock(&obd->obd_dev_lock); + RETURN(-EINVAL); + } + + if (strcmp(obd->obd_type->typ_name, LUSTRE_MDC_NAME) && + strcmp(obd->obd_type->typ_name, LUSTRE_OSC_NAME) && + strcmp(obd->obd_type->typ_name, LUSTRE_MGC_NAME)) { + CERROR("obd %s is not a client device\n", obdname); + spin_unlock(&obd->obd_dev_lock); + RETURN(-EINVAL); + } + spin_unlock(&obd->obd_dev_lock); + + down_read(&obd->u.cli.cl_sem); + if (obd->u.cli.cl_import == NULL) { + CERROR("import has gone\n"); + RETURN(-EINVAL); + } imp = class_import_get(obd->u.cli.cl_import); - LASSERT(imp->imp_sec); + up_read(&obd->u.cli.cl_sem); - /* force this import to use v2 msg */ - imp->imp_msg_magic = LUSTRE_MSG_MAGIC_V2; + if (imp->imp_deactive) { + CERROR("import has been deactivated\n"); + class_import_put(imp); + RETURN(-EINVAL); + } req = ptlrpc_request_alloc_pack(imp, &RQF_SEC_CTX, LUSTRE_OBD_VERSION, SEC_CTX_INIT); diff --git a/lustre/ptlrpc/gss/gss_keyring.c b/lustre/ptlrpc/gss/gss_keyring.c index 8301482..74c786d 100644 --- a/lustre/ptlrpc/gss/gss_keyring.c +++ b/lustre/ptlrpc/gss/gss_keyring.c @@ -1236,7 +1236,9 @@ int gss_kt_instantiate(struct key *key, const void *data, size_t datalen) */ LASSERT(cfs_current()->signal->session_keyring); + lockdep_off(); rc = key_link(cfs_current()->signal->session_keyring, key); + lockdep_on(); if (unlikely(rc)) { CERROR("failed to link key %08x to keyring %08x: %d\n", key->serial, @@ -1267,13 +1269,13 @@ int gss_kt_update(struct key *key, const void *data, size_t datalen) RETURN(-EINVAL); } - /* there's a race between userspace parent - child processes. if - * child finish negotiation too fast and call kt_update(), the ctx + /* if upcall finished negotiation too fast (mostly likely because + * of local error happened) and call kt_update(), the ctx * might be still NULL. but the key will finally be associate * with a context, or be revoked. if key status is fine, return * -EAGAIN to allow userspace sleep a while and call again. */ if (ctx == NULL) { - CWARN("race in userspace. key %p(%x) flags %lx\n", + CDEBUG(D_SEC, "update too soon: key %p(%x) flags %lx\n", key, key->serial, key->flags); rc = key_validate(key); diff --git a/lustre/ptlrpc/gss/gss_krb5.h b/lustre/ptlrpc/gss/gss_krb5.h index fa5db1a..feddfc5 100644 --- a/lustre/ptlrpc/gss/gss_krb5.h +++ b/lustre/ptlrpc/gss/gss_krb5.h @@ -49,8 +49,6 @@ #ifndef PTLRPC_GSS_KRB5_H #define PTLRPC_GSS_KRB5_H -extern spinlock_t krb5_seq_lock; - /* * RFC 4142 */ diff --git a/lustre/ptlrpc/gss/gss_krb5_mech.c b/lustre/ptlrpc/gss/gss_krb5_mech.c index 8d68ff5..a9a5388 100644 --- a/lustre/ptlrpc/gss/gss_krb5_mech.c +++ b/lustre/ptlrpc/gss/gss_krb5_mech.c @@ -77,7 +77,7 @@ #include "gss_asn1.h" #include "gss_krb5.h" -spinlock_t krb5_seq_lock = SPIN_LOCK_UNLOCKED; +static spinlock_t krb5_seq_lock; struct krb5_enctype { char *ke_dispname; @@ -1326,6 +1326,8 @@ int __init init_kerberos_module(void) { int status; + spin_lock_init(&krb5_seq_lock); + status = lgss_mech_register(&gss_kerberos_mech); if (status) CERROR("Failed to register kerberos gss mechanism!\n"); diff --git a/lustre/ptlrpc/gss/gss_pipefs.c b/lustre/ptlrpc/gss/gss_pipefs.c index cf96883..ade0f53 100644 --- a/lustre/ptlrpc/gss/gss_pipefs.c +++ b/lustre/ptlrpc/gss/gss_pipefs.c @@ -1210,7 +1210,7 @@ int __init gss_init_pipefs_upcall(void) de_pipes[MECH_KRB5] = de; CFS_INIT_LIST_HEAD(&upcall_lists[MECH_KRB5]); - upcall_locks[MECH_KRB5] = SPIN_LOCK_UNLOCKED; + spin_lock_init(&upcall_locks[MECH_KRB5]); return 0; } diff --git a/lustre/ptlrpc/gss/gss_svc_upcall.c b/lustre/ptlrpc/gss/gss_svc_upcall.c index 321ebfd..e97ae44 100644 --- a/lustre/ptlrpc/gss/gss_svc_upcall.c +++ b/lustre/ptlrpc/gss/gss_svc_upcall.c @@ -74,7 +74,7 @@ #define GSS_SVC_UPCALL_TIMEOUT (20) -static spinlock_t __ctx_index_lock = SPIN_LOCK_UNLOCKED; +static spinlock_t __ctx_index_lock; static __u64 __ctx_index; __u64 gss_get_next_ctx_index(void) @@ -1416,6 +1416,16 @@ int __init gss_init_svc_upcall(void) { int i; + spin_lock_init(&__ctx_index_lock); + /* + * this helps reducing context index confliction. after server reboot, + * conflicting request from clients might be filtered out by initial + * sequence number checking, thus no chance to sent error notification + * back to clients. + */ + get_random_bytes(&__ctx_index, sizeof(__ctx_index)); + + cache_register(&rsi_cache); cache_register(&rsc_cache); @@ -1436,12 +1446,6 @@ int __init gss_init_svc_upcall(void) CWARN("Init channel is not opened by lsvcgssd, following " "request might be dropped until lsvcgssd is active\n"); - /* this helps reducing context index confliction. after server reboot, - * conflicting request from clients might be filtered out by initial - * sequence number checking, thus no chance to sent error notification - * back to clients. */ - get_random_bytes(&__ctx_index, sizeof(__ctx_index)); - return 0; } diff --git a/lustre/ptlrpc/gss/lproc_gss.c b/lustre/ptlrpc/gss/lproc_gss.c index 208bad8..6398661 100644 --- a/lustre/ptlrpc/gss/lproc_gss.c +++ b/lustre/ptlrpc/gss/lproc_gss.c @@ -75,7 +75,6 @@ static struct { atomic_t oos_svc_replay[3]; /* server replay detected */ atomic_t oos_svc_pass[3]; /* server verified ok */ } gss_stat_oos = { - .oos_lock = SPIN_LOCK_UNLOCKED, .oos_cli_count = ATOMIC_INIT(0), .oos_cli_behind = 0, .oos_svc_replay = { ATOMIC_INIT(0), }, @@ -156,6 +155,8 @@ int gss_init_lproc(void) struct proc_dir_entry *ent; int rc; + spin_lock_init(&gss_stat_oos.oos_lock); + gss_proc_root = lprocfs_register("gss", sptlrpc_proc_root, gss_lprocfs_vars, NULL); diff --git a/lustre/ptlrpc/gss/sec_gss.c b/lustre/ptlrpc/gss/sec_gss.c index 1229091..f3aae3f 100644 --- a/lustre/ptlrpc/gss/sec_gss.c +++ b/lustre/ptlrpc/gss/sec_gss.c @@ -1087,7 +1087,7 @@ int gss_sec_create_common(struct gss_sec *gsec, sec->ps_id = sptlrpc_get_next_secid(); sec->ps_flvr = *sf; sec->ps_import = class_import_get(imp); - sec->ps_lock = SPIN_LOCK_UNLOCKED; + spin_lock_init(&sec->ps_lock); CFS_INIT_LIST_HEAD(&sec->ps_gc_list); if (!svcctx) { diff --git a/lustre/ptlrpc/niobuf.c b/lustre/ptlrpc/niobuf.c index 22f3565..786f929 100644 --- a/lustre/ptlrpc/niobuf.c +++ b/lustre/ptlrpc/niobuf.c @@ -343,6 +343,7 @@ static void ptlrpc_at_set_reply(struct ptlrpc_request *req, int flags) if (!(flags & PTLRPC_REPLY_EARLY) && (req->rq_type != PTL_RPC_MSG_ERR) && + (req->rq_reqmsg != NULL) && !(lustre_msg_get_flags(req->rq_reqmsg) & (MSG_RESENT | MSG_REPLAY | MSG_LAST_REPLAY))) { /* early replies, errors and recovery requests don't count diff --git a/lustre/ptlrpc/ptlrpc_internal.h b/lustre/ptlrpc/ptlrpc_internal.h index 03a2232..ea6704a 100644 --- a/lustre/ptlrpc/ptlrpc_internal.h +++ b/lustre/ptlrpc/ptlrpc_internal.h @@ -124,8 +124,17 @@ int sptlrpc_lproc_init(void); void sptlrpc_lproc_fini(void); /* sec_gc.c */ -int sptlrpc_gc_start_thread(void); -void sptlrpc_gc_stop_thread(void); +int sptlrpc_gc_init(void); +void sptlrpc_gc_fini(void); + +/* sec_config.c */ +void sptlrpc_conf_choose_flavor(enum lustre_sec_part from, + enum lustre_sec_part to, + struct obd_uuid *target, + lnet_nid_t nid, + struct sptlrpc_flavor *sf); +int sptlrpc_conf_init(void); +void sptlrpc_conf_fini(void); /* sec.c */ int __init sptlrpc_init(void); diff --git a/lustre/ptlrpc/sec.c b/lustre/ptlrpc/sec.c index 4f72364..87205d5 100644 --- a/lustre/ptlrpc/sec.c +++ b/lustre/ptlrpc/sec.c @@ -305,7 +305,7 @@ int sptlrpc_cli_ctx_display(struct ptlrpc_cli_ctx *ctx, char *buf, int bufsize) return ctx->cc_ops->display(ctx, buf, bufsize); } -static int sptlrpc_import_sec_check_expire(struct obd_import *imp) +static int import_sec_check_expire(struct obd_import *imp) { int adapt = 0; @@ -324,34 +324,47 @@ static int sptlrpc_import_sec_check_expire(struct obd_import *imp) return sptlrpc_import_sec_adapt(imp, NULL, 0); } -int sptlrpc_req_get_ctx(struct ptlrpc_request *req) +static int import_sec_validate_get(struct obd_import *imp, + struct ptlrpc_sec **sec) { - struct obd_import *imp = req->rq_import; - struct ptlrpc_sec *sec; - int rc; - ENTRY; - - LASSERT(!req->rq_cli_ctx); - LASSERT(imp); + int rc; if (unlikely(imp->imp_sec_expire)) { - rc = sptlrpc_import_sec_check_expire(imp); + rc = import_sec_check_expire(imp); if (rc) - RETURN(rc); + return rc; } - sec = sptlrpc_import_sec_ref(imp); - if (sec == NULL) { - CERROR("import %p (%s) with no ptlrpc_sec\n", + *sec = sptlrpc_import_sec_ref(imp); + if (*sec == NULL) { + CERROR("import %p (%s) with no sec\n", imp, ptlrpc_import_state_name(imp->imp_state)); - RETURN(-EACCES); + return -EACCES; } - if (unlikely(sec->ps_dying)) { + if (unlikely((*sec)->ps_dying)) { CERROR("attempt to use dying sec %p\n", sec); + sptlrpc_sec_put(*sec); return -EACCES; } + return 0; +} + +int sptlrpc_req_get_ctx(struct ptlrpc_request *req) +{ + struct obd_import *imp = req->rq_import; + struct ptlrpc_sec *sec; + int rc; + ENTRY; + + LASSERT(!req->rq_cli_ctx); + LASSERT(imp); + + rc = import_sec_validate_get(imp, &sec); + if (rc) + RETURN(rc); + req->rq_cli_ctx = get_my_ctx(sec); sptlrpc_sec_put(sec); @@ -447,9 +460,13 @@ int sptlrpc_req_ctx_switch(struct ptlrpc_request *req, return rc; } -/* +/** + * if current context has died, or if we resend after flavor switched, + * call this func to switch context. if no switch is needed, request + * will end up with the same context. + * * request must have a context. in any case of failure, restore the - * restore the old one. a request must have a ctx. + * restore the old one - a request must have a context. */ int sptlrpc_req_replace_dead_ctx(struct ptlrpc_request *req) { @@ -459,7 +476,6 @@ int sptlrpc_req_replace_dead_ctx(struct ptlrpc_request *req) ENTRY; LASSERT(oldctx); - LASSERT(test_bit(PTLRPC_CTX_DEAD_BIT, &oldctx->cc_flags)); sptlrpc_cli_ctx_get(oldctx); sptlrpc_req_put_ctx(req, 0); @@ -477,13 +493,16 @@ int sptlrpc_req_replace_dead_ctx(struct ptlrpc_request *req) LASSERT(newctx); if (unlikely(newctx == oldctx)) { - /* - * still get the old ctx, usually means system busy - */ - CWARN("ctx (%p, fl %lx) doesn't switch, relax a little bit\n", - newctx, newctx->cc_flags); - - cfs_schedule_timeout(CFS_TASK_INTERRUPTIBLE, HZ); + if (test_bit(PTLRPC_CTX_DEAD_BIT, &oldctx->cc_flags)) { + /* + * still get the old ctx, usually means system busy + */ + CWARN("ctx (%p, fl %lx) doesn't switch, " + "relax a little bit\n", + newctx, newctx->cc_flags); + + cfs_schedule_timeout(CFS_TASK_INTERRUPTIBLE, HZ); + } } else { rc = sptlrpc_req_ctx_switch(req, oldctx, newctx); if (rc) { @@ -564,23 +583,35 @@ void req_off_ctx_list(struct ptlrpc_request *req, struct ptlrpc_cli_ctx *ctx) int sptlrpc_req_refresh_ctx(struct ptlrpc_request *req, long timeout) { struct ptlrpc_cli_ctx *ctx = req->rq_cli_ctx; + struct ptlrpc_sec *sec; struct l_wait_info lwi; int rc; ENTRY; LASSERT(ctx); + if (req->rq_ctx_init || req->rq_ctx_fini) + RETURN(0); + /* * during the process a request's context might change type even * (e.g. from gss ctx to plain ctx), so each loop we need to re-check * everything */ again: - /* skip special ctxs */ - if (cli_ctx_is_eternal(ctx) || req->rq_ctx_init || req->rq_ctx_fini) + rc = import_sec_validate_get(req->rq_import, &sec); + if (rc) + RETURN(rc); + + if (sec->ps_flvr.sf_rpc != req->rq_flvr.sf_rpc) + sptlrpc_req_replace_dead_ctx(req); + + sptlrpc_sec_put(sec); + + if (cli_ctx_is_eternal(ctx)) RETURN(0); - if (test_bit(PTLRPC_CTX_NEW_BIT, &ctx->cc_flags)) { + if (unlikely(test_bit(PTLRPC_CTX_NEW_BIT, &ctx->cc_flags))) { LASSERT(ctx->cc_ops->refresh); ctx->cc_ops->refresh(ctx); } @@ -630,6 +661,15 @@ again: } if (unlikely(test_bit(PTLRPC_CTX_DEAD_BIT, &ctx->cc_flags))) { + /* + * don't switch ctx if import was deactivated + */ + if (req->rq_import->imp_deactive) { + req_off_ctx_list(req, ctx); + req->rq_err = 1; + RETURN(-EINTR); + } + rc = sptlrpc_req_replace_dead_ctx(req); if (rc) { LASSERT(ctx == req->rq_cli_ctx); @@ -659,9 +699,8 @@ again: list_add(&req->rq_ctx_chain, &ctx->cc_req_list); spin_unlock(&ctx->cc_lock); - if (timeout < 0) { + if (timeout < 0) RETURN(-EWOULDBLOCK); - } /* Clear any flags that may be present from previous sends */ LASSERT(req->rq_receiving_reply == 0); @@ -794,7 +833,7 @@ int sptlrpc_import_check_ctx(struct obd_import *imp) sptlrpc_sec_put(sec); if (!ctx) - RETURN(1); + RETURN(-ENOMEM); if (cli_ctx_is_eternal(ctx) || ctx->cc_ops->validate(ctx) == 0) { @@ -802,6 +841,11 @@ int sptlrpc_import_check_ctx(struct obd_import *imp) RETURN(0); } + if (cli_ctx_is_error(ctx)) { + sptlrpc_cli_ctx_put(ctx, 1); + RETURN(-EACCES); + } + OBD_ALLOC_PTR(req); if (!req) RETURN(-ENOMEM); @@ -811,6 +855,7 @@ int sptlrpc_import_check_ctx(struct obd_import *imp) CFS_INIT_LIST_HEAD(&req->rq_ctx_chain); cfs_waitq_init(&req->rq_reply_waitq); req->rq_import = imp; + req->rq_flvr = sec->ps_flvr; req->rq_cli_ctx = ctx; rc = sptlrpc_req_refresh_ctx(req, 0); @@ -1275,17 +1320,21 @@ int sptlrpc_import_sec_adapt(struct obd_import *imp, enum lustre_sec_part sp; int rc; + might_sleep(); + if (imp == NULL) return 0; conn = imp->imp_connection; if (svc_ctx == NULL) { + struct client_obd *cliobd = &imp->imp_obd->u.cli; /* normal import, determine flavor from rule set */ - sptlrpc_rule_set_choose(&imp->imp_obd->u.cli.cl_sptlrpc_rset, - LUSTRE_SP_ANY, conn->c_self, &sf); + sptlrpc_conf_choose_flavor(cliobd->cl_sp_me, cliobd->cl_sp_to, + &cliobd->cl_target_uuid, + conn->c_self, &sf); - sp = imp->imp_obd->u.cli.cl_sec_part; + sp = imp->imp_obd->u.cli.cl_sp_me; } else { /* reverse import, determine flavor from incoming reqeust */ sf.sf_rpc = rpc_flavor; @@ -1770,11 +1819,23 @@ int sptlrpc_target_export_check(struct obd_export *exp, spin_unlock(&exp->exp_lock); - CWARN("req %p: (%u|%u|%u|%u|%u) with unauthorized flavor %x\n", + CWARN("exp %p(%s): req %p (%u|%u|%u|%u|%u) with " + "unauthorized flavor %x, expect %x|%x(%+ld)|%x(%+ld)\n", + exp, exp->exp_obd->obd_name, req, req->rq_auth_gss, req->rq_ctx_init, req->rq_ctx_fini, - req->rq_auth_usr_root, req->rq_auth_usr_mdt, req->rq_flvr.sf_rpc); + req->rq_auth_usr_root, req->rq_auth_usr_mdt, req->rq_flvr.sf_rpc, + exp->exp_flvr.sf_rpc, + exp->exp_flvr_old[0].sf_rpc, + exp->exp_flvr_expire[0] ? + (unsigned long) (exp->exp_flvr_expire[0] - + cfs_time_current_sec()) : 0, + exp->exp_flvr_old[1].sf_rpc, + exp->exp_flvr_expire[1] ? + (unsigned long) (exp->exp_flvr_expire[1] - + cfs_time_current_sec()) : 0); return -EACCES; } +EXPORT_SYMBOL(sptlrpc_target_export_check); void sptlrpc_target_update_exp_flavor(struct obd_device *obd, struct sptlrpc_rule_set *rset) @@ -1794,15 +1855,16 @@ void sptlrpc_target_update_exp_flavor(struct obd_device *obd, * (exp_flvr_changed == 1), this will override the * previous one. */ spin_lock(&exp->exp_lock); - sptlrpc_rule_set_choose(rset, exp->exp_sp_peer, - exp->exp_connection->c_peer.nid, - &new_flvr); + sptlrpc_target_choose_flavor(rset, exp->exp_sp_peer, + exp->exp_connection->c_peer.nid, + &new_flvr); if (exp->exp_flvr_changed || memcmp(&new_flvr, &exp->exp_flvr, sizeof(new_flvr))) { exp->exp_flvr_old[1] = new_flvr; exp->exp_flvr_expire[1] = 0; exp->exp_flvr_changed = 1; exp->exp_flvr_adapt = 1; + CDEBUG(D_SEC, "exp %p (%s): updated flavor %x->%x\n", exp, sptlrpc_part2name(exp->exp_sp_peer), exp->exp_flvr.sf_rpc, @@ -1824,6 +1886,7 @@ static int sptlrpc_svc_check_from(struct ptlrpc_request *req, int svc_rc) case LUSTRE_SP_CLI: case LUSTRE_SP_MDT: case LUSTRE_SP_OST: + case LUSTRE_SP_MGC: case LUSTRE_SP_MGS: case LUSTRE_SP_ANY: break; @@ -2281,14 +2344,18 @@ int __init sptlrpc_init(void) rwlock_init(&policy_lock); - rc = sptlrpc_gc_start_thread(); + rc = sptlrpc_gc_init(); if (rc) goto out; - rc = sptlrpc_enc_pool_init(); + rc = sptlrpc_conf_init(); if (rc) goto out_gc; + rc = sptlrpc_enc_pool_init(); + if (rc) + goto out_conf; + rc = sptlrpc_null_init(); if (rc) goto out_pool; @@ -2309,8 +2376,10 @@ out_null: sptlrpc_null_fini(); out_pool: sptlrpc_enc_pool_fini(); +out_conf: + sptlrpc_conf_fini(); out_gc: - sptlrpc_gc_stop_thread(); + sptlrpc_gc_fini(); out: return rc; } @@ -2321,5 +2390,6 @@ void __exit sptlrpc_fini(void) sptlrpc_plain_fini(); sptlrpc_null_fini(); sptlrpc_enc_pool_fini(); - sptlrpc_gc_stop_thread(); + sptlrpc_conf_fini(); + sptlrpc_gc_fini(); } diff --git a/lustre/ptlrpc/sec_config.c b/lustre/ptlrpc/sec_config.c index 67287e8..84f926c 100644 --- a/lustre/ptlrpc/sec_config.c +++ b/lustre/ptlrpc/sec_config.c @@ -53,7 +53,10 @@ #include #include #include +#include +#include #include +#include #include #include "ptlrpc_internal.h" @@ -67,6 +70,8 @@ const char *sptlrpc_part2name(enum lustre_sec_part part) return "mdt"; case LUSTRE_SP_OST: return "ost"; + case LUSTRE_SP_MGC: + return "mgc"; case LUSTRE_SP_MGS: return "mgs"; case LUSTRE_SP_ANY: @@ -117,11 +122,11 @@ static void get_default_flavor(struct sptlrpc_flavor *sf) sf->sf_flags = 0; } -static void get_flavor_by_rpc(struct sptlrpc_rule *rule, __u16 rpc_flavor) +static void get_flavor_by_rpc(struct sptlrpc_flavor *flvr, __u16 rpc_flavor) { - get_default_flavor(&rule->sr_flvr); + get_default_flavor(flvr); - rule->sr_flvr.sf_rpc = rpc_flavor; + flvr->sf_rpc = rpc_flavor; switch (rpc_flavor) { case SPTLRPC_FLVR_NULL: @@ -129,46 +134,46 @@ static void get_flavor_by_rpc(struct sptlrpc_rule *rule, __u16 rpc_flavor) case SPTLRPC_FLVR_PLAIN: case SPTLRPC_FLVR_KRB5N: case SPTLRPC_FLVR_KRB5A: - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_DEFAULT; + flvr->sf_bulk_hash = BULK_HASH_ALG_DEFAULT; break; case SPTLRPC_FLVR_KRB5P: - rule->sr_flvr.sf_bulk_ciph = BULK_CIPH_ALG_AES128; + flvr->sf_bulk_ciph = BULK_CIPH_ALG_AES128; /* fall through */ case SPTLRPC_FLVR_KRB5I: - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_SHA1; + flvr->sf_bulk_hash = BULK_HASH_ALG_SHA1; break; default: LBUG(); } } -static void get_flavor_by_bulk(struct sptlrpc_rule *rule, +static void get_flavor_by_bulk(struct sptlrpc_flavor *flvr, __u16 rpc_flavor, bulk_type_t bulk_type) { switch (bulk_type) { case BULK_TYPE_N: - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_NULL; - rule->sr_flvr.sf_bulk_ciph = BULK_CIPH_ALG_NULL; + flvr->sf_bulk_hash = BULK_HASH_ALG_NULL; + flvr->sf_bulk_ciph = BULK_CIPH_ALG_NULL; break; case BULK_TYPE_I: switch (rpc_flavor) { case SPTLRPC_FLVR_PLAIN: case SPTLRPC_FLVR_KRB5N: case SPTLRPC_FLVR_KRB5A: - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_DEFAULT; + flvr->sf_bulk_hash = BULK_HASH_ALG_DEFAULT; break; case SPTLRPC_FLVR_KRB5I: case SPTLRPC_FLVR_KRB5P: - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_SHA1; + flvr->sf_bulk_hash = BULK_HASH_ALG_SHA1; break; default: LBUG(); } - rule->sr_flvr.sf_bulk_ciph = BULK_CIPH_ALG_NULL; + flvr->sf_bulk_ciph = BULK_CIPH_ALG_NULL; break; case BULK_TYPE_P: - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_SHA1; - rule->sr_flvr.sf_bulk_ciph = BULK_CIPH_ALG_AES128; + flvr->sf_bulk_hash = BULK_HASH_ALG_SHA1; + flvr->sf_bulk_ciph = BULK_CIPH_ALG_AES128; break; default: LBUG(); @@ -195,7 +200,7 @@ static __u16 __flavors[] = { * krb5i-bulkp * krb5i-bulkp:sha512/arc4 */ -static int parse_flavor(char *str, struct sptlrpc_rule *rule) +static int parse_flavor(const char *str, struct sptlrpc_flavor *flvr) { const char *f; char *bulk, *alg, *enc; @@ -205,7 +210,7 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) ENTRY; if (str == NULL || str[0] == '\0') { - rule->sr_flvr.sf_rpc = SPTLRPC_FLVR_INVALID; + flvr->sf_rpc = SPTLRPC_FLVR_INVALID; goto out; } @@ -231,7 +236,7 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) if (strcmp(buf, f) != 0) GOTO(invalid, -EINVAL); - get_flavor_by_rpc(rule, __flavors[i]); + get_flavor_by_rpc(flvr, __flavors[i]); if (bulk == NULL) goto out; @@ -243,8 +248,8 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) /* verify bulk section */ if (strcmp(bulk, "bulkn") == 0) { - rule->sr_flvr.sf_bulk_hash = BULK_HASH_ALG_NULL; - rule->sr_flvr.sf_bulk_ciph = BULK_CIPH_ALG_NULL; + flvr->sf_bulk_hash = BULK_HASH_ALG_NULL; + flvr->sf_bulk_ciph = BULK_CIPH_ALG_NULL; bulk_type = BULK_TYPE_N; } else if (strcmp(bulk, "bulki") == 0) bulk_type = BULK_TYPE_I; @@ -261,7 +266,7 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) if (__flavors[i] == SPTLRPC_FLVR_PLAIN && bulk_type == BULK_TYPE_P) GOTO(invalid, -EINVAL); - get_flavor_by_bulk(rule, __flavors[i], bulk_type); + get_flavor_by_bulk(flvr, __flavors[i], bulk_type); if (alg == NULL) goto out; @@ -274,7 +279,7 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) /* checksum algorithm */ for (i = 0; i < BULK_HASH_ALG_MAX; i++) { if (strcmp(alg, sptlrpc_get_hash_name(i)) == 0) { - rule->sr_flvr.sf_bulk_hash = i; + flvr->sf_bulk_hash = i; break; } } @@ -285,7 +290,7 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) if (enc) { for (i = 0; i < BULK_CIPH_ALG_MAX; i++) { if (strcmp(enc, sptlrpc_get_ciph_name(i)) == 0) { - rule->sr_flvr.sf_bulk_ciph = i; + flvr->sf_bulk_ciph = i; break; } } @@ -297,17 +302,17 @@ static int parse_flavor(char *str, struct sptlrpc_rule *rule) * bulk combination sanity checks */ if (bulk_type == BULK_TYPE_P && - rule->sr_flvr.sf_bulk_ciph == BULK_CIPH_ALG_NULL) + flvr->sf_bulk_ciph == BULK_CIPH_ALG_NULL) GOTO(invalid, -EINVAL); if (bulk_type == BULK_TYPE_I && - (rule->sr_flvr.sf_bulk_hash == BULK_HASH_ALG_NULL || - rule->sr_flvr.sf_bulk_ciph != BULK_CIPH_ALG_NULL)) + (flvr->sf_bulk_hash == BULK_HASH_ALG_NULL || + flvr->sf_bulk_ciph != BULK_CIPH_ALG_NULL)) GOTO(invalid, -EINVAL); if (bulk_type == BULK_TYPE_N && - (rule->sr_flvr.sf_bulk_hash != BULK_HASH_ALG_NULL || - rule->sr_flvr.sf_bulk_ciph != BULK_CIPH_ALG_NULL)) + (flvr->sf_bulk_hash != BULK_HASH_ALG_NULL || + flvr->sf_bulk_ciph != BULK_CIPH_ALG_NULL)) GOTO(invalid, -EINVAL); out: @@ -382,7 +387,7 @@ int sptlrpc_parse_rule(char *param, struct sptlrpc_rule *rule) } /* 2.1 flavor */ - rc = parse_flavor(flavor, rule); + rc = parse_flavor(flavor, &rule->sr_flvr); if (rc) RETURN(-EINVAL); @@ -418,22 +423,21 @@ int sptlrpc_rule_set_expand(struct sptlrpc_rule_set *rset, int expand) if (expand == 0) return -E2BIG; - if (rset->srs_nslot == 0) - nslot = 8; - else - nslot = rset->srs_nslot + 8; + nslot = rset->srs_nslot + 8; /* better use realloc() if available */ OBD_ALLOC(rules, nslot * sizeof(*rset->srs_rules)); if (rules == NULL) return -ENOMEM; - memcpy(rules, rset->srs_rules, - rset->srs_nrule * sizeof(*rset->srs_rules)); + if (rset->srs_nrule) { + LASSERT(rset->srs_nslot && rset->srs_rules); + memcpy(rules, rset->srs_rules, + rset->srs_nrule * sizeof(*rset->srs_rules)); - if (rset->srs_rules) OBD_FREE(rset->srs_rules, rset->srs_nslot * sizeof(*rset->srs_rules)); + } rset->srs_rules = rules; rset->srs_nslot = nslot; @@ -543,7 +547,7 @@ int sptlrpc_rule_set_merge(struct sptlrpc_rule_set *rset, memcpy(&rset->srs_rules[n], rule, sizeof(*rule)); rset->srs_nrule++; } else { - CWARN("ignore the unmatched deletion\n"); + CDEBUG(D_CONFIG, "ignore the unmatched deletion\n"); } } @@ -551,35 +555,15 @@ int sptlrpc_rule_set_merge(struct sptlrpc_rule_set *rset, } EXPORT_SYMBOL(sptlrpc_rule_set_merge); -int sptlrpc_rule_set_from_log(struct sptlrpc_rule_set *rset, - struct sptlrpc_conf_log *log) -{ - LASSERT(rset); - LASSERT(log); - - sptlrpc_rule_set_free(rset); - - if (log->scl_nrule == 0) - return 0; - - OBD_ALLOC(rset->srs_rules, log->scl_nrule * sizeof(*log->scl_rules)); - if (!rset->srs_rules) - return -ENOMEM; - - memcpy(rset->srs_rules, log->scl_rules, - log->scl_nrule * sizeof(*log->scl_rules)); - rset->srs_nslot = rset->srs_nrule = log->scl_nrule; - return 0; -} -EXPORT_SYMBOL(sptlrpc_rule_set_from_log); - -/* - * according to NID/from choose a flavor from rule set. +/** + * given from/to/nid, determine a matching flavor in ruleset. + * return 1 if a match found, otherwise return 0. */ -void sptlrpc_rule_set_choose(struct sptlrpc_rule_set *rset, - enum lustre_sec_part from, - lnet_nid_t nid, - struct sptlrpc_flavor *sf) +static int sptlrpc_rule_set_choose(struct sptlrpc_rule_set *rset, + enum lustre_sec_part from, + enum lustre_sec_part to, + lnet_nid_t nid, + struct sptlrpc_flavor *sf) { struct sptlrpc_rule *r; int n; @@ -596,14 +580,16 @@ void sptlrpc_rule_set_choose(struct sptlrpc_rule_set *rset, from != r->sr_from) continue; + if (to != LUSTRE_SP_ANY && r->sr_to != LUSTRE_SP_ANY && + to != r->sr_to) + continue; + *sf = r->sr_flvr; - return; + return 1; } - /* no match found, set as default flavor */ - get_default_flavor(sf); + return 0; } -EXPORT_SYMBOL(sptlrpc_rule_set_choose); void sptlrpc_rule_set_dump(struct sptlrpc_rule_set *rset) { @@ -618,71 +604,16 @@ void sptlrpc_rule_set_dump(struct sptlrpc_rule_set *rset) } EXPORT_SYMBOL(sptlrpc_rule_set_dump); -/**************************************** - * sptlrpc config log * - ****************************************/ - -struct sptlrpc_conf_log *sptlrpc_conf_log_alloc(void) -{ - struct sptlrpc_conf_log *log; - - OBD_ALLOC_PTR(log); - if (log == NULL) - return ERR_PTR(-ENOMEM); - - log->scl_max = SPTLRPC_CONF_LOG_MAX; - return log; -} -EXPORT_SYMBOL(sptlrpc_conf_log_alloc); - -void sptlrpc_conf_log_free(struct sptlrpc_conf_log *log) -{ - LASSERT(log->scl_max == SPTLRPC_CONF_LOG_MAX); - OBD_FREE_PTR(log); -} -EXPORT_SYMBOL(sptlrpc_conf_log_free); - -static __u32 get_log_rule_flags(enum lustre_sec_part from, - enum lustre_sec_part to, - unsigned int fl_udesc) -{ - /* MDT->MDT; MDT->OST */ - if (from == LUSTRE_SP_MDT) - return PTLRPC_SEC_FL_ROOTONLY; - /* CLI->OST */ - if (from == LUSTRE_SP_CLI && to == LUSTRE_SP_OST) - return PTLRPC_SEC_FL_ROOTONLY | PTLRPC_SEC_FL_BULK; - /* CLI->MDT */ - if (from == LUSTRE_SP_CLI && to == LUSTRE_SP_MDT) - if (fl_udesc) - return PTLRPC_SEC_FL_UDESC; - - return 0; -} - -/* - * generate config log: merge general and target rules, which - * match @from @to - */ -int sptlrpc_conf_log_populate(struct sptlrpc_rule_set *gen, - struct sptlrpc_rule_set *tgt, - enum lustre_sec_part from, - enum lustre_sec_part to, - unsigned int fl_udesc, - struct sptlrpc_conf_log *log) +static int sptlrpc_rule_set_extract(struct sptlrpc_rule_set *gen, + struct sptlrpc_rule_set *tgt, + enum lustre_sec_part from, + enum lustre_sec_part to, + struct sptlrpc_rule_set *rset) { struct sptlrpc_rule_set *src[2] = { gen, tgt }; - struct sptlrpc_rule_set dst; struct sptlrpc_rule *rule; - __u32 flags; int i, n, rc; - LASSERT(log); - - dst.srs_nslot = log->scl_max; - dst.srs_nrule = 0; - dst.srs_rules = log->scl_rules; - /* merge general rules firstly, then target-specific rules */ for (i = 0; i < 2; i++) { if (src[i] == NULL) @@ -700,7 +631,7 @@ int sptlrpc_conf_log_populate(struct sptlrpc_rule_set *gen, rule->sr_to != to) continue; - rc = sptlrpc_rule_set_merge(&dst, rule, 0); + rc = sptlrpc_rule_set_merge(rset, rule, 1); if (rc) { CERROR("can't merge: %d\n", rc); return rc; @@ -708,125 +639,755 @@ int sptlrpc_conf_log_populate(struct sptlrpc_rule_set *gen, } } - log->scl_nrule = dst.srs_nrule; + return 0; +} + +/********************************** + * sptlrpc configuration support * + **********************************/ + +struct sptlrpc_conf_tgt { + struct list_head sct_list; + char sct_name[MAX_OBD_NAME]; + struct sptlrpc_rule_set sct_rset; +}; + +struct sptlrpc_conf { + struct list_head sc_list; + char sc_fsname[MTI_NAME_MAXLEN]; + unsigned int sc_modified; /* modified during updating */ + unsigned int sc_updated:1, /* updated copy from MGS */ + sc_local:1; /* local copy from target */ + struct sptlrpc_rule_set sc_rset; /* fs general rules */ + struct list_head sc_tgts; /* target-specific rules */ +}; - /* set flags for each rule */ - flags = get_log_rule_flags(from, to, fl_udesc); +static struct mutex sptlrpc_conf_lock; +static CFS_LIST_HEAD(sptlrpc_confs); - for (i = 0; i < log->scl_nrule; i++) { - log->scl_rules[i].sr_flvr.sf_flags = flags; +static inline int is_hex(char c) +{ + return ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f')); +} - /* also clear the from/to fields which don't need to be known - * accordingly. @from == ANY means this log is for target, - * otherwise for client. */ - if (from != LUSTRE_SP_ANY) - log->scl_rules[i].sr_from = LUSTRE_SP_ANY; - log->scl_rules[i].sr_to = LUSTRE_SP_ANY; +static void target2fsname(const char *tgt, char *fsname, int buflen) +{ + const char *ptr; + int len; + + ptr = strrchr(tgt, '-'); + if (ptr) { + if ((strncmp(ptr, "-MDT", 4) != 0 && + strncmp(ptr, "-OST", 4) != 0) || + !is_hex(ptr[4]) || !is_hex(ptr[5]) || + !is_hex(ptr[6]) || !is_hex(ptr[7])) + ptr = NULL; } - return 0; + /* if we didn't find the pattern, treat the whole string as fsname */ + if (ptr == NULL) + len = strlen(tgt); + else + len = ptr - tgt; + + len = min(len, buflen - 1); + memcpy(fsname, tgt, len); + fsname[len] = '\0'; } -EXPORT_SYMBOL(sptlrpc_conf_log_populate); -/* - * extract config log from @lcfg +static void sptlrpc_conf_free_rsets(struct sptlrpc_conf *conf) +{ + struct sptlrpc_conf_tgt *conf_tgt, *conf_tgt_next; + + sptlrpc_rule_set_free(&conf->sc_rset); + + list_for_each_entry_safe(conf_tgt, conf_tgt_next, + &conf->sc_tgts, sct_list) { + sptlrpc_rule_set_free(&conf_tgt->sct_rset); + list_del(&conf_tgt->sct_list); + OBD_FREE_PTR(conf_tgt); + } + LASSERT(list_empty(&conf->sc_tgts)); + + conf->sc_updated = 0; + conf->sc_local = 0; +} + +static void sptlrpc_conf_free(struct sptlrpc_conf *conf) +{ + CDEBUG(D_SEC, "free sptlrpc conf %s\n", conf->sc_fsname); + + sptlrpc_conf_free_rsets(conf); + list_del(&conf->sc_list); + OBD_FREE_PTR(conf); +} + +static +struct sptlrpc_conf_tgt *sptlrpc_conf_get_tgt(struct sptlrpc_conf *conf, + const char *name, + int create) +{ + struct sptlrpc_conf_tgt *conf_tgt; + + list_for_each_entry(conf_tgt, &conf->sc_tgts, sct_list) { + if (strcmp(conf_tgt->sct_name, name) == 0) + return conf_tgt; + } + + if (!create) + return NULL; + + OBD_ALLOC_PTR(conf_tgt); + if (conf_tgt) { + strncpy(conf_tgt->sct_name, name, sizeof(conf_tgt->sct_name)); + sptlrpc_rule_set_init(&conf_tgt->sct_rset); + list_add(&conf_tgt->sct_list, &conf->sc_tgts); + } + + return conf_tgt; +} + +static +struct sptlrpc_conf *sptlrpc_conf_get(const char *fsname, + int create) +{ + struct sptlrpc_conf *conf; + + list_for_each_entry(conf, &sptlrpc_confs, sc_list) { + if (strcmp(conf->sc_fsname, fsname) == 0) + return conf; + } + + if (!create) + return NULL; + + OBD_ALLOC_PTR(conf); + if (conf == NULL) + return NULL; + + strcpy(conf->sc_fsname, fsname); + sptlrpc_rule_set_init(&conf->sc_rset); + CFS_INIT_LIST_HEAD(&conf->sc_tgts); + list_add(&conf->sc_list, &sptlrpc_confs); + + CDEBUG(D_SEC, "create sptlrpc conf %s\n", conf->sc_fsname); + return conf; +} + +/** + * caller must hold conf_lock already. */ -struct sptlrpc_conf_log *sptlrpc_conf_log_extract(struct lustre_cfg *lcfg) +static int sptlrpc_conf_merge_rule(struct sptlrpc_conf *conf, + const char *target, + struct sptlrpc_rule *rule) { - struct sptlrpc_conf_log *log; - struct sptlrpc_rule *r; - int i; + struct sptlrpc_conf_tgt *conf_tgt; + struct sptlrpc_rule_set *rule_set; + + /* fsname == target means general rules for the whole fs */ + if (strcmp(conf->sc_fsname, target) == 0) { + rule_set = &conf->sc_rset; + } else { + conf_tgt = sptlrpc_conf_get_tgt(conf, target, 1); + if (conf_tgt) { + rule_set = &conf_tgt->sct_rset; + } else { + CERROR("out of memory, can't merge rule!\n"); + return -ENOMEM; + } + } + + return sptlrpc_rule_set_merge(rule_set, rule, 1); +} + +/** + * process one LCFG_SPTLRPC_CONF record. if \a conf is NULL, we + * find one through the target name in the record inside conf_lock; + * otherwise means caller already hold conf_lock. + */ +static int __sptlrpc_process_config(struct lustre_cfg *lcfg, + struct sptlrpc_conf *conf) +{ + char *target, *param; + char fsname[MTI_NAME_MAXLEN]; + struct sptlrpc_rule rule; + int rc; ENTRY; - log = lustre_cfg_buf(lcfg, 1); - if (log == NULL) { - CERROR("no sptlrpc config data\n"); - RETURN(ERR_PTR(-EINVAL)); + target = lustre_cfg_string(lcfg, 1); + if (target == NULL) { + CERROR("missing target name\n"); + RETURN(-EINVAL); } - if (lcfg->lcfg_version == __swab32(LUSTRE_CFG_VERSION)) { - __swab32s(&log->scl_max); - __swab32s(&log->scl_nrule); + param = lustre_cfg_string(lcfg, 2); + if (param == NULL) { + CERROR("missing parameter\n"); + RETURN(-EINVAL); } - if (LUSTRE_CFG_BUFLEN(lcfg, 1) < - log->scl_max * sizeof(log->scl_rules[0])) { - CERROR("mal-formed config log\n"); - RETURN(ERR_PTR(-EINVAL)); + /* parse rule to make sure the format is correct */ + if (strncmp(param, PARAM_SRPC_FLVR, sizeof(PARAM_SRPC_FLVR) - 1) != 0) { + CERROR("Invalid sptlrpc parameter: %s\n", param); + RETURN(-EINVAL); } + param += sizeof(PARAM_SRPC_FLVR) - 1; + + rc = sptlrpc_parse_rule(param, &rule); + if (rc) + RETURN(-EINVAL); + + if (conf == NULL) { + target2fsname(target, fsname, sizeof(fsname)); - if (lcfg->lcfg_version == __swab32(LUSTRE_CFG_VERSION)) { - for (i = 0; i < log->scl_nrule; i++) { - r = &log->scl_rules[i]; - __swab32s(&r->sr_netid); - __swab16s(&r->sr_flvr.sf_rpc); - __swab32s(&r->sr_flvr.sf_flags); + mutex_lock(&sptlrpc_conf_lock); + conf = sptlrpc_conf_get(fsname, 0); + if (conf == NULL) { + CERROR("can't find conf\n"); + rc = -ENOMEM; + } else { + rc = sptlrpc_conf_merge_rule(conf, target, &rule); } + mutex_unlock(&sptlrpc_conf_lock); + } else { + LASSERT(mutex_is_locked(&sptlrpc_conf_lock)); + rc = sptlrpc_conf_merge_rule(conf, target, &rule); } - RETURN(log); + if (rc == 0) + conf->sc_modified++; + + RETURN(rc); } -EXPORT_SYMBOL(sptlrpc_conf_log_extract); -void sptlrpc_conf_log_cleanup(struct sptlrpc_conf_log *log) +int sptlrpc_process_config(struct lustre_cfg *lcfg) { - log->scl_nrule = 0; - memset(log->scl_rules, 0, sizeof(log->scl_rules)); + return __sptlrpc_process_config(lcfg, NULL); } -EXPORT_SYMBOL(sptlrpc_conf_log_cleanup); +EXPORT_SYMBOL(sptlrpc_process_config); -void sptlrpc_conf_log_dump(struct sptlrpc_conf_log *log) +static int logname2fsname(const char *logname, char *buf, int buflen) { - struct sptlrpc_rule *r; - int n; + char *ptr; + int len; + + ptr = strrchr(logname, '-'); + if (ptr == NULL || strcmp(ptr, "-sptlrpc")) { + CERROR("%s is not a sptlrpc config log\n", logname); + return -EINVAL; + } - CWARN("max %u, rule# %u part %u\n", - log->scl_max, log->scl_nrule, log->scl_part); + len = min((int) (ptr - logname), buflen - 1); - for (n = 0; n < log->scl_nrule; n++) { - r = &log->scl_rules[n]; - CWARN("<%02d> %x -> %x, net %x, rpc %x\n", n, - r->sr_from, r->sr_to, r->sr_netid, r->sr_flvr.sf_rpc); + memcpy(buf, logname, len); + buf[len] = '\0'; + return 0; +} + +void sptlrpc_conf_log_update_begin(const char *logname) +{ + struct sptlrpc_conf *conf; + char fsname[16]; + + if (logname2fsname(logname, fsname, sizeof(fsname))) + return; + + mutex_lock(&sptlrpc_conf_lock); + + conf = sptlrpc_conf_get(fsname, 0); + if (conf && conf->sc_local) { + LASSERT(conf->sc_updated == 0); + sptlrpc_conf_free_rsets(conf); } + conf->sc_modified = 0; + + mutex_unlock(&sptlrpc_conf_lock); } -EXPORT_SYMBOL(sptlrpc_conf_log_dump); +EXPORT_SYMBOL(sptlrpc_conf_log_update_begin); -/* - * caller should guarantee that no concurrent calls to this function +/** + * mark a config log has been updated */ +void sptlrpc_conf_log_update_end(const char *logname) +{ + struct sptlrpc_conf *conf; + char fsname[16]; + + if (logname2fsname(logname, fsname, sizeof(fsname))) + return; + + mutex_lock(&sptlrpc_conf_lock); + + conf = sptlrpc_conf_get(fsname, 0); + if (conf) { + /* + * if original state is not updated, make sure the + * modified counter > 0 to enforce updating local copy. + */ + if (conf->sc_updated == 0) + conf->sc_modified++; + + conf->sc_updated = 1; + } + + mutex_unlock(&sptlrpc_conf_lock); +} +EXPORT_SYMBOL(sptlrpc_conf_log_update_end); + +void sptlrpc_conf_log_start(const char *logname) +{ + struct sptlrpc_conf *conf; + char fsname[16]; + + if (logname2fsname(logname, fsname, sizeof(fsname))) + return; + + mutex_lock(&sptlrpc_conf_lock); + conf = sptlrpc_conf_get(fsname, 1); + mutex_unlock(&sptlrpc_conf_lock); +} +EXPORT_SYMBOL(sptlrpc_conf_log_start); + +void sptlrpc_conf_log_stop(const char *logname) +{ + struct sptlrpc_conf *conf; + char fsname[16]; + + if (logname2fsname(logname, fsname, sizeof(fsname))) + return; + + mutex_lock(&sptlrpc_conf_lock); + conf = sptlrpc_conf_get(fsname, 0); + if (conf) + sptlrpc_conf_free(conf); + mutex_unlock(&sptlrpc_conf_lock); +} +EXPORT_SYMBOL(sptlrpc_conf_log_stop); + +static void inline flavor_set_flags(struct sptlrpc_flavor *sf, + enum lustre_sec_part from, + enum lustre_sec_part to, + unsigned int fl_udesc) +{ + if (from == LUSTRE_SP_MDT) { + /* MDT->MDT; MDT->OST */ + sf->sf_flags |= PTLRPC_SEC_FL_ROOTONLY; + } else if (from == LUSTRE_SP_CLI && to == LUSTRE_SP_OST) { + /* CLI->OST */ + sf->sf_flags |= PTLRPC_SEC_FL_ROOTONLY | PTLRPC_SEC_FL_BULK; + } else if (from == LUSTRE_SP_CLI && to == LUSTRE_SP_MDT) { + /* CLI->MDT */ + if (fl_udesc && sf->sf_rpc != SPTLRPC_FLVR_NULL) + sf->sf_flags |= PTLRPC_SEC_FL_UDESC; + } +} + +void sptlrpc_conf_choose_flavor(enum lustre_sec_part from, + enum lustre_sec_part to, + struct obd_uuid *target, + lnet_nid_t nid, + struct sptlrpc_flavor *sf) +{ + struct sptlrpc_conf *conf; + struct sptlrpc_conf_tgt *conf_tgt; + char name[MTI_NAME_MAXLEN]; + int len, rc = 0; + + target2fsname(target->uuid, name, sizeof(name)); + + mutex_lock(&sptlrpc_conf_lock); + + conf = sptlrpc_conf_get(name, 0); + if (conf == NULL) + goto out; + + /* convert uuid name (supposed end with _UUID) to target name */ + len = strlen(target->uuid); + LASSERT(len > 5); + memcpy(name, target->uuid, len - 5); + name[len - 5] = '\0'; + + conf_tgt = sptlrpc_conf_get_tgt(conf, name, 0); + if (conf_tgt) { + rc = sptlrpc_rule_set_choose(&conf_tgt->sct_rset, + from, to, nid, sf); + if (rc) + goto out; + } + + rc = sptlrpc_rule_set_choose(&conf->sc_rset, from, to, nid, sf); +out: + mutex_unlock(&sptlrpc_conf_lock); + + if (rc == 0) + get_default_flavor(sf); + + flavor_set_flags(sf, from, to, 1); +} + +/** + * called by target devices, determine the expected flavor from + * certain peer (from, nid). + */ +void sptlrpc_target_choose_flavor(struct sptlrpc_rule_set *rset, + enum lustre_sec_part from, + lnet_nid_t nid, + struct sptlrpc_flavor *sf) +{ + if (sptlrpc_rule_set_choose(rset, from, LUSTRE_SP_ANY, nid, sf) == 0) + get_default_flavor(sf); +} +EXPORT_SYMBOL(sptlrpc_target_choose_flavor); + #define SEC_ADAPT_DELAY (10) -int sptlrpc_cliobd_process_config(struct obd_device *obd, - struct lustre_cfg *lcfg) +/** + * called by client devices, notify the sptlrpc config has changed and + * do import_sec_adapt later. + */ +void sptlrpc_conf_client_adapt(struct obd_device *obd) { - struct sptlrpc_conf_log *log; - struct obd_import *imp; - int rc; + struct obd_import *imp; + ENTRY; - log = sptlrpc_conf_log_extract(lcfg); - if (IS_ERR(log)) { - CERROR("extract log error: %ld\n", PTR_ERR(log)); - return PTR_ERR(log); + LASSERT(strcmp(obd->obd_type->typ_name, LUSTRE_MDC_NAME) == 0 || + strcmp(obd->obd_type->typ_name, LUSTRE_OSC_NAME) ==0); + CDEBUG(D_SEC, "obd %s\n", obd->u.cli.cl_target_uuid.uuid); + + /* serialize with connect/disconnect import */ + down_read(&obd->u.cli.cl_sem); + + imp = obd->u.cli.cl_import; + if (imp) { + spin_lock(&imp->imp_lock); + if (imp->imp_sec) + imp->imp_sec_expire = cfs_time_current_sec() + + SEC_ADAPT_DELAY; + spin_unlock(&imp->imp_lock); } - obd->u.cli.cl_sec_part = log->scl_part; + up_read(&obd->u.cli.cl_sem); + EXIT; +} +EXPORT_SYMBOL(sptlrpc_conf_client_adapt); + +#ifdef __KERNEL__ + +static void rule2string(struct sptlrpc_rule *r, char *buf, int buflen) +{ + char dirbuf[8]; + char *net; + char *ptr = buf; + + if (r->sr_netid == LNET_NIDNET(LNET_NID_ANY)) + net = "default"; + else + net = libcfs_net2str(r->sr_netid); + + if (r->sr_from == LUSTRE_SP_ANY && r->sr_to == LUSTRE_SP_ANY) + dirbuf[0] = '\0'; + else + snprintf(dirbuf, sizeof(dirbuf), ".%s2%s", + sptlrpc_part2name(r->sr_from), + sptlrpc_part2name(r->sr_to)); + + ptr += snprintf(buf, buflen, "srpc.flavor.%s%s=", net, dirbuf); + + sptlrpc_flavor2name(&r->sr_flvr, ptr, buflen - (ptr - buf)); + buf[buflen - 1] = '\0'; +} + +static int sptlrpc_record_rule_set(struct llog_handle *llh, + char *target, + struct sptlrpc_rule_set *rset) +{ + struct lustre_cfg_bufs bufs; + struct lustre_cfg *lcfg; + struct llog_rec_hdr rec; + int buflen; + char param[48]; + int i, rc; + + for (i = 0; i < rset->srs_nrule; i++) { + rule2string(&rset->srs_rules[i], param, sizeof(param)); + + lustre_cfg_bufs_reset(&bufs, NULL); + lustre_cfg_bufs_set_string(&bufs, 1, target); + lustre_cfg_bufs_set_string(&bufs, 2, param); + lcfg = lustre_cfg_new(LCFG_SPTLRPC_CONF, &bufs); + LASSERT(lcfg); + + buflen = lustre_cfg_len(lcfg->lcfg_bufcount, + lcfg->lcfg_buflens); + rec.lrh_len = llog_data_len(buflen); + rec.lrh_type = OBD_CFG_REC; + rc = llog_write_rec(llh, &rec, NULL, 0, (void *)lcfg, -1); + if (rc) + CERROR("failed to write a rec: rc = %d\n", rc); + lustre_cfg_free(lcfg); + } + return 0; +} + +static int sptlrpc_record_rules(struct llog_handle *llh, + struct sptlrpc_conf *conf) +{ + struct sptlrpc_conf_tgt *conf_tgt; + + sptlrpc_record_rule_set(llh, conf->sc_fsname, &conf->sc_rset); + + list_for_each_entry(conf_tgt, &conf->sc_tgts, sct_list) { + sptlrpc_record_rule_set(llh, conf_tgt->sct_name, + &conf_tgt->sct_rset); + } + return 0; +} + +#define LOG_SPTLRPC_TMP "sptlrpc.tmp" +#define LOG_SPTLRPC "sptlrpc" + +static +int sptlrpc_target_local_copy_conf(struct obd_device *obd, + struct sptlrpc_conf *conf) +{ + struct llog_handle *llh = NULL; + struct llog_ctxt *ctxt; + struct lvfs_run_ctxt saved; + struct dentry *dentry; + int rc; + ENTRY; + + ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); + if (ctxt == NULL) { + CERROR("missing llog context\n"); + RETURN(-EINVAL); + } + + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + + dentry = lookup_one_len(MOUNT_CONFIGS_DIR, current->fs->pwd, + strlen(MOUNT_CONFIGS_DIR)); + if (IS_ERR(dentry)) { + rc = PTR_ERR(dentry); + CERROR("cannot lookup %s directory: rc = %d\n", + MOUNT_CONFIGS_DIR, rc); + GOTO(out_ctx, rc); + } + + /* erase the old tmp log */ + rc = llog_create(ctxt, &llh, NULL, LOG_SPTLRPC_TMP); + if (rc == 0) { + rc = llog_init_handle(llh, LLOG_F_IS_PLAIN, NULL); + if (rc == 0) { + rc = llog_destroy(llh); + llog_free_handle(llh); + } else { + llog_close(llh); + } + } - rc = sptlrpc_rule_set_from_log(&obd->u.cli.cl_sptlrpc_rset, log); if (rc) { - CERROR("failed create rule set: %d\n", rc); - return rc; + CERROR("target %s: cannot erase temporary sptlrpc log: " + "rc = %d\n", obd->obd_name, rc); + GOTO(out_dput, rc); } - imp = obd->u.cli.cl_import; - if (imp == NULL) - return 0; - - /* even if imp_sec_expire is already set, we'll override it to a - * newer (later) time */ - spin_lock(&imp->imp_lock); - if (imp->imp_sec) - imp->imp_sec_expire = cfs_time_current_sec() + SEC_ADAPT_DELAY; - spin_unlock(&imp->imp_lock); + /* write temporary log */ + rc = llog_create(ctxt, &llh, NULL, LOG_SPTLRPC_TMP); + if (rc) + GOTO(out_dput, rc); + rc = llog_init_handle(llh, LLOG_F_IS_PLAIN, NULL); + if (rc) + GOTO(out_close, rc); + + rc = sptlrpc_record_rules(llh, conf); + +out_close: + llog_close(llh); + + if (rc == 0) { + rc = lustre_rename(dentry, obd->obd_lvfs_ctxt.pwdmnt, + LOG_SPTLRPC_TMP, LOG_SPTLRPC); + } + +out_dput: + l_dput(dentry); +out_ctx: + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + llog_ctxt_put(ctxt); + CDEBUG(D_SEC, "target %s: write local sptlrpc conf: rc = %d\n", + obd->obd_name, rc); + RETURN(rc); +} + +static int local_read_handler(struct llog_handle *llh, + struct llog_rec_hdr *rec, + void *data) +{ + struct sptlrpc_conf *conf = (struct sptlrpc_conf *) data; + struct lustre_cfg *lcfg = (struct lustre_cfg *)(rec + 1); + int cfg_len, rc; + ENTRY; + + if (rec->lrh_type != OBD_CFG_REC) { + CERROR("unhandled lrh_type: %#x\n", rec->lrh_type); + RETURN(-EINVAL); + } + + cfg_len = rec->lrh_len - sizeof(struct llog_rec_hdr) - + sizeof(struct llog_rec_tail); + + rc = lustre_cfg_sanity_check(lcfg, cfg_len); + if (rc) { + CERROR("Insane cfg\n"); + RETURN(rc); + } + + if (lcfg->lcfg_command != LCFG_SPTLRPC_CONF) { + CERROR("invalid command (%x)\n", lcfg->lcfg_command); + RETURN(-EINVAL); + } + + RETURN(__sptlrpc_process_config(lcfg, conf)); +} + +static +int sptlrpc_target_local_read_conf(struct obd_device *obd, + struct sptlrpc_conf *conf) +{ + struct llog_handle *llh = NULL; + struct llog_ctxt *ctxt; + struct lvfs_run_ctxt saved; + int rc; + ENTRY; + + LASSERT(conf->sc_updated == 0 && conf->sc_local == 0); + + ctxt = llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT); + if (ctxt == NULL) { + CERROR("missing llog context\n"); + RETURN(-EINVAL); + } + + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + + rc = llog_create(ctxt, &llh, NULL, LOG_SPTLRPC); + if (rc) + GOTO(out_pop, rc); + + rc = llog_init_handle(llh, LLOG_F_IS_PLAIN, NULL); + if (rc) + GOTO(out_close, rc); + + if (llog_get_size(llh) <= 1) { + CDEBUG(D_SEC, "no local sptlrpc copy found\n"); + GOTO(out_close, rc = 0); + } + + rc = llog_process(llh, local_read_handler, (void *) conf, NULL); + + if (rc == 0) { + conf->sc_local = 1; + } else { + sptlrpc_conf_free_rsets(conf); + } + +out_close: + llog_close(llh); +out_pop: + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + llog_ctxt_put(ctxt); + CDEBUG(D_SEC, "target %s: read local sptlrpc conf: rc = %d\n", + obd->obd_name, rc); + RETURN(rc); +} + +#endif /* __KRENEL__ */ + +/** + * called by target devices, extract sptlrpc rules which applies to + * this target, to be used for future rpc flavor checking. + */ +int sptlrpc_conf_target_get_rules(struct obd_device *obd, + struct sptlrpc_rule_set *rset, + int initial) +{ + struct sptlrpc_conf *conf; + struct sptlrpc_conf_tgt *conf_tgt; + enum lustre_sec_part sp_dst; + char fsname[MTI_NAME_MAXLEN]; + int rc = 0; + ENTRY; + + if (strcmp(obd->obd_type->typ_name, LUSTRE_MDT_NAME) == 0) { + sp_dst = LUSTRE_SP_MDT; + } else if (strcmp(obd->obd_type->typ_name, LUSTRE_OST_NAME) == 0) { + sp_dst = LUSTRE_SP_OST; + } else { + CERROR("unexpected obd type %s\n", obd->obd_type->typ_name); + RETURN(-EINVAL); + } + CDEBUG(D_SEC, "get rules for target %s\n", obd->obd_uuid.uuid); + + target2fsname(obd->obd_uuid.uuid, fsname, sizeof(fsname)); + + mutex_lock(&sptlrpc_conf_lock); + + conf = sptlrpc_conf_get(fsname, 0); + if (conf == NULL) { + CERROR("missing sptlrpc config log\n"); + GOTO(out, rc); + } + +#ifdef __KERNEL__ + if (conf->sc_updated == 0) { + /* + * always read from local copy. here another option is + * if we already have a local copy (read from another + * target device hosted on the same node) we simply use that. + */ + if (conf->sc_local) + sptlrpc_conf_free_rsets(conf); + + sptlrpc_target_local_read_conf(obd, conf); + } else { + LASSERT(conf->sc_local == 0); + + /* write a local copy */ + if (initial || conf->sc_modified) + sptlrpc_target_local_copy_conf(obd, conf); + else + CDEBUG(D_SEC, "unchanged, skip updating local copy\n"); + } +#endif + + /* extract rule set for this target */ + conf_tgt = sptlrpc_conf_get_tgt(conf, obd->obd_name, 0); + + rc = sptlrpc_rule_set_extract(&conf->sc_rset, + conf_tgt ? &conf_tgt->sct_rset: NULL, + LUSTRE_SP_ANY, sp_dst, rset); +out: + mutex_unlock(&sptlrpc_conf_lock); + RETURN(rc); +} +EXPORT_SYMBOL(sptlrpc_conf_target_get_rules); + +int sptlrpc_conf_init(void) +{ + mutex_init(&sptlrpc_conf_lock); return 0; } -EXPORT_SYMBOL(sptlrpc_cliobd_process_config); + +void sptlrpc_conf_fini(void) +{ + struct sptlrpc_conf *conf, *conf_next; + + mutex_lock(&sptlrpc_conf_lock); + list_for_each_entry_safe(conf, conf_next, &sptlrpc_confs, sc_list) { + sptlrpc_conf_free(conf); + } + LASSERT(list_empty(&sptlrpc_confs)); + mutex_unlock(&sptlrpc_conf_lock); +} diff --git a/lustre/ptlrpc/sec_gc.c b/lustre/ptlrpc/sec_gc.c index bc1fe0d..76c4296 100644 --- a/lustre/ptlrpc/sec_gc.c +++ b/lustre/ptlrpc/sec_gc.c @@ -58,12 +58,12 @@ #ifdef __KERNEL__ -static DECLARE_MUTEX(sec_gc_mutex); +static struct mutex sec_gc_mutex; static CFS_LIST_HEAD(sec_gc_list); -static spinlock_t sec_gc_list_lock = SPIN_LOCK_UNLOCKED; +static spinlock_t sec_gc_list_lock; static CFS_LIST_HEAD(sec_gc_ctx_list); -static spinlock_t sec_gc_ctx_list_lock = SPIN_LOCK_UNLOCKED; +static spinlock_t sec_gc_ctx_list_lock; static struct ptlrpc_thread sec_gc_thread; static atomic_t sec_gc_wait_del = ATOMIC_INIT(0); @@ -100,8 +100,8 @@ void sptlrpc_gc_del_sec(struct ptlrpc_sec *sec) spin_unlock(&sec_gc_list_lock); /* barrier */ - mutex_down(&sec_gc_mutex); - mutex_up(&sec_gc_mutex); + mutex_lock(&sec_gc_mutex); + mutex_unlock(&sec_gc_mutex); atomic_dec(&sec_gc_wait_del); @@ -190,19 +190,19 @@ again: * to trace each sec as order of expiry time. * another issue here is we wakeup as fixed interval instead of * according to each sec's expiry time */ - mutex_down(&sec_gc_mutex); + mutex_lock(&sec_gc_mutex); list_for_each_entry(sec, &sec_gc_list, ps_gc_list) { /* if someone is waiting to be deleted, let it * proceed as soon as possible. */ if (atomic_read(&sec_gc_wait_del)) { CWARN("deletion pending, start over\n"); - mutex_up(&sec_gc_mutex); + mutex_unlock(&sec_gc_mutex); goto again; } sec_do_gc(sec); } - mutex_up(&sec_gc_mutex); + mutex_unlock(&sec_gc_mutex); /* check ctx list again before sleep */ sec_process_ctx_list(); @@ -223,11 +223,15 @@ again: return 0; } -int sptlrpc_gc_start_thread(void) +int sptlrpc_gc_init(void) { struct l_wait_info lwi = { 0 }; int rc; + mutex_init(&sec_gc_mutex); + spin_lock_init(&sec_gc_list_lock); + spin_lock_init(&sec_gc_ctx_list_lock); + /* initialize thread control */ memset(&sec_gc_thread, 0, sizeof(sec_gc_thread)); cfs_waitq_init(&sec_gc_thread.t_ctl_waitq); @@ -244,7 +248,7 @@ int sptlrpc_gc_start_thread(void) return 0; } -void sptlrpc_gc_stop_thread(void) +void sptlrpc_gc_fini(void) { struct l_wait_info lwi = { 0 }; @@ -263,11 +267,11 @@ void sptlrpc_gc_add_sec(struct ptlrpc_sec *sec) void sptlrpc_gc_del_sec(struct ptlrpc_sec *sec) { } -int sptlrpc_gc_start_thread(void) +int sptlrpc_gc_init(void) { return 0; } -void sptlrpc_gc_stop_thread(void) +void sptlrpc_gc_fini(void) { } diff --git a/lustre/ptlrpc/sec_null.c b/lustre/ptlrpc/sec_null.c index c63401b..7b4368d 100644 --- a/lustre/ptlrpc/sec_null.c +++ b/lustre/ptlrpc/sec_null.c @@ -96,7 +96,7 @@ int null_ctx_sign(struct ptlrpc_cli_ctx *ctx, struct ptlrpc_request *req) if (!req->rq_import->imp_dlm_fake) { struct obd_device *obd = req->rq_import->imp_obd; null_encode_sec_part(req->rq_reqbuf, - obd->u.cli.cl_sec_part); + obd->u.cli.cl_sp_me); } req->rq_reqdata_len = req->rq_reqlen; return 0; diff --git a/lustre/ptlrpc/sec_plain.c b/lustre/ptlrpc/sec_plain.c index 4f164c1..eb9ee82 100644 --- a/lustre/ptlrpc/sec_plain.c +++ b/lustre/ptlrpc/sec_plain.c @@ -328,7 +328,7 @@ struct ptlrpc_sec *plain_create_sec(struct obd_import *imp, sec->ps_id = sptlrpc_get_next_secid(); sec->ps_import = class_import_get(imp); sec->ps_flvr = *sf; - sec->ps_lock = SPIN_LOCK_UNLOCKED; + spin_lock_init(&sec->ps_lock); CFS_INIT_LIST_HEAD(&sec->ps_gc_list); sec->ps_gc_interval = 0; sec->ps_gc_next = 0; diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 865ead7..913f695 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -183,12 +183,12 @@ if [ "$ONLY" == "cleanup" ]; then exit fi +init_gss + #create single point mountpoint gen_config -init_gss - test_0() { setup check_mount || return 41 @@ -903,6 +903,7 @@ test_29() { writeconf start_mds start_ost + sleep 5 cleanup } run_test 29 "permanently remove an OST" @@ -1418,7 +1419,6 @@ run_test 42 "invalid config param should not prevent client from mounting" umount_client $MOUNT cleanup_nocli -cleanup_gss test_45() { #17310 setup @@ -1440,6 +1440,6 @@ test_45() { #17310 } run_test 45 "long unlink handling in ptlrpcd" - +cleanup_gss equals_msg `basename $0`: test complete [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true diff --git a/lustre/tests/sanity-gss.sh b/lustre/tests/sanity-gss.sh index 9024134..31cfae0 100644 --- a/lustre/tests/sanity-gss.sh +++ b/lustre/tests/sanity-gss.sh @@ -624,7 +624,7 @@ test_7() { [ $num_osts -lt 2 ] && echo "skipping $TESTNAME (must have >= 2 OSTs)" && return mkdir $tdir || error - $LFS setstripe $tdir 0 -1 -1 || error + $LFS setstripe -c $num_osts $tdir || error echo "creating..." for ((i=0;i<20;i++)); do @@ -640,14 +640,15 @@ run_test 7 "exercise enlarge_reqbuf()" test_8() { - debugsave - sysctl -w lnet.debug="other" + sleep $TIMEOUT $LCTL dk > /dev/null + debugsave + sysctl -w lnet.debug="+other" # sleep sometime in ctx handle - do_facet mds sysctl -w lustre.fail_val=60 + do_facet mds lctl set_param fail_val=30 #define OBD_FAIL_SEC_CTX_HDL_PAUSE 0x1204 - do_facet mds sysctl -w lustre.fail_loc=0x1204 + do_facet mds lctl set_param fail_loc=0x1204 $RUNAS $LFS flushctx || error "can't flush ctx" @@ -780,6 +781,8 @@ test_90() { $LFS flushctx done check_dbench + #sleep to let ctxs be re-established + sleep 10 stop_dbench } run_test 90 "recoverable from losing contexts under load" @@ -787,7 +790,7 @@ run_test 90 "recoverable from losing contexts under load" test_99() { local nrule_old=0 local nrule_new=0 - local max=32 + local max=64 # # general rules @@ -799,7 +802,6 @@ test_99() { for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME elan$i any krb5n || error "set rule $i" done - set_rule $FSNAME elan100 any krb5n && error "set $max rule should fail" for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME elan$i any || error "remove rule $i" done @@ -820,7 +822,6 @@ test_99() { for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME-MDT0000 elan$i any krb5i || error "set rule $i" done - set_rule $FSNAME-MDT0000 elan100 any krb5i && error "set $max rule should fail" for ((i = $nrule_old; i < $max; i++)); do set_rule $FSNAME-MDT0000 elan$i any || error "remove rule $i" done @@ -831,7 +832,7 @@ test_99() { error "general rule: $nrule_new != $nrule_old" fi } -run_test 99 "maximum sptlrpc rules limitation" +run_test 99 "set large number of sptlrpc rules" error_dbench() { diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 5b08adc..75187c6 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -5358,6 +5358,8 @@ run_test 125 "don't return EPROTO when a dir has a non-default striping and ACLs test_126() { # bug 12829/13455 [ -z "$(lctl get_param -n llite.*.client_type | grep local)" ] && skip "must run as local client" && return [ "$UID" != 0 ] && echo "skipping $TESTNAME (must run as root)" && return + $GSS && skip "must run as gss disabled" && return + $RUNAS -u 0 -g 1 touch $DIR/$tfile || error "touch failed" gid=`ls -n $DIR/$tfile | awk '{print $4}'` rm -f $DIR/$tfile diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 2588ea0..668adb8 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1229,6 +1229,10 @@ setupall() { done fi + # wait a while to allow sptlrpc configuration be propogated to targets, + # only needed when mounting new target devices. + $GSS && sleep 10 + [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE mount_client $MOUNT [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT diff --git a/lustre/utils/llog_reader.c b/lustre/utils/llog_reader.c index d2f1f1a..ae62a39 100644 --- a/lustre/utils/llog_reader.c +++ b/lustre/utils/llog_reader.c @@ -363,6 +363,11 @@ void print_lustre_cfg(struct lustre_cfg *lcfg, int *skip) print_1_cfg(lcfg); break; } + case(LCFG_SPTLRPC_CONF):{ + printf("sptlrpc_conf "); + print_1_cfg(lcfg); + break; + } case(LCFG_MARKER):{ struct cfg_marker *marker = lustre_cfg_buf(lcfg, 1); char createtime[26], canceltime[26] = "";