X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosp%2Fosp_dev.c;h=0cd88ec2f48d7ae36ee915cb5a2609856b46f2b7;hp=00e5da00da95f9cb4a479067ed44c22590b4fb77;hb=9175d6f862039cfbf6b85333d72a9d91e64ea42a;hpb=ec37d78caf6cbd8118cfe1ff012c828950356e7a diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c index 00e5da0..0cd88ec 100644 --- a/lustre/osp/osp_dev.c +++ b/lustre/osp/osp_dev.c @@ -27,7 +27,7 @@ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2012, 2013, Intel Corporation. + * Copyright (c) 2012, 2015, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -74,11 +74,11 @@ #define DEBUG_SUBSYSTEM S_MDS +#include #include #include #include #include -#include #include "osp_internal.h" @@ -111,9 +111,9 @@ static struct lu_kmem_descr osp_caches[] = { * \retval object object being created if the creation succeed. * \retval NULL NULL if the creation failed. */ -struct lu_object *osp_object_alloc(const struct lu_env *env, - const struct lu_object_header *hdr, - struct lu_device *d) +static struct lu_object *osp_object_alloc(const struct lu_env *env, + const struct lu_object_header *hdr, + struct lu_device *d) { struct lu_object_header *h = NULL; struct osp_object *o; @@ -181,7 +181,7 @@ static struct dt_object RETURN(dto); /* Get attributes of the local object. */ - rc = dt_attr_get(env, dto, attr, NULL); + rc = dt_attr_get(env, dto, attr); if (rc) { CERROR("%s: can't be initialized: rc = %d\n", osp->opd_obd->obd_name, rc); @@ -484,6 +484,95 @@ static int osp_disconnect(struct osp_device *d) } /** + * Initialize the osp_update structure in OSP device + * + * Allocate osp update structure and start update thread. + * + * \param[in] osp OSP device + * + * \retval 0 if initialization succeeds. + * \retval negative errno if initialization fails. + */ +static int osp_update_init(struct osp_device *osp) +{ + struct l_wait_info lwi = { 0 }; + struct task_struct *task; + + ENTRY; + + LASSERT(osp->opd_connect_mdt); + + OBD_ALLOC_PTR(osp->opd_update); + if (osp->opd_update == NULL) + RETURN(-ENOMEM); + + init_waitqueue_head(&osp->opd_update_thread.t_ctl_waitq); + init_waitqueue_head(&osp->opd_update->ou_waitq); + spin_lock_init(&osp->opd_update->ou_lock); + INIT_LIST_HEAD(&osp->opd_update->ou_list); + osp->opd_update->ou_rpc_version = 1; + osp->opd_update->ou_version = 1; + + /* start thread handling sending updates to the remote MDT */ + task = kthread_run(osp_send_update_thread, osp, + "osp_up%u-%u", osp->opd_index, osp->opd_group); + if (IS_ERR(task)) { + int rc = PTR_ERR(task); + + OBD_FREE_PTR(osp->opd_update); + osp->opd_update = NULL; + CERROR("%s: can't start precreate thread: rc = %d\n", + osp->opd_obd->obd_name, rc); + RETURN(rc); + } + + l_wait_event(osp->opd_update_thread.t_ctl_waitq, + osp_send_update_thread_running(osp) || + osp_send_update_thread_stopped(osp), &lwi); + + RETURN(0); +} + +/** + * Finialize osp_update structure in OSP device + * + * Stop the OSP update sending thread, then delete the left + * osp thandle in the sending list. + * + * \param [in] osp OSP device. + */ +static void osp_update_fini(const struct lu_env *env, struct osp_device *osp) +{ + struct osp_update_request *our; + struct osp_update_request *tmp; + struct osp_updates *ou = osp->opd_update; + + if (ou == NULL) + return; + + osp->opd_update_thread.t_flags = SVC_STOPPING; + wake_up(&ou->ou_waitq); + + wait_event(osp->opd_update_thread.t_ctl_waitq, + osp->opd_update_thread.t_flags & SVC_STOPPED); + + /* Remove the left osp thandle from the list */ + spin_lock(&ou->ou_lock); + list_for_each_entry_safe(our, tmp, &ou->ou_list, + our_list) { + list_del_init(&our->our_list); + LASSERT(our->our_th != NULL); + osp_trans_callback(env, our->our_th, -EIO); + /* our will be destroyed in osp_thandle_put() */ + osp_thandle_put(our->our_th); + } + spin_unlock(&ou->ou_lock); + + OBD_FREE_PTR(ou); + osp->opd_update = NULL; +} + +/** * Cleanup OSP, which includes disconnect import, cleanup unlink log, stop * precreate threads etc. * @@ -502,9 +591,10 @@ static int osp_shutdown(const struct lu_env *env, struct osp_device *d) rc = osp_disconnect(d); - osp_sync_fini(d); - if (!d->opd_connect_mdt) { + /* stop sync thread */ + osp_sync_fini(d); + /* stop precreate thread */ osp_precreate_fini(d); @@ -543,6 +633,7 @@ static int osp_process_config(const struct lu_env *env, switch (lcfg->lcfg_command) { case LCFG_PRE_CLEANUP: rc = osp_disconnect(d); + osp_update_fini(env, d); break; case LCFG_CLEANUP: lu_dev_del_linkage(dev->ld_site, dev); @@ -550,8 +641,9 @@ static int osp_process_config(const struct lu_env *env, break; case LCFG_PARAM: LASSERT(obd); - rc = class_process_proc_param(PARAM_OSC, obd->obd_vars, - lcfg, obd); + rc = class_process_proc_param(d->opd_connect_mdt ? + PARAM_OSP : PARAM_OSC, + obd->obd_vars, lcfg, obd); if (rc > 0) rc = 0; if (rc == -ENOSYS) { @@ -627,9 +719,13 @@ static int osp_statfs(const struct lu_env *env, struct dt_device *dev, struct obd_statfs *sfs) { struct osp_device *d = dt2osp_dev(dev); + struct obd_import *imp = d->opd_obd->u.cli.cl_import; ENTRY; + if (imp->imp_state == LUSTRE_IMP_CLOSED) + RETURN(-ESHUTDOWN); + if (unlikely(d->opd_imp_active == 0)) RETURN(-ENOTCONN); @@ -690,6 +786,10 @@ static int osp_sync(const struct lu_env *env, struct dt_device *dev) unsigned long start = cfs_time_current(); ENTRY; + /* No Sync between MDTs yet. */ + if (d->opd_connect_mdt) + RETURN(0); + if (unlikely(d->opd_imp_active == 0)) RETURN(-ENOTCONN); @@ -787,6 +887,7 @@ const struct dt_device_operations osp_dt_ops = { .dt_trans_create = osp_trans_create, .dt_trans_start = osp_trans_start, .dt_trans_stop = osp_trans_stop, + .dt_trans_cb_add = osp_trans_cb_add, }; /** @@ -845,6 +946,27 @@ out: } /** + * Determine if the lock needs to be cancelled + * + * Determine if the unused lock should be cancelled before replay, see + * (ldlm_cancel_no_wait_policy()). Currently, only inode bits lock exists + * between MDTs. + * + * \param[in] lock lock to be checked. + * + * \retval 1 if the lock needs to be cancelled before replay. + * \retval 0 if the lock does not need to be cancelled before + * replay. + */ +static int osp_cancel_weight(struct ldlm_lock *lock) +{ + if (lock->l_resource->lr_type != LDLM_IBITS) + RETURN(0); + + RETURN(1); +} + +/** * Initialize OSP device according to the parameters in the configuration * log \a cfg. * @@ -1047,16 +1169,22 @@ static int osp_init0(const struct lu_env *env, struct osp_device *osp, rc = osp_init_precreate(osp); if (rc) GOTO(out_last_used, rc); + + /* + * Initialize synhronization mechanism taking + * care of propogating changes to OST in near + * transactional manner. + */ + rc = osp_sync_init(env, osp); + if (rc < 0) + GOTO(out_precreat, rc); + } else { + rc = osp_update_init(osp); + if (rc != 0) + GOTO(out_fid, rc); } - /* - * Initialize synhronization mechanism taking - * care of propogating changes to OST in near - * transactional manner. - */ - rc = osp_sync_init(env, osp); - if (rc) - GOTO(out_precreat, rc); + ns_register_cancel(obd->obd_namespace, osp_cancel_weight); /* * Initiate connect to OST @@ -1074,12 +1202,15 @@ static int osp_init0(const struct lu_env *env, struct osp_device *osp, RETURN(0); out: - /* stop sync thread */ - osp_sync_fini(osp); + if (!osp->opd_connect_mdt) + /* stop sync thread */ + osp_sync_fini(osp); out_precreat: /* stop precreate thread */ if (!osp->opd_connect_mdt) osp_precreate_fini(osp); + else + osp_update_fini(env, osp); out_last_used: if (!osp->opd_connect_mdt) osp_last_used_fini(env, osp); @@ -1193,7 +1324,7 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, ENTRY; if (osp->opd_async_requests != NULL) { - dt_update_request_destroy(osp->opd_async_requests); + osp_update_request_destroy(osp->opd_async_requests); osp->opd_async_requests = NULL; } @@ -1202,11 +1333,6 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, imp = osp->opd_obd->u.cli.cl_import; - if (imp->imp_rq_pool) { - ptlrpc_free_rq_pool(imp->imp_rq_pool); - imp->imp_rq_pool = NULL; - } - if (osp->opd_symlink) lprocfs_remove(&osp->opd_symlink); @@ -1310,6 +1436,9 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, if (rc) { CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc); GOTO(out, rc); + } else { + osp->opd_obd->u.cli.cl_seq->lcs_exp = + class_export_get(osp->opd_exp); } ptlrpc_pinger_add_import(imp); @@ -1427,26 +1556,6 @@ out: } /** - * Prepare fid client. - * - * This function prepares the FID client for the OSP. It will check and assign - * the export (to MDT0) for its FID client, so OSP can allocate super sequence - * or lookup sequence in FLDB of MDT0. - * - * \param[in] osp OSP device - */ -static void osp_prepare_fid_client(struct osp_device *osp) -{ - LASSERT(osp->opd_obd->u.cli.cl_seq != NULL); - if (osp->opd_obd->u.cli.cl_seq->lcs_exp != NULL) - return; - - LASSERT(osp->opd_exp != NULL); - osp->opd_obd->u.cli.cl_seq->lcs_exp = - class_export_get(osp->opd_exp); -} - -/** * Implementation of obd_ops::o_import_event * * This function is called when some related import event happens. It will @@ -1481,6 +1590,7 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, break; case IMP_EVENT_INACTIVE: d->opd_imp_active = 0; + d->opd_imp_connected = 0; if (d->opd_connect_mdt) break; @@ -1494,7 +1604,6 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, case IMP_EVENT_ACTIVE: d->opd_imp_active = 1; - osp_prepare_fid_client(d); if (d->opd_got_disconnected) d->opd_new_connection = 1; d->opd_imp_connected = 1; @@ -1509,6 +1618,9 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, CDEBUG(D_HA, "got connected\n"); break; case IMP_EVENT_INVALIDATE: + if (d->opd_connect_mdt) + osp_invalidate_request(d); + if (obd->obd_namespace == NULL) break; ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY); @@ -1542,7 +1654,7 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, * \retval negative negative errno if the ioctl handling failed. */ static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len, - void *karg, void *uarg) + void *karg, void __user *uarg) { struct obd_device *obd = exp->exp_obd; struct osp_device *d; @@ -1584,6 +1696,7 @@ static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len, return rc; } + /** * Implementation of obd_ops::o_get_info * @@ -1597,14 +1710,12 @@ static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len, * \param[in] key the key * \param[out] vallen length of \a val * \param[out] val holds the value returned by the key - * \param[in] unused necessary for the interface but unused * * \retval 0 0 if getting information succeeded. * \retval negative negative errno if getting information failed. */ static int osp_obd_get_info(const struct lu_env *env, struct obd_export *exp, - __u32 keylen, void *key, __u32 *vallen, void *val, - struct lov_stripe_md *unused) + __u32 keylen, void *key, __u32 *vallen, void *val) { int rc = -EINVAL; @@ -1628,6 +1739,58 @@ static int osp_obd_get_info(const struct lu_env *env, struct obd_export *exp, RETURN(rc); } +static int osp_obd_set_info_async(const struct lu_env *env, + struct obd_export *exp, + u32 keylen, void *key, + u32 vallen, void *val, + struct ptlrpc_request_set *set) +{ + struct obd_device *obd = exp->exp_obd; + struct obd_import *imp = obd->u.cli.cl_import; + struct osp_device *osp; + struct ptlrpc_request *req; + char *tmp; + int rc; + + if (KEY_IS(KEY_SPTLRPC_CONF)) { + sptlrpc_conf_client_adapt(exp->exp_obd); + RETURN(0); + } + + LASSERT(set != NULL); + if (!obd->obd_set_up || obd->obd_stopping) + RETURN(-EAGAIN); + osp = lu2osp_dev(obd->obd_lu_dev); + + req = ptlrpc_request_alloc(imp, &RQF_OBD_SET_INFO); + if (req == NULL) + RETURN(-ENOMEM); + + req_capsule_set_size(&req->rq_pill, &RMF_SETINFO_KEY, + RCL_CLIENT, keylen); + req_capsule_set_size(&req->rq_pill, &RMF_SETINFO_VAL, + RCL_CLIENT, vallen); + if (osp->opd_connect_mdt) + rc = ptlrpc_request_pack(req, LUSTRE_MDS_VERSION, MDS_SET_INFO); + else + rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_SET_INFO); + if (rc) { + ptlrpc_request_free(req); + RETURN(rc); + } + + tmp = req_capsule_client_get(&req->rq_pill, &RMF_SETINFO_KEY); + memcpy(tmp, key, keylen); + tmp = req_capsule_client_get(&req->rq_pill, &RMF_SETINFO_VAL); + memcpy(tmp, val, vallen); + + ptlrpc_request_set_replen(req); + ptlrpc_set_add_req(set, req); + ptlrpc_check_set(NULL, set); + + RETURN(0); +} + /** * Implementation of obd_ops: o_fid_alloc * @@ -1651,8 +1814,8 @@ static int osp_obd_get_info(const struct lu_env *env, struct obd_export *exp, * requested from seq meta server * \retval negative negative errno if FID allocation failed. */ -int osp_fid_alloc(const struct lu_env *env, struct obd_export *exp, - struct lu_fid *fid, struct md_op_data *unused) +static int osp_fid_alloc(const struct lu_env *env, struct obd_export *exp, + struct lu_fid *fid, struct md_op_data *unused) { struct client_obd *cli = &exp->exp_obd->u.cli; struct osp_device *osp = lu2osp_dev(exp->exp_obd->obd_lu_dev); @@ -1661,8 +1824,7 @@ int osp_fid_alloc(const struct lu_env *env, struct obd_export *exp, LASSERT(osp->opd_obd->u.cli.cl_seq != NULL); /* Sigh, fid client is not ready yet */ - if (osp->opd_obd->u.cli.cl_seq->lcs_exp == NULL) - RETURN(-ENOTCONN); + LASSERT(osp->opd_obd->u.cli.cl_seq->lcs_exp != NULL); RETURN(seq_client_alloc_fid(env, seq, fid)); } @@ -1722,6 +1884,7 @@ static struct obd_ops osp_obd_device_ops = { .o_connect = osp_obd_connect, .o_disconnect = osp_obd_disconnect, .o_get_info = osp_obd_get_info, + .o_set_info_async = osp_obd_set_info_async, .o_import_event = osp_import_event, .o_iocontrol = osp_iocontrol, .o_statfs = osp_obd_statfs, @@ -1744,7 +1907,7 @@ struct llog_operations osp_mds_ost_orig_logops; * \retval 0 0 if initialization succeeds. * \retval negative negative errno if initialization failed. */ -static int __init osp_mod_init(void) +static int __init osp_init(void) { struct obd_type *type; int rc; @@ -1780,8 +1943,8 @@ static int __init osp_mod_init(void) return rc; type = class_search_type(LUSTRE_OSP_NAME); - type->typ_procsym = lprocfs_seq_register("osc", proc_lustre_root, - NULL, NULL); + type->typ_procsym = lprocfs_register("osc", proc_lustre_root, + NULL, NULL); if (IS_ERR(type->typ_procsym)) { CERROR("osp: can't create compat entry \"osc\": %d\n", (int) PTR_ERR(type->typ_procsym)); @@ -1796,15 +1959,17 @@ static int __init osp_mod_init(void) * This callback is called when kernel unloads OSP module from memory, and * it will deregister OSP and LWP device type from obd_types (\see class_obd.c). */ -static void __exit osp_mod_exit(void) +static void __exit osp_exit(void) { class_unregister_type(LUSTRE_LWP_NAME); class_unregister_type(LUSTRE_OSP_NAME); lu_kmem_fini(osp_caches); } -MODULE_AUTHOR("Intel, Inc. "); -MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")"); +MODULE_AUTHOR("OpenSFS, Inc. "); +MODULE_DESCRIPTION("Lustre OSD Storage Proxy ("LUSTRE_OSP_NAME")"); +MODULE_VERSION(LUSTRE_VERSION_STRING); MODULE_LICENSE("GPL"); -cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit); +module_init(osp_init); +module_exit(osp_exit);