X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosp%2Fosp_dev.c;h=78d4943ea5322e9b6827bf5c929ca712bc1c70bf;hp=1d2c5fbda5fb5c928ce4106e3213ed851699153a;hb=63cd5455a6931d43f40ea35934817e2244656f75;hpb=65d814c242d03659ec5cc8542f1e1cfa5f7e2588 diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c index 1d2c5fb..78d4943 100644 --- a/lustre/osp/osp_dev.c +++ b/lustre/osp/osp_dev.c @@ -132,6 +132,7 @@ static int osp_last_used_init(const struct lu_env *env, struct osp_device *m) if (rc) GOTO(out, rc); + /* object will be released in device cleanup path */ m->opd_last_used_file = o; if (osi->osi_attr.la_size >= sizeof(osi->osi_id) * @@ -173,34 +174,32 @@ static int osp_last_used_init(const struct lu_env *env, struct osp_device *m) } RETURN(0); out: - /* object will be released in device cleanup path */ CERROR("%s: can't initialize lov_objid: %d\n", m->opd_obd->obd_name, rc); lu_object_put(env, &o->do_lu); m->opd_last_used_file = NULL; - RETURN(rc); + return rc; } static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d) { - lu_object_put(env, &d->opd_last_used_file->do_lu); - d->opd_last_used_file = NULL; + if (d->opd_last_used_file != NULL) { + lu_object_put(env, &d->opd_last_used_file->do_lu); + d->opd_last_used_file = NULL; + } } -static int osp_shutdown(const struct lu_env *env, struct osp_device *d) +int osp_disconnect(struct osp_device *d) { - struct obd_import *imp; - int rc = 0; - ENTRY; - - /* release last_used file */ - osp_last_used_fini(env, d); + struct obd_import *imp; + int rc = 0; imp = d->opd_obd->u.cli.cl_import; /* Mark import deactivated now, so we don't try to reconnect if any * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't * fully deactivate the import, or that would drop all requests. */ + LASSERT(imp != NULL); cfs_spin_lock(&imp->imp_lock); imp->imp_deactive = 1; cfs_spin_unlock(&imp->imp_lock); @@ -222,6 +221,31 @@ static int osp_shutdown(const struct lu_env *env, struct osp_device *d) RETURN(rc); } +static int osp_shutdown(const struct lu_env *env, struct osp_device *d) +{ + int rc = 0; + ENTRY; + + if (is_osp_on_ost(d->opd_obd->obd_name)) { + rc = osp_disconnect(d); + RETURN(rc); + } + + LASSERT(env); + /* release last_used file */ + osp_last_used_fini(env, d); + + rc = osp_disconnect(d); + + /* stop precreate thread */ + osp_precreate_fini(d); + + /* stop sync thread */ + osp_sync_fini(d); + + RETURN(rc); +} + static int osp_process_config(const struct lu_env *env, struct lu_device *dev, struct lustre_cfg *lcfg) { @@ -233,14 +257,15 @@ static int osp_process_config(const struct lu_env *env, switch (lcfg->lcfg_command) { case LCFG_CLEANUP: - lu_dev_del_linkage(dev->ld_site, dev); + if (!is_osp_on_ost(d->opd_obd->obd_name)) + lu_dev_del_linkage(dev->ld_site, dev); rc = osp_shutdown(env, d); break; case LCFG_PARAM: lprocfs_osp_init_vars(&lvars); LASSERT(d->opd_obd); - rc = class_process_proc_param(PARAM_OSP, lvars.obd_vars, + rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars, lcfg, d->opd_obd); if (rc > 0) rc = 0; @@ -273,6 +298,7 @@ static int osp_recovery_complete(const struct lu_env *env, ENTRY; osp->opd_recovery_completed = 1; + cfs_waitq_signal(&osp->opd_pre_waitq); RETURN(rc); } @@ -282,6 +308,49 @@ const struct lu_device_operations osp_lu_ops = { .ldo_recovery_complete = osp_recovery_complete, }; +/** + * provides with statfs from corresponded OST + * + */ +static int osp_statfs(const struct lu_env *env, struct dt_device *dev, + struct obd_statfs *sfs) +{ + struct osp_device *d = dt2osp_dev(dev); + + ENTRY; + + if (unlikely(d->opd_imp_active == 0)) { + /* + * in case of inactive OST we return nulls + * so that caller can understand this device + * is unusable for new objects + * + * XXX: shouldn't we take normal statfs and fill + * just few specific fields with zeroes? + */ + memset(sfs, 0, sizeof(*sfs)); + sfs->os_bsize = 4096; + RETURN(0); + } + + /* return recently updated data */ + *sfs = d->opd_statfs; + + /* + * layer above osp (usually lod) can use ffree to estimate + * how many objects are available for immediate creation + */ + cfs_spin_lock(&d->opd_pre_lock); + sfs->os_ffree = d->opd_pre_last_created - d->opd_pre_next; + cfs_spin_unlock(&d->opd_pre_lock); + + CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, " + LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name, + sfs->os_blocks, sfs->os_bfree, sfs->os_bavail, + sfs->os_files, sfs->os_ffree); + RETURN(0); +} + static int osp_sync(const struct lu_env *env, struct dt_device *dev) { ENTRY; @@ -293,7 +362,8 @@ static int osp_sync(const struct lu_env *env, struct dt_device *dev) RETURN(0); } -static const struct dt_device_operations osp_dt_ops = { +const struct dt_device_operations osp_dt_ops = { + .dt_statfs = osp_statfs, .dt_sync = osp_sync, }; @@ -455,6 +525,21 @@ static int osp_init0(const struct lu_env *env, struct osp_device *m, GOTO(out_proc, rc); /* + * Initialize precreation thread, it handles new connections as well + */ + rc = osp_init_precreate(m); + if (rc) + GOTO(out_last_used, rc); + + /* + * Initialize synhronization mechanism taking care of propogating + * changes to OST in near transactional manner + */ + rc = osp_sync_init(env, m); + if (rc) + GOTO(out_precreat, rc); + + /* * Initiate connect to OST */ ll_generate_random_uuid(uuid); @@ -470,6 +555,12 @@ static int osp_init0(const struct lu_env *env, struct osp_device *m, RETURN(0); out: + /* stop sync thread */ + osp_sync_fini(m); +out_precreat: + /* stop precreate thread */ + osp_precreate_fini(m); +out_last_used: osp_last_used_fini(env, m); out_proc: ptlrpc_lprocfs_unregister_obd(m->opd_obd); @@ -493,6 +584,10 @@ static struct lu_device *osp_device_free(const struct lu_env *env, ENTRY; + if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) { + LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); + lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer); + } dt_device_fini(&m->opd_dt_dev); OBD_FREE_PTR(m); RETURN(NULL); @@ -513,7 +608,10 @@ static struct lu_device *osp_device_alloc(const struct lu_env *env, l = osp2lu_dev(m); dt_device_init(&m->opd_dt_dev, t); - rc = osp_init0(env, m, t, lcfg); + if (is_osp_on_ost(lustre_cfg_string(lcfg, 0))) + rc = osp_init_for_ost(env, m, t, lcfg); + else + rc = osp_init0(env, m, t, lcfg); if (rc != 0) { osp_device_free(env, l); l = ERR_PTR(rc); @@ -531,8 +629,11 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, ENTRY; - LASSERT(m->opd_storage_exp); - obd_disconnect(m->opd_storage_exp); + if (m->opd_storage_exp) + obd_disconnect(m->opd_storage_exp); + + if (is_osp_on_ost(m->opd_obd->obd_name)) + osp_fini_for_ost(m); imp = m->opd_obd->u.cli.cl_import; @@ -589,6 +690,8 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, RETURN(rc); *exp = class_conn2export(&conn); + if (is_osp_on_ost(obd->obd_name)) + osp->opd_exp = *exp; /* Why should there ever be more than 1 connect? */ osp->opd_connects++; @@ -608,7 +711,8 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, OBD_CONNECT_OSS_CAPA | OBD_CONNECT_REQPORTAL | OBD_CONNECT_SKIP_ORPHAN | - OBD_CONNECT_VERSION; + OBD_CONNECT_VERSION | + OBD_CONNECT_FID; ocd->ocd_version = LUSTRE_VERSION_CODE; LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX); ocd->ocd_index = data->ocd_index; @@ -635,7 +739,6 @@ static int osp_obd_disconnect(struct obd_export *exp) struct obd_device *obd = exp->exp_obd; struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev); int rc; - ENTRY; /* Only disconnect the underlying layers on the final disconnect. */ @@ -643,14 +746,192 @@ static int osp_obd_disconnect(struct obd_export *exp) osp->opd_connects--; rc = class_disconnect(exp); + if (rc) { + CERROR("%s: class disconnect error: rc = %d\n", + obd->obd_name, rc); + RETURN(rc); + } /* destroy the device */ - if (rc == 0) + if (!is_osp_on_ost(obd->obd_name)) class_manual_cleanup(obd); RETURN(rc); } +/* + * lprocfs helpers still use OBD API, let's keep obd_statfs() support + */ +static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp, + struct obd_statfs *osfs, __u64 max_age, __u32 flags) +{ + struct obd_statfs *msfs; + struct ptlrpc_request *req; + struct obd_import *imp = NULL; + int rc; + + ENTRY; + + /* Since the request might also come from lprocfs, so we need + * sync this with client_disconnect_export Bug15684 */ + cfs_down_read(&exp->exp_obd->u.cli.cl_sem); + if (exp->exp_obd->u.cli.cl_import) + imp = class_import_get(exp->exp_obd->u.cli.cl_import); + cfs_up_read(&exp->exp_obd->u.cli.cl_sem); + if (!imp) + RETURN(-ENODEV); + + /* We could possibly pass max_age in the request (as an absolute + * timestamp or a "seconds.usec ago") so the target can avoid doing + * extra calls into the filesystem if that isn't necessary (e.g. + * during mount that would help a bit). Having relative timestamps + * is not so great if request processing is slow, while absolute + * timestamps are not ideal because they need time synchronization. */ + req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS); + + class_import_put(imp); + + if (req == NULL) + RETURN(-ENOMEM); + + rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS); + if (rc) { + ptlrpc_request_free(req); + RETURN(rc); + } + ptlrpc_request_set_replen(req); + req->rq_request_portal = OST_CREATE_PORTAL; + ptlrpc_at_set_req_timeout(req); + + if (flags & OBD_STATFS_NODELAY) { + /* procfs requests not want stat in wait for avoid deadlock */ + req->rq_no_resend = 1; + req->rq_no_delay = 1; + } + + rc = ptlrpc_queue_wait(req); + if (rc) + GOTO(out, rc); + + msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS); + if (msfs == NULL) + GOTO(out, rc = -EPROTO); + + *osfs = *msfs; + + EXIT; +out: + ptlrpc_req_finished(req); + return rc; +} + +static int osp_import_event(struct obd_device *obd, struct obd_import *imp, + enum obd_import_event event) +{ + struct osp_device *d = lu2osp_dev(obd->obd_lu_dev); + + switch (event) { + case IMP_EVENT_DISCON: + d->opd_got_disconnected = 1; + d->opd_imp_connected = 0; + if (is_osp_on_ost(d->opd_obd->obd_name)) + break; + osp_pre_update_status(d, -ENODEV); + cfs_waitq_signal(&d->opd_pre_waitq); + CDEBUG(D_HA, "got disconnected\n"); + break; + case IMP_EVENT_INACTIVE: + d->opd_imp_active = 0; + if (is_osp_on_ost(d->opd_obd->obd_name)) + break; + osp_pre_update_status(d, -ENODEV); + cfs_waitq_signal(&d->opd_pre_waitq); + CDEBUG(D_HA, "got inactive\n"); + break; + case IMP_EVENT_ACTIVE: + d->opd_imp_active = 1; + if (d->opd_got_disconnected) + d->opd_new_connection = 1; + d->opd_imp_connected = 1; + d->opd_imp_seen_connected = 1; + if (is_osp_on_ost(d->opd_obd->obd_name)) + break; + cfs_waitq_signal(&d->opd_pre_waitq); + __osp_sync_check_for_work(d); + CDEBUG(D_HA, "got connected\n"); + break; + case IMP_EVENT_INVALIDATE: + if (obd->obd_namespace == NULL) + break; + ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY); + break; + case IMP_EVENT_OCD: + break; + default: + CERROR("%s: unsupported import event: %#x\n", + obd->obd_name, event); + } + return 0; +} + +static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len, + void *karg, void *uarg) +{ + struct obd_device *obd = exp->exp_obd; + struct osp_device *d; + struct obd_ioctl_data *data = karg; + int rc = 0; + + ENTRY; + + LASSERT(obd->obd_lu_dev); + d = lu2osp_dev(obd->obd_lu_dev); + LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops); + + if (!cfs_try_module_get(THIS_MODULE)) { + CERROR("%s: can't get module. Is it alive?", obd->obd_name); + return -EINVAL; + } + + switch (cmd) { + case OBD_IOC_CLIENT_RECOVER: + rc = ptlrpc_recover_import(obd->u.cli.cl_import, + data->ioc_inlbuf1, 0); + if (rc > 0) + rc = 0; + break; + case IOC_OSC_SET_ACTIVE: + rc = ptlrpc_set_import_active(obd->u.cli.cl_import, + data->ioc_offset); + break; + case OBD_IOC_PING_TARGET: + rc = ptlrpc_obd_ping(obd); + break; + default: + CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name, + cmd, cfs_curproc_comm()); + rc = -ENOTTY; + } + cfs_module_put(THIS_MODULE); + return rc; +} + +static int osp_obd_health_check(const struct lu_env *env, + struct obd_device *obd) +{ + struct osp_device *d = lu2osp_dev(obd->obd_lu_dev); + + ENTRY; + + /* + * 1.8/2.0 behaviour is that OST being connected once at least + * is considired "healthy". and one "healty" OST is enough to + * allow lustre clients to connect to MDS + */ + LASSERT(d); + RETURN(!d->opd_imp_seen_connected); +} + /* context key constructor/destructor: mdt_key_init, mdt_key_fini */ LU_KEY_INIT_FINI(osp, struct osp_thread_info); static void osp_key_exit(const struct lu_context *ctx, @@ -705,6 +986,10 @@ static struct obd_ops osp_obd_device_ops = { .o_reconnect = osp_reconnect, .o_connect = osp_obd_connect, .o_disconnect = osp_obd_disconnect, + .o_health_check = osp_obd_health_check, + .o_import_event = osp_import_event, + .o_iocontrol = osp_iocontrol, + .o_statfs = osp_obd_statfs, }; static int __init osp_mod_init(void)