From 7f412954ad38b540ccbbf241b25c594faf5ca79d Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Fri, 15 Feb 2019 19:12:34 +0800 Subject: [PATCH] LU-11213 mdc: add async statfs Add obd_statfs_async() interface for MDC, the statfs request is sent by ptlrpcd. This statfs result is for each MDT separately, it's different from current cached statfs which is aggregated statfs of all MDTs. The max age of statfs result is decided by lmv_desc.ld_qos_maxage. It will deactivate MDC on failure, and activate MDC on success. Signed-off-by: Lai Siyao Change-Id: I8e1bd104fb60ff81e2eb26e49a89a5baf8050d47 Reviewed-on: https://review.whamcloud.com/34359 Reviewed-by: Andreas Dilger Reviewed-by: Hongchao Zhang Reviewed-by: Alex Zhuravlev Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/obd.h | 4 +++ lustre/include/obd_class.h | 17 ++-------- lustre/lmv/lmv_internal.h | 2 ++ lustre/lmv/lmv_obd.c | 44 ++++++++++++++++++++++++++ lustre/mdc/mdc_request.c | 79 +++++++++++++++++++++++++++++++++++++--------- lustre/osc/osc_request.c | 16 ++++++++++ 6 files changed, 132 insertions(+), 30 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index fdb2ecd..b566809 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -87,6 +87,8 @@ typedef int (*obd_enqueue_update_f)(void *cookie, int rc); struct obd_info { /* OBD_STATFS_* flags */ __u64 oi_flags; + struct obd_device *oi_obd; + struct lmv_tgt_desc *oi_tgt; /* statfs data specific for every OSC, if needed at all. */ struct obd_statfs *oi_osfs; /* An update callback which is called to update some data on upper @@ -440,6 +442,8 @@ struct lmv_tgt_desc { struct obd_export *ltd_exp; __u32 ltd_idx; struct mutex ltd_fid_mutex; + struct obd_statfs ltd_statfs; + time64_t ltd_statfs_age; unsigned long ltd_active:1; /* target up for requests */ }; diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index a4e64b2b8..7f58d316 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1023,21 +1023,8 @@ static inline int obd_statfs_async(struct obd_export *exp, CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n", obd->obd_name, obd->obd_osfs_age, max_age); - if (obd->obd_osfs_age < max_age) { - rc = OBP(obd, statfs_async)(exp, oinfo, max_age, rqset); - } else { - CDEBUG(D_SUPER, - "%s: use %p cache blocks %llu/%llu objects %llu/%llu\n", - obd->obd_name, &obd->obd_osfs, - obd->obd_osfs.os_bavail, obd->obd_osfs.os_blocks, - obd->obd_osfs.os_ffree, obd->obd_osfs.os_files); - spin_lock(&obd->obd_osfs_lock); - memcpy(oinfo->oi_osfs, &obd->obd_osfs, sizeof(*oinfo->oi_osfs)); - spin_unlock(&obd->obd_osfs_lock); - oinfo->oi_flags |= OBD_STATFS_FROM_CACHE; - if (oinfo->oi_cb_up) - oinfo->oi_cb_up(oinfo, 0); - } + rc = OBP(obd, statfs_async)(exp, oinfo, max_age, rqset); + RETURN(rc); } diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h index 0ad7432..44debf0 100644 --- a/lustre/lmv/lmv_internal.h +++ b/lustre/lmv/lmv_internal.h @@ -61,6 +61,8 @@ int lmv_revalidate_slaves(struct obd_export *exp, int lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data, struct ptlrpc_request **preq); +int lmv_statfs_check_update(struct obd_device *obd, struct lmv_tgt_desc *tgt); + static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv) { return container_of0(lmv, struct obd_device, u.lmv); diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 5110b22..da0a7102 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -360,6 +360,8 @@ int lmv_connect_mdc(struct obd_device *obd, struct lmv_tgt_desc *tgt) mdc_obd->obd_name, mdc_obd->obd_uuid.uuid, atomic_read(&obd->obd_refcount)); + lmv_statfs_check_update(obd, tgt); + if (lmv->lmv_tgts_kobj) /* Even if we failed to create the link, that's fine */ rc = sysfs_create_link(lmv->lmv_tgts_kobj, @@ -1289,6 +1291,7 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg) obd_str2uuid(&lmv->desc.ld_uuid, desc->ld_uuid.uuid); lmv->desc.ld_tgt_count = 0; lmv->desc.ld_active_tgt_count = 0; + lmv->desc.ld_qos_maxage = 60; lmv->max_def_easize = 0; lmv->max_easize = 0; @@ -1454,6 +1457,47 @@ out_free_temp: return rc; } +static int lmv_statfs_update(void *cookie, int rc) +{ + struct obd_info *oinfo = cookie; + struct obd_device *obd = oinfo->oi_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt = oinfo->oi_tgt; + struct obd_statfs *osfs = oinfo->oi_osfs; + + /* + * NB: don't deactivate TGT upon error, because we may not trigger async + * statfs any longer, then there is no chance to activate TGT. + */ + if (!rc) { + spin_lock(&lmv->lmv_lock); + tgt->ltd_statfs = *osfs; + tgt->ltd_statfs_age = ktime_get_seconds(); + spin_unlock(&lmv->lmv_lock); + } + + return rc; +} + +/* update tgt statfs async if it's ld_qos_maxage old */ +int lmv_statfs_check_update(struct obd_device *obd, struct lmv_tgt_desc *tgt) +{ + struct obd_info oinfo = { + .oi_obd = obd, + .oi_tgt = tgt, + .oi_cb_up = lmv_statfs_update, + }; + int rc; + + if (ktime_get_seconds() - tgt->ltd_statfs_age < + obd->u.lmv.desc.ld_qos_maxage) + return 0; + + rc = obd_statfs_async(tgt->ltd_exp, &oinfo, 0, NULL); + + return rc; +} + static int lmv_get_root(struct obd_export *exp, const char *fileset, struct lu_fid *fid) { diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 291bb03..855ce59 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1571,6 +1571,54 @@ fail: goto out_unlock; } +static int mdc_statfs_interpret(const struct lu_env *env, + struct ptlrpc_request *req, void *args, int rc) +{ + struct obd_info *oinfo = args; + struct obd_statfs *osfs; + + if (!rc) { + osfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS); + if (!osfs) + return -EPROTO; + + oinfo->oi_osfs = osfs; + + CDEBUG(D_CACHE, "blocks=%llu free=%llu avail=%llu " + "objects=%llu free=%llu state=%x\n", + osfs->os_blocks, osfs->os_bfree, osfs->os_bavail, + osfs->os_files, osfs->os_ffree, osfs->os_state); + } + + oinfo->oi_cb_up(oinfo, rc); + + return rc; +} + +static int mdc_statfs_async(struct obd_export *exp, + struct obd_info *oinfo, time64_t max_age, + struct ptlrpc_request_set *unused) +{ + struct ptlrpc_request *req; + struct obd_info *aa; + + req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), &RQF_MDS_STATFS, + LUSTRE_MDS_VERSION, MDS_STATFS); + if (req == NULL) + return -ENOMEM; + + ptlrpc_request_set_replen(req); + req->rq_interpret_reply = mdc_statfs_interpret; + + CLASSERT(sizeof(*aa) <= sizeof(req->rq_async_args)); + aa = ptlrpc_req_async_args(req); + *aa = *oinfo; + + ptlrpcd_add_req(req); + + return 0; +} + static int mdc_statfs(const struct lu_env *env, struct obd_export *exp, struct obd_statfs *osfs, time64_t max_age, __u32 flags) @@ -2784,25 +2832,26 @@ static int mdc_cleanup(struct obd_device *obd) } static struct obd_ops mdc_obd_ops = { - .o_owner = THIS_MODULE, - .o_setup = mdc_setup, - .o_precleanup = mdc_precleanup, - .o_cleanup = mdc_cleanup, - .o_add_conn = client_import_add_conn, - .o_del_conn = client_import_del_conn, - .o_connect = client_connect_import, + .o_owner = THIS_MODULE, + .o_setup = mdc_setup, + .o_precleanup = mdc_precleanup, + .o_cleanup = mdc_cleanup, + .o_add_conn = client_import_add_conn, + .o_del_conn = client_import_del_conn, + .o_connect = client_connect_import, .o_reconnect = osc_reconnect, .o_disconnect = osc_disconnect, - .o_iocontrol = mdc_iocontrol, - .o_set_info_async = mdc_set_info_async, - .o_statfs = mdc_statfs, + .o_iocontrol = mdc_iocontrol, + .o_set_info_async = mdc_set_info_async, + .o_statfs = mdc_statfs, + .o_statfs_async = mdc_statfs_async, .o_fid_init = client_fid_init, .o_fid_fini = client_fid_fini, - .o_fid_alloc = mdc_fid_alloc, - .o_import_event = mdc_import_event, - .o_get_info = mdc_get_info, - .o_get_uuid = mdc_get_uuid, - .o_quotactl = mdc_quotactl, + .o_fid_alloc = mdc_fid_alloc, + .o_import_event = mdc_import_event, + .o_get_info = mdc_get_info, + .o_get_uuid = mdc_get_uuid, + .o_quotactl = mdc_quotactl, }; static struct md_ops mdc_md_ops = { diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index a44e8b7..44c7ad1 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -2730,6 +2730,22 @@ static int osc_statfs_async(struct obd_export *exp, int rc; ENTRY; + if (obd->obd_osfs_age >= max_age) { + CDEBUG(D_SUPER, + "%s: use %p cache blocks %llu/%llu objects %llu/%llu\n", + obd->obd_name, &obd->obd_osfs, + obd->obd_osfs.os_bavail, obd->obd_osfs.os_blocks, + obd->obd_osfs.os_ffree, obd->obd_osfs.os_files); + spin_lock(&obd->obd_osfs_lock); + memcpy(oinfo->oi_osfs, &obd->obd_osfs, sizeof(*oinfo->oi_osfs)); + spin_unlock(&obd->obd_osfs_lock); + oinfo->oi_flags |= OBD_STATFS_FROM_CACHE; + if (oinfo->oi_cb_up) + oinfo->oi_cb_up(oinfo, 0); + + RETURN(0); + } + /* We could possibly pass max_age in the request (as an absolute * timestamp or a "seconds.usec ago") so the target can avoid doing * extra calls into the filesystem if that isn't necessary (e.g. -- 1.8.3.1