LU-11213 mdc: add async statfs

Add obd_statfs_async() interface for MDC, the statfs request
is sent by ptlrpcd.

This statfs result is for each MDT separately, it's different
from current cached statfs which is aggregated statfs of all
MDTs.

The max age of statfs result is decided by lmv_desc.ld_qos_maxage.

It will deactivate MDC on failure, and activate MDC on success.

Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: I8e1bd104fb60ff81e2eb26e49a89a5baf8050d47
Reviewed-on: https://review.whamcloud.com/34359
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Hongchao Zhang <hongchao@whamcloud.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Tested-by: Jenkins
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index fdb2ecd..b566809 100644 (file)
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -87,6 +87,8 @@ typedef int (*obd_enqueue_update_f)(void *cookie, int rc);
 struct obd_info {
        /* OBD_STATFS_* flags */
        __u64                   oi_flags;
 struct obd_info {
        /* OBD_STATFS_* flags */
        __u64                   oi_flags;

+       struct obd_device      *oi_obd;
+       struct lmv_tgt_desc    *oi_tgt;

         /* statfs data specific for every OSC, if needed at all. */
         struct obd_statfs      *oi_osfs;
         /* An update callback which is called to update some data on upper
         /* statfs data specific for every OSC, if needed at all. */
         struct obd_statfs      *oi_osfs;
         /* An update callback which is called to update some data on upper


@@ -440,6 +442,8 @@ struct lmv_tgt_desc {
        struct obd_export       *ltd_exp;
        __u32                   ltd_idx;
        struct mutex            ltd_fid_mutex;
        struct obd_export       *ltd_exp;
        __u32                   ltd_idx;
        struct mutex            ltd_fid_mutex;

+       struct obd_statfs       ltd_statfs;
+       time64_t                ltd_statfs_age;

        unsigned long           ltd_active:1; /* target up for requests */
 };
 
        unsigned long           ltd_active:1; /* target up for requests */
 };
 

diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index a4e64b2..7f58d31 100644 (file)
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -1023,21 +1023,8 @@ static inline int obd_statfs_async(struct obd_export *exp,
 
        CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n",
               obd->obd_name, obd->obd_osfs_age, max_age);
 
        CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n",
               obd->obd_name, obd->obd_osfs_age, max_age);

-       if (obd->obd_osfs_age < max_age) {
-               rc = OBP(obd, statfs_async)(exp, oinfo, max_age, rqset);
-       } else {
-               CDEBUG(D_SUPER,
-                      "%s: use %p cache blocks %llu/%llu objects %llu/%llu\n",
-                      obd->obd_name, &obd->obd_osfs,
-                      obd->obd_osfs.os_bavail, obd->obd_osfs.os_blocks,
-                      obd->obd_osfs.os_ffree, obd->obd_osfs.os_files);
-               spin_lock(&obd->obd_osfs_lock);
-               memcpy(oinfo->oi_osfs, &obd->obd_osfs, sizeof(*oinfo->oi_osfs));
-               spin_unlock(&obd->obd_osfs_lock);
-               oinfo->oi_flags |= OBD_STATFS_FROM_CACHE;
-               if (oinfo->oi_cb_up)
-                       oinfo->oi_cb_up(oinfo, 0);
-       }
+       rc = OBP(obd, statfs_async)(exp, oinfo, max_age, rqset);
+

        RETURN(rc);
 }
 
        RETURN(rc);
 }
 

diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h
index 0ad7432..44debf0 100644 (file)
--- a/lustre/lmv/lmv_internal.h
+++ b/lustre/lmv/lmv_internal.h
@@ -61,6 +61,8 @@ int lmv_revalidate_slaves(struct obd_export *exp,
 int lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
                     struct ptlrpc_request **preq);
 
 int lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
                     struct ptlrpc_request **preq);
 

+int lmv_statfs_check_update(struct obd_device *obd, struct lmv_tgt_desc *tgt);
+

 static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv)
 {
        return container_of0(lmv, struct obd_device, u.lmv);
 static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv)
 {
        return container_of0(lmv, struct obd_device, u.lmv);

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 5110b22..da0a710 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -360,6 +360,8 @@ int lmv_connect_mdc(struct obd_device *obd, struct lmv_tgt_desc *tgt)
                mdc_obd->obd_name, mdc_obd->obd_uuid.uuid,
                atomic_read(&obd->obd_refcount));
 
                mdc_obd->obd_name, mdc_obd->obd_uuid.uuid,
                atomic_read(&obd->obd_refcount));
 

+       lmv_statfs_check_update(obd, tgt);
+

        if (lmv->lmv_tgts_kobj)
                /* Even if we failed to create the link, that's fine */
                rc = sysfs_create_link(lmv->lmv_tgts_kobj,
        if (lmv->lmv_tgts_kobj)
                /* Even if we failed to create the link, that's fine */
                rc = sysfs_create_link(lmv->lmv_tgts_kobj,


@@ -1289,6 +1291,7 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
        obd_str2uuid(&lmv->desc.ld_uuid, desc->ld_uuid.uuid);
        lmv->desc.ld_tgt_count = 0;
        lmv->desc.ld_active_tgt_count = 0;
        obd_str2uuid(&lmv->desc.ld_uuid, desc->ld_uuid.uuid);
        lmv->desc.ld_tgt_count = 0;
        lmv->desc.ld_active_tgt_count = 0;

+       lmv->desc.ld_qos_maxage = 60;

        lmv->max_def_easize = 0;
        lmv->max_easize = 0;
 
        lmv->max_def_easize = 0;
        lmv->max_easize = 0;
 


@@ -1454,6 +1457,47 @@ out_free_temp:
         return rc;
 }
 
         return rc;
 }
 

+static int lmv_statfs_update(void *cookie, int rc)
+{
+       struct obd_info *oinfo = cookie;
+       struct obd_device *obd = oinfo->oi_obd;
+       struct lmv_obd *lmv = &obd->u.lmv;
+       struct lmv_tgt_desc *tgt = oinfo->oi_tgt;
+       struct obd_statfs *osfs = oinfo->oi_osfs;
+
+       /*
+        * NB: don't deactivate TGT upon error, because we may not trigger async
+        * statfs any longer, then there is no chance to activate TGT.
+        */
+       if (!rc) {
+               spin_lock(&lmv->lmv_lock);
+               tgt->ltd_statfs = *osfs;
+               tgt->ltd_statfs_age = ktime_get_seconds();
+               spin_unlock(&lmv->lmv_lock);
+       }
+
+       return rc;
+}
+
+/* update tgt statfs async if it's ld_qos_maxage old */
+int lmv_statfs_check_update(struct obd_device *obd, struct lmv_tgt_desc *tgt)
+{
+       struct obd_info oinfo = {
+               .oi_obd = obd,
+               .oi_tgt = tgt,
+               .oi_cb_up = lmv_statfs_update,
+       };
+       int rc;
+
+       if (ktime_get_seconds() - tgt->ltd_statfs_age <
+           obd->u.lmv.desc.ld_qos_maxage)
+               return 0;
+
+       rc = obd_statfs_async(tgt->ltd_exp, &oinfo, 0, NULL);
+
+       return rc;
+}
+

 static int lmv_get_root(struct obd_export *exp, const char *fileset,
                        struct lu_fid *fid)
 {
 static int lmv_get_root(struct obd_export *exp, const char *fileset,
                        struct lu_fid *fid)
 {

diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index 291bb03..855ce59 100644 (file)
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -1571,6 +1571,54 @@ fail:
        goto out_unlock;
 }
 
        goto out_unlock;
 }
 

+static int mdc_statfs_interpret(const struct lu_env *env,
+                               struct ptlrpc_request *req, void *args, int rc)
+{
+       struct obd_info *oinfo = args;
+       struct obd_statfs *osfs;
+
+       if (!rc) {
+               osfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
+               if (!osfs)
+                       return -EPROTO;
+
+               oinfo->oi_osfs = osfs;
+
+               CDEBUG(D_CACHE, "blocks=%llu free=%llu avail=%llu "
+                      "objects=%llu free=%llu state=%x\n",
+                       osfs->os_blocks, osfs->os_bfree, osfs->os_bavail,
+                       osfs->os_files, osfs->os_ffree, osfs->os_state);
+       }
+
+       oinfo->oi_cb_up(oinfo, rc);
+
+       return rc;
+}
+
+static int mdc_statfs_async(struct obd_export *exp,
+                           struct obd_info *oinfo, time64_t max_age,
+                           struct ptlrpc_request_set *unused)
+{
+       struct ptlrpc_request *req;
+       struct obd_info *aa;
+
+       req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), &RQF_MDS_STATFS,
+                                       LUSTRE_MDS_VERSION, MDS_STATFS);
+       if (req == NULL)
+               return -ENOMEM;
+
+       ptlrpc_request_set_replen(req);
+       req->rq_interpret_reply = mdc_statfs_interpret;
+
+       CLASSERT(sizeof(*aa) <= sizeof(req->rq_async_args));
+       aa = ptlrpc_req_async_args(req);
+       *aa = *oinfo;
+
+       ptlrpcd_add_req(req);
+
+       return 0;
+}
+

 static int mdc_statfs(const struct lu_env *env,
                       struct obd_export *exp, struct obd_statfs *osfs,
                      time64_t max_age, __u32 flags)
 static int mdc_statfs(const struct lu_env *env,
                       struct obd_export *exp, struct obd_statfs *osfs,
                      time64_t max_age, __u32 flags)


@@ -2784,25 +2832,26 @@ static int mdc_cleanup(struct obd_device *obd)
 }
 
 static struct obd_ops mdc_obd_ops = {
 }
 
 static struct obd_ops mdc_obd_ops = {

-        .o_owner            = THIS_MODULE,
-        .o_setup            = mdc_setup,
-        .o_precleanup       = mdc_precleanup,
-        .o_cleanup          = mdc_cleanup,
-        .o_add_conn         = client_import_add_conn,
-        .o_del_conn         = client_import_del_conn,
-        .o_connect          = client_connect_import,
+       .o_owner            = THIS_MODULE,
+       .o_setup            = mdc_setup,
+       .o_precleanup       = mdc_precleanup,
+       .o_cleanup          = mdc_cleanup,
+       .o_add_conn         = client_import_add_conn,
+       .o_del_conn         = client_import_del_conn,
+       .o_connect          = client_connect_import,

        .o_reconnect        = osc_reconnect,
        .o_disconnect       = osc_disconnect,
        .o_reconnect        = osc_reconnect,
        .o_disconnect       = osc_disconnect,

-        .o_iocontrol        = mdc_iocontrol,
-        .o_set_info_async   = mdc_set_info_async,
-        .o_statfs           = mdc_statfs,
+       .o_iocontrol        = mdc_iocontrol,
+       .o_set_info_async   = mdc_set_info_async,
+       .o_statfs           = mdc_statfs,
+       .o_statfs_async     = mdc_statfs_async,

        .o_fid_init         = client_fid_init,
        .o_fid_fini         = client_fid_fini,
        .o_fid_init         = client_fid_init,
        .o_fid_fini         = client_fid_fini,

-        .o_fid_alloc        = mdc_fid_alloc,
-        .o_import_event     = mdc_import_event,
-        .o_get_info         = mdc_get_info,
-        .o_get_uuid         = mdc_get_uuid,
-        .o_quotactl         = mdc_quotactl,
+       .o_fid_alloc        = mdc_fid_alloc,
+       .o_import_event     = mdc_import_event,
+       .o_get_info         = mdc_get_info,
+       .o_get_uuid         = mdc_get_uuid,
+       .o_quotactl         = mdc_quotactl,

 };
 
 static struct md_ops mdc_md_ops = {
 };
 
 static struct md_ops mdc_md_ops = {

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index a44e8b7..44c7ad1 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -2730,6 +2730,22 @@ static int osc_statfs_async(struct obd_export *exp,
        int rc;
         ENTRY;
 
        int rc;
         ENTRY;
 

+       if (obd->obd_osfs_age >= max_age) {
+               CDEBUG(D_SUPER,
+                      "%s: use %p cache blocks %llu/%llu objects %llu/%llu\n",
+                      obd->obd_name, &obd->obd_osfs,
+                      obd->obd_osfs.os_bavail, obd->obd_osfs.os_blocks,
+                      obd->obd_osfs.os_ffree, obd->obd_osfs.os_files);
+               spin_lock(&obd->obd_osfs_lock);
+               memcpy(oinfo->oi_osfs, &obd->obd_osfs, sizeof(*oinfo->oi_osfs));
+               spin_unlock(&obd->obd_osfs_lock);
+               oinfo->oi_flags |= OBD_STATFS_FROM_CACHE;
+               if (oinfo->oi_cb_up)
+                       oinfo->oi_cb_up(oinfo, 0);
+
+               RETURN(0);
+       }
+

         /* We could possibly pass max_age in the request (as an absolute
          * timestamp or a "seconds.usec ago") so the target can avoid doing
          * extra calls into the filesystem if that isn't necessary (e.g.
         /* We could possibly pass max_age in the request (as an absolute
          * timestamp or a "seconds.usec ago") so the target can avoid doing
          * extra calls into the filesystem if that isn't necessary (e.g.