From: John L. Hammond Date: Thu, 13 Dec 2012 21:47:00 +0000 (-0600) Subject: LU-2484 obd: add md_stats to MDC and LMV devices X-Git-Tag: 2.4.92~12 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;ds=sidebyside;h=7b28134cbb00365f6fd6996979986408edab721e;p=fs%2Flustre-release.git LU-2484 obd: add md_stats to MDC and LMV devices Enable md_stats for MDC and LMV devices and fix the definition of EXP_MD_COUNTER_INCREMENT() so that they will be tallied. These stats track usage of the md_ops methods (from the OBD layer, not to be confused with md_device methods) and are exported through the files /proc/fs/lustre/{lmv,mdc}/*/md_stats. Rename m_sync to m_fsync making the counter name (fsync) more intuitive. Prune the minimally useful operations from set of md_ops to be counted. The operations counted are close, create, enqueue, getattr, intent_lock, link, rename, setattr, fsync, readpage, unlink, setxattr, egtxattr, intent_getattr_async, and revalidate_lock. Add assertions to lprocfs_counter_{add,sub}() to ensure that the counter index lies in the appropriate range for stats. Remove the corresponding assertions from OBD_COUNTER_INCREMENT(), EXP_COUNTER_INCREMENT(), and EXP_MD_COUNTER_INCREMENT(). Add macros NUM_OBD_STATS and NUM_MD_STATS for use when allocating stats. In mdt_export_stats_init() allocate and initialize the nid_stats member of struct exp_nid_stats using only the MDT private counters. This does not affect the contents of /proc/fs/lustre/mdt/*/exports/*/stats but does save memory. Add a flag to struct obd_device to indicate if EXP_COUNTER_INCREMENT() should increment stats in the nid_stats member (as in the case of OFD) or not (MDT). Remove the unused member exp_md_stats from struct obd_export. Signed-off-by: John L. Hammond Signed-off-by: John L. Hammond Change-Id: Idccd30e13adbc11d10b544cdfbe1d23123176e94 Reviewed-on: http://review.whamcloud.com/4827 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: wangdi Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h index ad49db6..fabea52 100644 --- a/lustre/include/lustre_export.h +++ b/lustre/include/lustre_export.h @@ -191,7 +191,6 @@ struct obd_export { */ struct obd_import *exp_imp_reverse; struct nid_stat *exp_nid_stats; - struct lprocfs_stats *exp_md_stats; /** Active connetion */ struct ptlrpc_connection *exp_connection; /** Connection count value from last succesful reconnect rpc */ diff --git a/lustre/include/obd.h b/lustre/include/obd.h index ce55df9..a3801d2 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -841,35 +841,38 @@ struct obd_llog_group { #define OBD_DEV_BY_DEVNAME 0xffffd0de struct obd_device { - struct obd_type *obd_type; - __u32 obd_magic; + struct obd_type *obd_type; + __u32 obd_magic; /* common and UUID name of this device */ - char obd_name[MAX_OBD_NAME]; - struct obd_uuid obd_uuid; - - struct lu_device *obd_lu_dev; - - int obd_minor; - /* bitfield modification is protected by obd_dev_lock */ - unsigned long obd_attached:1, /* finished attach */ - obd_set_up:1, /* finished setup */ - obd_recovering:1, /* there are recoverable clients */ - obd_abort_recovery:1,/* recovery expired */ - obd_version_recov:1, /* obd uses version checking */ - obd_replayable:1, /* recovery is enabled; inform clients */ - obd_no_transno:1, /* no committed-transno notification */ - obd_no_recov:1, /* fail instead of retry messages */ - obd_stopping:1, /* started cleanup */ - obd_starting:1, /* started setup */ - obd_force:1, /* cleanup with > 0 obd refcount */ - obd_fail:1, /* cleanup with failover */ - obd_async_recov:1, /* allow asynchronous orphan cleanup */ - obd_no_conn:1, /* deny new connections */ - obd_inactive:1, /* device active/inactive - * (for /proc/status only!!) */ - obd_no_ir:1, /* no imperative recovery. */ - obd_process_conf:1; /* device is processing mgs config */ + char obd_name[MAX_OBD_NAME]; + struct obd_uuid obd_uuid; + int obd_minor; + struct lu_device *obd_lu_dev; + + /* bitfield modification is protected by obd_dev_lock */ + unsigned long + obd_attached:1, /* finished attach */ + obd_set_up:1, /* finished setup */ + obd_recovering:1, /* there are recoverable clients */ + obd_abort_recovery:1, /* recovery expired */ + obd_version_recov:1, /* obd uses version checking */ + obd_replayable:1, /* recovery is enabled; + * inform clients */ + obd_no_transno:1, /* no committed-transno notification */ + obd_no_recov:1, /* fail instead of retry messages */ + obd_stopping:1, /* started cleanup */ + obd_starting:1, /* started setup */ + obd_force:1, /* cleanup with > 0 obd refcount */ + obd_fail:1, /* cleanup with failover */ + obd_async_recov:1, /* allow asynchronous orphan cleanup */ + obd_no_conn:1, /* deny new connections */ + obd_inactive:1, /* device active/inactive + * (for /proc/status only!!) */ + obd_no_ir:1, /* no imperative recovery. */ + obd_process_conf:1, /* device is processing mgs config */ + obd_uses_nid_stats:1; /* maintain per-client OBD stats */ + /* use separate field as it is set in interrupt to don't mess with * protection of other bits using _bh lock */ unsigned long obd_recovery_expired:1; @@ -881,7 +884,6 @@ struct obd_device { cfs_hash_t *obd_nid_stats_hash; cfs_list_t obd_nid_stats; cfs_atomic_t obd_refcount; - cfs_waitq_t obd_refcount_waitq; cfs_list_t obd_exports; cfs_list_t obd_unlinked_exports; cfs_list_t obd_delayed_exports; @@ -935,7 +937,6 @@ struct obd_device { cfs_list_t obd_req_replay_queue; cfs_list_t obd_lock_replay_queue; cfs_list_t obd_final_req_queue; - int obd_recovery_stage; union { #ifdef HAVE_SERVER_SUPPORT @@ -953,8 +954,8 @@ struct obd_device { unsigned int obd_cntr_base; struct lprocfs_stats *obd_stats; - unsigned int md_cntr_base; - struct lprocfs_stats *md_stats; + unsigned int obd_md_cntr_base; + struct lprocfs_stats *obd_md_stats; cfs_proc_dir_entry_t *obd_proc_entry; cfs_proc_dir_entry_t *obd_proc_exports_entry; @@ -1355,90 +1356,67 @@ struct md_open_data { struct lookup_intent; struct md_ops { - int (*m_getstatus)(struct obd_export *, struct lu_fid *, - struct obd_capa **); - int (*m_null_inode)(struct obd_export *, const struct lu_fid *); - int (*m_find_cbdata)(struct obd_export *, const struct lu_fid *, - ldlm_iterator_t, void *); - int (*m_close)(struct obd_export *, struct md_op_data *, - struct md_open_data *, struct ptlrpc_request **); - int (*m_create)(struct obd_export *, struct md_op_data *, - const void *, int, int, __u32, __u32, cfs_cap_t, - __u64, struct ptlrpc_request **); - int (*m_done_writing)(struct obd_export *, struct md_op_data *, - struct md_open_data *); - int (*m_enqueue)(struct obd_export *, struct ldlm_enqueue_info *, - struct lookup_intent *, struct md_op_data *, - struct lustre_handle *, void *, int, + /* Every operation from MD_STATS_FIRST_OP up to and including + * MD_STATS_LAST_OP will be counted by EXP_MD_OP_INCREMENT() + * and will appear in /proc/fs/lustre/{lmv,mdc}/.../md_stats. + * Operations after MD_STATS_LAST_OP are excluded from stats. + * There are a few reasons for doing this: we prune the 17 + * counters which will be of minimal use in understanding + * metadata utilization, we save memory by allocating 15 + * instead of 32 counters, we save cycles by not counting. + * + * MD_STATS_FIRST_OP must be the first member of md_ops. + */ +#define MD_STATS_FIRST_OP m_close + int (*m_close)(struct obd_export *, struct md_op_data *, + struct md_open_data *, struct ptlrpc_request **); + + int (*m_create)(struct obd_export *, struct md_op_data *, + const void *, int, int, __u32, __u32, cfs_cap_t, + __u64, struct ptlrpc_request **); + + int (*m_enqueue)(struct obd_export *, struct ldlm_enqueue_info *, + struct lookup_intent *, struct md_op_data *, + struct lustre_handle *, void *, int, struct ptlrpc_request **, __u64); - int (*m_getattr)(struct obd_export *, struct md_op_data *, - struct ptlrpc_request **); - int (*m_getattr_name)(struct obd_export *, struct md_op_data *, - struct ptlrpc_request **); - int (*m_intent_lock)(struct obd_export *, struct md_op_data *, - void *, int, struct lookup_intent *, int, - struct ptlrpc_request **, + + int (*m_getattr)(struct obd_export *, struct md_op_data *, + struct ptlrpc_request **); + + int (*m_intent_lock)(struct obd_export *, struct md_op_data *, + void *, int, struct lookup_intent *, int, + struct ptlrpc_request **, ldlm_blocking_callback, __u64); - int (*m_link)(struct obd_export *, struct md_op_data *, - struct ptlrpc_request **); - int (*m_rename)(struct obd_export *, struct md_op_data *, - const char *, int, const char *, int, - struct ptlrpc_request **); - int (*m_is_subdir)(struct obd_export *, const struct lu_fid *, - const struct lu_fid *, - struct ptlrpc_request **); - int (*m_setattr)(struct obd_export *, struct md_op_data *, void *, - int , void *, int, struct ptlrpc_request **, - struct md_open_data **mod); - int (*m_sync)(struct obd_export *, const struct lu_fid *, - struct obd_capa *, struct ptlrpc_request **); - int (*m_readpage)(struct obd_export *, struct md_op_data *, - struct page **, struct ptlrpc_request **); - - int (*m_unlink)(struct obd_export *, struct md_op_data *, - struct ptlrpc_request **); - - int (*m_setxattr)(struct obd_export *, const struct lu_fid *, - struct obd_capa *, obd_valid, const char *, - const char *, int, int, int, __u32, - struct ptlrpc_request **); - - int (*m_getxattr)(struct obd_export *, const struct lu_fid *, - struct obd_capa *, obd_valid, const char *, - const char *, int, int, int, - struct ptlrpc_request **); - - int (*m_init_ea_size)(struct obd_export *, int, int, int); - - int (*m_get_lustre_md)(struct obd_export *, struct ptlrpc_request *, - struct obd_export *, struct obd_export *, - struct lustre_md *); - - int (*m_free_lustre_md)(struct obd_export *, struct lustre_md *); - - int (*m_set_open_replay_data)(struct obd_export *, - struct obd_client_handle *, - struct ptlrpc_request *); - int (*m_clear_open_replay_data)(struct obd_export *, - struct obd_client_handle *); - int (*m_set_lock_data)(struct obd_export *, __u64 *, void *, __u64 *); - ldlm_mode_t (*m_lock_match)(struct obd_export *, __u64, - const struct lu_fid *, ldlm_type_t, - ldlm_policy_data_t *, ldlm_mode_t, - struct lustre_handle *); + int (*m_link)(struct obd_export *, struct md_op_data *, + struct ptlrpc_request **); - int (*m_cancel_unused)(struct obd_export *, const struct lu_fid *, - ldlm_policy_data_t *, ldlm_mode_t, - ldlm_cancel_flags_t flags, void *opaque); - int (*m_renew_capa)(struct obd_export *, struct obd_capa *oc, - renew_capa_cb_t cb); - int (*m_unpack_capa)(struct obd_export *, struct ptlrpc_request *, - const struct req_msg_field *, struct obd_capa **); + int (*m_rename)(struct obd_export *, struct md_op_data *, + const char *, int, const char *, int, + struct ptlrpc_request **); - int (*m_get_remote_perm)(struct obd_export *, const struct lu_fid *, - struct obd_capa *, __u32, - struct ptlrpc_request **); + int (*m_setattr)(struct obd_export *, struct md_op_data *, void *, + int , void *, int, struct ptlrpc_request **, + struct md_open_data **mod); + + int (*m_fsync)(struct obd_export *, const struct lu_fid *, + struct obd_capa *, struct ptlrpc_request **); + + int (*m_readpage)(struct obd_export *, struct md_op_data *, + struct page **, struct ptlrpc_request **); + + int (*m_unlink)(struct obd_export *, struct md_op_data *, + struct ptlrpc_request **); + + int (*m_setxattr)(struct obd_export *, const struct lu_fid *, + struct obd_capa *, obd_valid, const char *, + const char *, int, int, int, __u32, + struct ptlrpc_request **); + + int (*m_getxattr)(struct obd_export *, const struct lu_fid *, + struct obd_capa *, obd_valid, const char *, + const char *, int, int, int, + struct ptlrpc_request **); int (*m_intent_getattr_async)(struct obd_export *, struct md_enqueue_info *, @@ -1446,12 +1424,61 @@ struct md_ops { int (*m_revalidate_lock)(struct obd_export *, struct lookup_intent *, struct lu_fid *, __u64 *bits); +#define MD_STATS_LAST_OP m_revalidate_lock - /* - * NOTE: If adding ops, add another LPROCFS_MD_OP_INIT() line to - * lprocfs_alloc_md_stats() in obdclass/lprocfs_status.c. Also, add a - * wrapper function in include/linux/obd_class.h. - */ + int (*m_getstatus)(struct obd_export *, struct lu_fid *, + struct obd_capa **); + + int (*m_null_inode)(struct obd_export *, const struct lu_fid *); + + int (*m_find_cbdata)(struct obd_export *, const struct lu_fid *, + ldlm_iterator_t, void *); + + int (*m_done_writing)(struct obd_export *, struct md_op_data *, + struct md_open_data *); + + int (*m_getattr_name)(struct obd_export *, struct md_op_data *, + struct ptlrpc_request **); + + int (*m_is_subdir)(struct obd_export *, const struct lu_fid *, + const struct lu_fid *, + struct ptlrpc_request **); + + int (*m_init_ea_size)(struct obd_export *, int, int, int); + + int (*m_get_lustre_md)(struct obd_export *, struct ptlrpc_request *, + struct obd_export *, struct obd_export *, + struct lustre_md *); + + int (*m_free_lustre_md)(struct obd_export *, struct lustre_md *); + + int (*m_set_open_replay_data)(struct obd_export *, + struct obd_client_handle *, + struct ptlrpc_request *); + + int (*m_clear_open_replay_data)(struct obd_export *, + struct obd_client_handle *); + + int (*m_set_lock_data)(struct obd_export *, __u64 *, void *, __u64 *); + + ldlm_mode_t (*m_lock_match)(struct obd_export *, __u64, + const struct lu_fid *, ldlm_type_t, + ldlm_policy_data_t *, ldlm_mode_t, + struct lustre_handle *); + + int (*m_cancel_unused)(struct obd_export *, const struct lu_fid *, + ldlm_policy_data_t *, ldlm_mode_t, + ldlm_cancel_flags_t flags, void *opaque); + + int (*m_renew_capa)(struct obd_export *, struct obd_capa *oc, + renew_capa_cb_t cb); + + int (*m_unpack_capa)(struct obd_export *, struct ptlrpc_request *, + const struct req_msg_field *, struct obd_capa **); + + int (*m_get_remote_perm)(struct obd_export *, const struct lu_fid *, + struct obd_capa *, __u32, + struct ptlrpc_request **); }; struct lsm_operations { diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 5e91034..905825f 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -372,64 +372,57 @@ do { \ #ifdef LPROCFS -#define OBD_COUNTER_OFFSET(op) \ - ((offsetof(struct obd_ops, o_ ## op) - \ - offsetof(struct obd_ops, o_iocontrol)) \ - / sizeof(((struct obd_ops *)(0))->o_iocontrol)) - -#define OBD_COUNTER_INCREMENT(obdx, op) \ - if ((obdx)->obd_stats != NULL) { \ - unsigned int coffset; \ - coffset = (unsigned int)((obdx)->obd_cntr_base) + \ - OBD_COUNTER_OFFSET(op); \ - LASSERT(coffset < (obdx)->obd_stats->ls_num); \ - lprocfs_counter_incr((obdx)->obd_stats, coffset); \ - } - -#define EXP_COUNTER_INCREMENT(export, op) \ - if ((export)->exp_obd->obd_stats != NULL) { \ - unsigned int coffset; \ - coffset = (unsigned int)((export)->exp_obd->obd_cntr_base) + \ - OBD_COUNTER_OFFSET(op); \ - LASSERT(coffset < (export)->exp_obd->obd_stats->ls_num); \ - lprocfs_counter_incr((export)->exp_obd->obd_stats, coffset); \ - if ((export)->exp_nid_stats != NULL && \ - (export)->exp_nid_stats->nid_stats != NULL) \ - lprocfs_counter_incr( \ - (export)->exp_nid_stats->nid_stats, coffset);\ - } - -#define MD_COUNTER_OFFSET(op) \ - ((offsetof(struct md_ops, m_ ## op) - \ - offsetof(struct md_ops, m_getstatus)) \ - / sizeof(((struct md_ops *)(0))->m_getstatus)) - -#define MD_COUNTER_INCREMENT(obdx, op) \ - if ((obd)->md_stats != NULL) { \ - unsigned int coffset; \ - coffset = (unsigned int)((obdx)->md_cntr_base) + \ - MD_COUNTER_OFFSET(op); \ - LASSERT(coffset < (obdx)->md_stats->ls_num); \ - lprocfs_counter_incr((obdx)->md_stats, coffset); \ - } - -#define EXP_MD_COUNTER_INCREMENT(export, op) \ - if ((export)->exp_obd->obd_stats != NULL) { \ - unsigned int coffset; \ - coffset = (unsigned int)((export)->exp_obd->md_cntr_base) + \ - MD_COUNTER_OFFSET(op); \ - LASSERT(coffset < (export)->exp_obd->md_stats->ls_num); \ - lprocfs_counter_incr((export)->exp_obd->md_stats, coffset); \ - if ((export)->exp_md_stats != NULL) \ - lprocfs_counter_incr( \ - (export)->exp_md_stats, coffset); \ - } +#define OBD_COUNTER_OFFSET(op) \ + ((offsetof(struct obd_ops, o_ ## op) - \ + offsetof(struct obd_ops, o_iocontrol)) \ + / sizeof(((struct obd_ops *)NULL)->o_iocontrol)) + +/* The '- 1' below is for o_owner. */ +#define NUM_OBD_STATS \ + (sizeof(struct obd_ops) / \ + sizeof(((struct obd_ops *)NULL)->o_iocontrol) - 1) + +#define OBD_COUNTER_INCREMENT(obd, op) \ + lprocfs_counter_incr((obd)->obd_stats, \ + (obd)->obd_cntr_base + OBD_COUNTER_OFFSET(op)) + +#define EXP_COUNTER_INCREMENT(exp, op) \ + do { \ + unsigned int _off; \ + _off = (exp)->exp_obd->obd_cntr_base + OBD_COUNTER_OFFSET(op); \ + lprocfs_counter_incr((exp)->exp_obd->obd_stats, _off); \ + if ((exp)->exp_obd->obd_uses_nid_stats && \ + (exp)->exp_nid_stats != NULL) \ + lprocfs_counter_incr((exp)->exp_nid_stats->nid_stats, \ + _off); \ + } while (0) + +#define _MD_COUNTER_OFFSET(m_op) \ + ((offsetof(struct md_ops, m_op) - \ + offsetof(struct md_ops, MD_STATS_FIRST_OP)) / \ + sizeof(((struct md_ops *)NULL)->MD_STATS_FIRST_OP)) + +#define MD_COUNTER_OFFSET(op) _MD_COUNTER_OFFSET(m_ ## op) + +#define NUM_MD_STATS \ + (_MD_COUNTER_OFFSET(MD_STATS_LAST_OP) - \ + _MD_COUNTER_OFFSET(MD_STATS_FIRST_OP) + 1) + +/* Note that we only increment md counters for ops whose offset is less + * than NUM_MD_STATS. This is explained in a comment in the definition + * of struct md_ops. */ +#define EXP_MD_COUNTER_INCREMENT(exp, op) \ + do { \ + if (MD_COUNTER_OFFSET(op) < NUM_MD_STATS) \ + lprocfs_counter_incr((exp)->exp_obd->obd_md_stats, \ + (exp)->exp_obd->obd_md_cntr_base + \ + MD_COUNTER_OFFSET(op)); \ + } while (0) #else #define OBD_COUNTER_OFFSET(op) #define OBD_COUNTER_INCREMENT(obd, op) #define EXP_COUNTER_INCREMENT(exp, op) -#define MD_COUNTER_INCREMENT(obd, op) #define EXP_MD_COUNTER_INCREMENT(exp, op) #endif @@ -447,16 +440,6 @@ static inline int lprocfs_nid_ldlm_stats_init(struct nid_stat* tmp) tmp->nid_ldlm_stats); } -#define OBD_CHECK_MD_OP(obd, op, err) \ -do { \ - if (!OBT(obd) || !MDP((obd), op)) { \ - if (err) \ - CERROR("md_" #op ": dev %s/%d no operation\n", \ - obd->obd_name, obd->obd_minor); \ - RETURN(err); \ - } \ -} while (0) - #define EXP_CHECK_MD_OP(exp, op) \ do { \ if ((exp) == NULL) { \ @@ -2021,15 +2004,15 @@ static inline int md_setattr(struct obd_export *exp, struct md_op_data *op_data, RETURN(rc); } -static inline int md_sync(struct obd_export *exp, const struct lu_fid *fid, - struct obd_capa *oc, struct ptlrpc_request **request) +static inline int md_fsync(struct obd_export *exp, const struct lu_fid *fid, + struct obd_capa *oc, struct ptlrpc_request **request) { - int rc; - ENTRY; - EXP_CHECK_MD_OP(exp, sync); - EXP_MD_COUNTER_INCREMENT(exp, sync); - rc = MDP(exp->exp_obd, sync)(exp, fid, oc, request); - RETURN(rc); + int rc; + ENTRY; + EXP_CHECK_MD_OP(exp, fsync); + EXP_MD_COUNTER_INCREMENT(exp, fsync); + rc = MDP(exp->exp_obd, fsync)(exp, fid, oc, request); + RETURN(rc); } static inline int md_readpage(struct obd_export *exp, struct md_op_data *opdata, diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 42290a7..75ce1d8 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -2819,8 +2819,8 @@ int ll_fsync(struct file *file, struct dentry *dentry, int datasync) } oc = ll_mdscapa_get(inode); - err = md_sync(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), oc, - &req); + err = md_fsync(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), oc, + &req); capa_put(oc); if (!rc) rc = err; diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 61fe8bd..3877c76 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -1387,6 +1387,7 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg) lprocfs_lmv_init_vars(&lvars); lprocfs_obd_setup(obd, lvars.obd_vars); + lprocfs_alloc_md_stats(obd, 0); #ifdef LPROCFS { rc = lprocfs_seq_create(obd->obd_proc_entry, "target_obd", @@ -2105,25 +2106,25 @@ static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data, RETURN(rc); } -static int lmv_sync(struct obd_export *exp, const struct lu_fid *fid, - struct obd_capa *oc, struct ptlrpc_request **request) +static int lmv_fsync(struct obd_export *exp, const struct lu_fid *fid, + struct obd_capa *oc, struct ptlrpc_request **request) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + ENTRY; - rc = lmv_check_connect(obd); - if (rc) - RETURN(rc); + rc = lmv_check_connect(obd); + if (rc != 0) + RETURN(rc); - tgt = lmv_find_target(lmv, fid); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + tgt = lmv_find_target(lmv, fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); - rc = md_sync(tgt->ltd_exp, fid, oc, request); - RETURN(rc); + rc = md_fsync(tgt->ltd_exp, fid, oc, request); + RETURN(rc); } /* @@ -2371,6 +2372,7 @@ static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) case OBD_CLEANUP_EXPORTS: fld_client_proc_fini(&lmv->lmv_fld); lprocfs_obd_cleanup(obd); + lprocfs_free_md_stats(obd); break; default: break; @@ -2917,7 +2919,7 @@ struct md_ops lmv_md_ops = { .m_rename = lmv_rename, .m_setattr = lmv_setattr, .m_setxattr = lmv_setxattr, - .m_sync = lmv_sync, + .m_fsync = lmv_fsync, .m_readpage = lmv_readpage, .m_unlink = lmv_unlink, .m_init_ea_size = lmv_init_ea_size, diff --git a/lustre/lvfs/lvfs_lib.c b/lustre/lvfs/lvfs_lib.c index 3db6b25..7f5af3d 100644 --- a/lustre/lvfs/lvfs_lib.c +++ b/lustre/lvfs/lvfs_lib.c @@ -58,6 +58,9 @@ void lprocfs_counter_add(struct lprocfs_stats *stats, int idx, long amount) if (stats == NULL) return; + LASSERTF(0 <= idx && idx < stats->ls_num, + "idx %d, ls_num %hu\n", idx, stats->ls_num); + /* With per-client stats, statistics are allocated only for * single CPU area, so the smp_id should be 0 always. */ smp_id = lprocfs_stats_lock(stats, LPROCFS_GET_SMP_ID, &flags); @@ -106,6 +109,9 @@ void lprocfs_counter_sub(struct lprocfs_stats *stats, int idx, long amount) if (stats == NULL) return; + LASSERTF(0 <= idx && idx < stats->ls_num, + "idx %d, ls_num %hu\n", idx, stats->ls_num); + /* With per-client stats, statistics are allocated only for * single CPU area, so the smp_id should be 0 always. */ smp_id = lprocfs_stats_lock(stats, LPROCFS_GET_SMP_ID, &flags); diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index d6cc2ea..26b75ad 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -2306,8 +2306,8 @@ static int mdc_unpin(struct obd_export *exp, struct obd_client_handle *handle, RETURN(rc); } -int mdc_sync(struct obd_export *exp, const struct lu_fid *fid, - struct obd_capa *oc, struct ptlrpc_request **request) +int mdc_fsync(struct obd_export *exp, const struct lu_fid *fid, + struct obd_capa *oc, struct ptlrpc_request **request) { struct ptlrpc_request *req; int rc; @@ -2460,6 +2460,7 @@ static int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg) GOTO(err_close_lock, rc); lprocfs_mdc_init_vars(&lvars); lprocfs_obd_setup(obd, lvars.obd_vars); + lprocfs_alloc_md_stats(obd, 0); sptlrpc_lprocfs_cliobd_attach(obd); ptlrpc_lprocfs_register_obd(obd); @@ -2522,6 +2523,7 @@ static int mdc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) obd_cleanup_client_import(obd); ptlrpc_lprocfs_unregister_obd(obd); lprocfs_obd_cleanup(obd); + lprocfs_free_md_stats(obd); rc = obd_llog_finish(obd, 0); if (rc != 0) @@ -2736,7 +2738,7 @@ struct md_ops mdc_md_ops = { .m_setattr = mdc_setattr, .m_setxattr = mdc_setxattr, .m_getxattr = mdc_getxattr, - .m_sync = mdc_sync, + .m_fsync = mdc_fsync, .m_readpage = mdc_readpage, .m_unlink = mdc_unlink, .m_cancel_unused = mdc_cancel_unused, diff --git a/lustre/mdt/mdt_fs.c b/lustre/mdt/mdt_fs.c index acdd86f..c3b43b3 100644 --- a/lustre/mdt/mdt_fs.c +++ b/lustre/mdt/mdt_fs.c @@ -50,6 +50,8 @@ int mdt_export_stats_init(struct obd_device *obd, int rc, newnid; ENTRY; + LASSERT(!obd->obd_uses_nid_stats); + rc = lprocfs_exp_setup(exp, client_nid, &newnid); if (rc) { /* Mask error for already created @@ -60,15 +62,11 @@ int mdt_export_stats_init(struct obd_device *obd, } if (newnid) { struct nid_stat *tmp = exp->exp_nid_stats; - int num_stats; - num_stats = (sizeof(*obd->obd_type->typ_md_ops) / sizeof(void *)) + - LPROC_MDT_LAST; - tmp->nid_stats = lprocfs_alloc_stats(num_stats, - LPROCFS_STATS_FLAG_NOPERCPU); - if (tmp->nid_stats == NULL) - return -ENOMEM; - lprocfs_init_mps_stats(LPROC_MDT_LAST, tmp->nid_stats); + tmp->nid_stats = lprocfs_alloc_stats(LPROC_MDT_LAST, + LPROCFS_STATS_FLAG_NOPERCPU); + if (tmp->nid_stats == NULL) + return -ENOMEM; mdt_stats_counter_init(tmp->nid_stats); rc = lprocfs_register_stats(tmp->nid_proc, "stats", tmp->nid_stats); diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c index 90d0c85..0ae4fd0 100644 --- a/lustre/mdt/mdt_lproc.c +++ b/lustre/mdt/mdt_lproc.c @@ -239,7 +239,7 @@ int mdt_procfs_init(struct mdt_device *mdt, const char *name) rc = lprocfs_alloc_md_stats(obd, LPROC_MDT_LAST); if (rc) return rc; - mdt_stats_counter_init(obd->md_stats); + mdt_stats_counter_init(obd->obd_md_stats); rc = lprocfs_job_stats_init(obd, LPROC_MDT_LAST, mdt_stats_counter_init); @@ -1045,8 +1045,8 @@ void mdt_counter_incr(struct ptlrpc_request *req, int opcode) { struct obd_export *exp = req->rq_export; - if (exp->exp_obd && exp->exp_obd->md_stats) - lprocfs_counter_incr(exp->exp_obd->md_stats, opcode); + if (exp->exp_obd && exp->exp_obd->obd_md_stats) + lprocfs_counter_incr(exp->exp_obd->obd_md_stats, opcode); if (exp->exp_nid_stats && exp->exp_nid_stats->nid_stats != NULL) lprocfs_counter_incr(exp->exp_nid_stats->nid_stats, opcode); if (exp->exp_obd && exp->exp_obd->u.obt.obt_jobstats.ojs_hash && diff --git a/lustre/mgs/mgs_fs.c b/lustre/mgs/mgs_fs.c index 6d237c4..3f31626 100644 --- a/lustre/mgs/mgs_fs.c +++ b/lustre/mgs/mgs_fs.c @@ -47,45 +47,43 @@ #include "mgs_internal.h" int mgs_export_stats_init(struct obd_device *obd, struct obd_export *exp, - void *localdata) - + void *localdata) { - lnet_nid_t *client_nid = localdata; - int rc, newnid; - ENTRY; - - rc = lprocfs_exp_setup(exp, client_nid, &newnid); - if (rc) { - /* Mask error for already created - * /proc entries */ - if (rc == -EALREADY) - rc = 0; - RETURN(rc); - } - if (newnid) { - struct nid_stat *tmp = exp->exp_nid_stats; - int num_stats = 0; - - num_stats = (sizeof(*obd->obd_type->typ_dt_ops) / sizeof(void *)) + - LPROC_MGS_LAST - 1; - tmp->nid_stats = lprocfs_alloc_stats(num_stats, - LPROCFS_STATS_FLAG_NOPERCPU); - if (tmp->nid_stats == NULL) - return -ENOMEM; - lprocfs_init_ops_stats(LPROC_MGS_LAST, tmp->nid_stats); - mgs_stats_counter_init(tmp->nid_stats); - rc = lprocfs_register_stats(tmp->nid_proc, "stats", - tmp->nid_stats); - if (rc) - GOTO(clean, rc); - - rc = lprocfs_nid_ldlm_stats_init(tmp); - if (rc) - GOTO(clean, rc); + lnet_nid_t *client_nid = localdata; + struct nid_stat *tmp; + int rc, is_new_nid; + ENTRY; + + rc = lprocfs_exp_setup(exp, client_nid, &is_new_nid); + if (rc != 0) { + /* Mask error for already created /proc entries */ + if (rc == -EALREADY) + rc = 0; + GOTO(out, rc = 0); } - RETURN(0); -clean: - return rc; + + if (!is_new_nid) + GOTO(out, rc = 0); + + tmp = exp->exp_nid_stats; + tmp->nid_stats = lprocfs_alloc_stats(NUM_OBD_STATS + LPROC_MGS_LAST, + LPROCFS_STATS_FLAG_NOPERCPU); + if (tmp->nid_stats == NULL) + GOTO(out, rc = -ENOMEM); + + lprocfs_init_ops_stats(LPROC_MGS_LAST, tmp->nid_stats); + mgs_stats_counter_init(tmp->nid_stats); + rc = lprocfs_register_stats(tmp->nid_proc, "stats", tmp->nid_stats); + if (rc != 0) + GOTO(out, rc); + + rc = lprocfs_nid_ldlm_stats_init(tmp); + if (rc != 0) + GOTO(out, rc); + + RETURN(0); +out: + return rc; } /** diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index 89cb281..62e2714 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -1619,6 +1619,8 @@ void lprocfs_init_ops_stats(int num_private_stats, struct lprocfs_stats *stats) LPROCFS_OBD_OP_INIT(num_private_stats, stats, pool_del); LPROCFS_OBD_OP_INIT(num_private_stats, stats, getref); LPROCFS_OBD_OP_INIT(num_private_stats, stats, putref); + + CLASSERT(NUM_OBD_STATS == OBD_COUNTER_OFFSET(putref) + 1); } EXPORT_SYMBOL(lprocfs_init_ops_stats); @@ -1632,8 +1634,7 @@ int lprocfs_alloc_obd_stats(struct obd_device *obd, unsigned num_private_stats) LASSERT(obd->obd_proc_entry != NULL); LASSERT(obd->obd_cntr_base == 0); - num_stats = ((int)sizeof(*obd->obd_type->typ_dt_ops) / sizeof(void *)) + - num_private_stats - 1 /* o_owner */; + num_stats = NUM_OBD_STATS + num_private_stats; stats = lprocfs_alloc_stats(num_stats, 0); if (stats == NULL) return -ENOMEM; @@ -1668,12 +1669,18 @@ void lprocfs_free_obd_stats(struct obd_device *obd) } EXPORT_SYMBOL(lprocfs_free_obd_stats); -#define LPROCFS_MD_OP_INIT(base, stats, op) \ -do { \ - unsigned int coffset = base + MD_COUNTER_OFFSET(op); \ - LASSERT(coffset < stats->ls_num); \ - lprocfs_counter_init(stats, coffset, 0, #op, "reqs"); \ -} while (0) +/* Note that we only init md counters for ops whose offset is less + * than NUM_MD_STATS. This is explained in a comment in the definition + * of struct md_ops. */ +#define LPROCFS_MD_OP_INIT(base, stats, op) \ + do { \ + unsigned int _idx = base + MD_COUNTER_OFFSET(op); \ + \ + if (MD_COUNTER_OFFSET(op) < NUM_MD_STATS) { \ + LASSERT(_idx < stats->ls_num); \ + lprocfs_counter_init(stats, _idx, 0, #op, "reqs"); \ + } \ + } while (0) void lprocfs_init_mps_stats(int num_private_stats, struct lprocfs_stats *stats) { @@ -1691,7 +1698,7 @@ void lprocfs_init_mps_stats(int num_private_stats, struct lprocfs_stats *stats) LPROCFS_MD_OP_INIT(num_private_stats, stats, rename); LPROCFS_MD_OP_INIT(num_private_stats, stats, is_subdir); LPROCFS_MD_OP_INIT(num_private_stats, stats, setattr); - LPROCFS_MD_OP_INIT(num_private_stats, stats, sync); + LPROCFS_MD_OP_INIT(num_private_stats, stats, fsync); LPROCFS_MD_OP_INIT(num_private_stats, stats, readpage); LPROCFS_MD_OP_INIT(num_private_stats, stats, unlink); LPROCFS_MD_OP_INIT(num_private_stats, stats, setxattr); @@ -1713,52 +1720,65 @@ void lprocfs_init_mps_stats(int num_private_stats, struct lprocfs_stats *stats) EXPORT_SYMBOL(lprocfs_init_mps_stats); int lprocfs_alloc_md_stats(struct obd_device *obd, - unsigned num_private_stats) -{ - struct lprocfs_stats *stats; - unsigned int num_stats; - int rc, i; - - LASSERT(obd->md_stats == NULL); - LASSERT(obd->obd_proc_entry != NULL); - LASSERT(obd->md_cntr_base == 0); + unsigned int num_private_stats) +{ + struct lprocfs_stats *stats; + unsigned int num_stats; + int rc, i; + + CLASSERT(offsetof(struct md_ops, MD_STATS_FIRST_OP) == 0); + CLASSERT(_MD_COUNTER_OFFSET(MD_STATS_FIRST_OP) == 0); + CLASSERT(_MD_COUNTER_OFFSET(MD_STATS_LAST_OP) > 0); + + /* TODO Ensure that this function is only used where + * appropriate by adding an assertion to the effect that + * obd->obd_type->typ_md_ops is not NULL. We can't do this now + * because mdt_procfs_init() uses this function to allocate + * the stats backing /proc/fs/lustre/mdt/.../md_stats but the + * mdt layer does not use the md_ops interface. This is + * confusing and a waste of memory. See LU-2484. + */ + LASSERT(obd->obd_proc_entry != NULL); + LASSERT(obd->obd_md_stats == NULL); + LASSERT(obd->obd_md_cntr_base == 0); - num_stats = 1 + MD_COUNTER_OFFSET(revalidate_lock) + - num_private_stats; - stats = lprocfs_alloc_stats(num_stats, 0); - if (stats == NULL) - return -ENOMEM; + num_stats = NUM_MD_STATS + num_private_stats; + stats = lprocfs_alloc_stats(num_stats, 0); + if (stats == NULL) + return -ENOMEM; - lprocfs_init_mps_stats(num_private_stats, stats); + lprocfs_init_mps_stats(num_private_stats, stats); - for (i = num_private_stats; i < num_stats; i++) { + for (i = num_private_stats; i < num_stats; i++) { if (stats->ls_cnt_header[i].lc_name == NULL) { - CERROR("Missing md_stat initializer md_op " - "operation at offset %d. Aborting.\n", - i - num_private_stats); - LBUG(); - } - } - rc = lprocfs_register_stats(obd->obd_proc_entry, "md_stats", stats); - if (rc < 0) { - lprocfs_free_stats(&stats); - } else { - obd->md_stats = stats; - obd->md_cntr_base = num_private_stats; - } - return rc; + CERROR("Missing md_stat initializer md_op " + "operation at offset %d. Aborting.\n", + i - num_private_stats); + LBUG(); + } + } + + rc = lprocfs_register_stats(obd->obd_proc_entry, "md_stats", stats); + if (rc < 0) { + lprocfs_free_stats(&stats); + } else { + obd->obd_md_stats = stats; + obd->obd_md_cntr_base = num_private_stats; + } + + return rc; } EXPORT_SYMBOL(lprocfs_alloc_md_stats); void lprocfs_free_md_stats(struct obd_device *obd) { - struct lprocfs_stats *stats = obd->md_stats; + struct lprocfs_stats *stats = obd->obd_md_stats; - if (stats != NULL) { - obd->md_stats = NULL; - obd->md_cntr_base = 0; - lprocfs_free_stats(&stats); - } + if (stats != NULL) { + obd->obd_md_stats = NULL; + obd->obd_md_cntr_base = 0; + lprocfs_free_stats(&stats); + } } EXPORT_SYMBOL(lprocfs_free_md_stats); diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c index a9ce020..b7d5348 100644 --- a/lustre/ofd/ofd_dev.c +++ b/lustre/ofd/ofd_dev.c @@ -448,6 +448,8 @@ static int ofd_procfs_init(struct ofd_device *ofd) lprocfs_counter_init(obd->obd_stats, LPROC_OFD_WRITE_BYTES, LPROCFS_CNTR_AVGMINMAX, "write_bytes", "bytes"); + obd->obd_uses_nid_stats = 1; + entry = lprocfs_register("exports", obd->obd_proc_entry, NULL, NULL); if (IS_ERR(entry)) { rc = PTR_ERR(entry); diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index ab8c3f5..f4f8bcd 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -58,6 +58,8 @@ static int ofd_export_stats_init(struct ofd_device *ofd, ENTRY; + LASSERT(obd->obd_uses_nid_stats); + if (obd_uuid_equals(&exp->exp_client_uuid, &obd->obd_uuid)) /* Self-export gets no proc entry */ RETURN(0); @@ -77,9 +79,7 @@ static int ofd_export_stats_init(struct ofd_device *ofd, stats = exp->exp_nid_stats; LASSERT(stats != NULL); - num_stats = (sizeof(*obd->obd_type->typ_dt_ops) / sizeof(void *)) + - LPROC_OFD_LAST - 1; - + num_stats = NUM_OBD_STATS + LPROC_OFD_LAST; stats->nid_stats = lprocfs_alloc_stats(num_stats, LPROCFS_STATS_FLAG_NOPERCPU); if (stats->nid_stats == NULL)