From 962304a1dafb62ed040445559e6bf52241c60f2b Mon Sep 17 00:00:00 2001 From: bobijam Date: Thu, 28 May 2009 08:35:27 +0000 Subject: [PATCH] Branch HEAD b=18688 o=adilger i=nathan.rutman i=johann Description: Allow tuning service thread via /proc Details : For each service a new /proc/fs/lustre/{service}/*/thread_{min,max,started} entry is created that can be used to set min/max thread counts, and get the current number of running threads. --- lustre/ChangeLog | 10 ++++- lustre/include/lustre_net.h | 9 ++--- lustre/ptlrpc/lproc_ptlrpc.c | 95 +++++++++++++++++++++++++++++++++++++++++--- lustre/ptlrpc/recov_thread.c | 28 ++++++------- 4 files changed, 117 insertions(+), 25 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 491de4e..396de8c 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -13,6 +13,14 @@ tbd Sun Microsystems, Inc. removed cwd "./" (refer to Bugzilla 14399). * File join has been disabled in this release, refer to Bugzilla 16929. +Severity : enhancement +Bugzilla : 18688 +Description: Allow tuning service thread via /proc +Details : For each service a new + /proc/fs/lustre/{service}/*/thread_{min,max,started} entry is + created that can be used to set min/max thread counts, and get the + current number of running threads. + Severity : normal Bugzilla : 18382 Descriptoin: don't return error if have particaly created objects for file. @@ -35,7 +43,7 @@ Severity : enhancement Bugzilla : 19293 Description: move AT tunable parameters for more consistent usage Details : add AT tunables under /proc/sys/lustre, add to conf_param parsing - + Severity : enhancement Bugzilla : 17974 Description: add lazystatfs mount option to allow statfs(2) to skip down OSTs diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index f94116b..18d0df0 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -104,9 +104,8 @@ * considered full when less than ?_MAXREQSIZE is left in them. */ -#define LDLM_THREADS_AUTO_MIN \ - min((int)(num_online_cpus() * num_online_cpus() * 2), 8) -#define LDLM_THREADS_AUTO_MAX (LDLM_THREADS_AUTO_MIN * 16) +#define LDLM_THREADS_AUTO_MIN (2) +#define LDLM_THREADS_AUTO_MAX min(num_online_cpus()*num_online_cpus()*32, 128) #define LDLM_BL_THREADS LDLM_THREADS_AUTO_MIN #define LDLM_NBUFS (64 * num_online_cpus()) #define LDLM_BUFSIZE (8 * 1024) @@ -146,7 +145,7 @@ * except in the open case where there are a large number of OSTs in a LOV. */ #define MDS_MAXREQSIZE (5 * 1024) -#define MDS_MAXREPSIZE max(9 * 1024, 280 + LOV_MAX_STRIPE_COUNT * 56) +#define MDS_MAXREPSIZE max(9 * 1024, 362 + LOV_MAX_STRIPE_COUNT * 56) /* FLD_MAXREQSIZE == lustre_msg + __u32 padding + ptlrpc_body + opc + md_fld */ #define FLD_MAXREQSIZE (160) @@ -1291,7 +1290,7 @@ int ptlrpc_pinger_del_import(struct obd_import *imp); int ptlrpc_add_timeout_client(int time, enum timeout_event event, timeout_cb_t cb, void *data, struct list_head *obd_list); -int ptlrpc_del_timeout_client(struct list_head *obd_list, +int ptlrpc_del_timeout_client(struct list_head *obd_list, enum timeout_event event); struct ptlrpc_request * ptlrpc_prep_ping(struct obd_import *imp); int ptlrpc_obd_ping(struct obd_device *obd); diff --git a/lustre/ptlrpc/lproc_ptlrpc.c b/lustre/ptlrpc/lproc_ptlrpc.c index 4d02807..f1b9972 100644 --- a/lustre/ptlrpc/lproc_ptlrpc.c +++ b/lustre/ptlrpc/lproc_ptlrpc.c @@ -293,6 +293,81 @@ ptlrpc_lprocfs_write_req_history_max(struct file *file, const char *buffer, return count; } +static int +ptlrpc_lprocfs_rd_threads_min(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct ptlrpc_service *svc = data; + + return snprintf(page, count, "%d\n", svc->srv_threads_min); +} + +static int +ptlrpc_lprocfs_wr_threads_min(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct ptlrpc_service *svc = data; + int val; + int rc = lprocfs_write_helper(buffer, count, &val); + + if (rc < 0) + return rc; + + if (val < 2) + return -ERANGE; + + if (val > svc->srv_threads_max) + return -ERANGE; + + spin_lock(&svc->srv_lock); + svc->srv_threads_min = val; + spin_unlock(&svc->srv_lock); + + return count; +} + +static int +ptlrpc_lprocfs_rd_threads_started(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct ptlrpc_service *svc = data; + + return snprintf(page, count, "%d\n", svc->srv_threads_started); +} + +static int +ptlrpc_lprocfs_rd_threads_max(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct ptlrpc_service *svc = data; + + return snprintf(page, count, "%d\n", svc->srv_threads_max); +} + +static int +ptlrpc_lprocfs_wr_threads_max(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct ptlrpc_service *svc = data; + int val; + int rc = lprocfs_write_helper(buffer, count, &val); + + if (rc < 0) + return rc; + + if (val < 2) + return -ERANGE; + + if (val < svc->srv_threads_min) + return -ERANGE; + + spin_lock(&svc->srv_lock); + svc->srv_threads_max = val; + spin_unlock(&svc->srv_lock); + + return count; +} + struct ptlrpc_srh_iterator { __u64 srhi_seq; struct ptlrpc_request *srhi_req; @@ -545,21 +620,31 @@ void ptlrpc_lprocfs_register_service(struct proc_dir_entry *entry, struct ptlrpc_service *svc) { struct lprocfs_vars lproc_vars[] = { + {.name = "high_priority_ratio", + .read_fptr = ptlrpc_lprocfs_rd_hp_ratio, + .write_fptr = ptlrpc_lprocfs_wr_hp_ratio, + .data = svc}, {.name = "req_buffer_history_len", - .write_fptr = NULL, .read_fptr = ptlrpc_lprocfs_read_req_history_len, .data = svc}, {.name = "req_buffer_history_max", .write_fptr = ptlrpc_lprocfs_write_req_history_max, .read_fptr = ptlrpc_lprocfs_read_req_history_max, .data = svc}, + {.name = "threads_min", + .read_fptr = ptlrpc_lprocfs_rd_threads_min, + .write_fptr = ptlrpc_lprocfs_wr_threads_min, + .data = svc}, + {.name = "threads_max", + .read_fptr = ptlrpc_lprocfs_rd_threads_max, + .write_fptr = ptlrpc_lprocfs_wr_threads_max, + .data = svc}, + {.name = "threads_started", + .read_fptr = ptlrpc_lprocfs_rd_threads_started, + .data = svc}, {.name = "timeouts", .read_fptr = ptlrpc_lprocfs_rd_timeouts, .data = svc}, - {.name = "high_priority_ratio", - .read_fptr = ptlrpc_lprocfs_rd_hp_ratio, - .write_fptr = ptlrpc_lprocfs_wr_hp_ratio, - .data = svc}, {NULL} }; static struct file_operations req_history_fops = { diff --git a/lustre/ptlrpc/recov_thread.c b/lustre/ptlrpc/recov_thread.c index 6433cc7..adbaf03 100644 --- a/lustre/ptlrpc/recov_thread.c +++ b/lustre/ptlrpc/recov_thread.c @@ -76,8 +76,8 @@ enum { LLOG_LCM_FL_EXIT = 1 << 1 }; -static void llcd_print(struct llog_canceld_ctxt *llcd, - const char *func, int line) +static void llcd_print(struct llog_canceld_ctxt *llcd, + const char *func, int line) { CDEBUG(D_RPCTRACE, "Llcd (%p) at %s:%d:\n", llcd, func, line); CDEBUG(D_RPCTRACE, " size: %d\n", llcd->llcd_size); @@ -145,14 +145,14 @@ static void llcd_free(struct llog_canceld_ctxt *llcd) atomic_dec(&lcm->lcm_count); spin_unlock(&lcm->lcm_lock); - CDEBUG(D_RPCTRACE, "Free llcd %p on lcm %p (%d)\n", + CDEBUG(D_RPCTRACE, "Free llcd %p on lcm %p (%d)\n", llcd, lcm, atomic_read(&lcm->lcm_count)); } LASSERT(atomic_read(&llcd_count) > 0); atomic_dec(&llcd_count); - size = offsetof(struct llog_canceld_ctxt, llcd_cookies) + + size = offsetof(struct llog_canceld_ctxt, llcd_cookies) + llcd->llcd_size; OBD_SLAB_FREE(llcd, llcd_cache, size); } @@ -161,7 +161,7 @@ static void llcd_free(struct llog_canceld_ctxt *llcd) * Checks if passed cookie fits into llcd free space buffer. Returns * 1 if yes and 0 otherwise. */ -static inline int +static inline int llcd_fit(struct llog_canceld_ctxt *llcd, struct llog_cookie *cookies) { return (llcd->llcd_size - llcd->llcd_cookiebytes >= sizeof(*cookies)); @@ -170,11 +170,11 @@ llcd_fit(struct llog_canceld_ctxt *llcd, struct llog_cookie *cookies) /** * Copy passed @cookies to @llcd. */ -static inline void +static inline void llcd_copy(struct llog_canceld_ctxt *llcd, struct llog_cookie *cookies) { LASSERT(llcd_fit(llcd, cookies)); - memcpy((char *)llcd->llcd_cookies + llcd->llcd_cookiebytes, + memcpy((char *)llcd->llcd_cookies + llcd->llcd_cookiebytes, cookies, sizeof(*cookies)); llcd->llcd_cookiebytes += sizeof(*cookies); } @@ -321,7 +321,7 @@ static struct llog_canceld_ctxt *llcd_detach(struct llog_ctxt *ctxt) if (!llcd) return NULL; - CDEBUG(D_RPCTRACE, "Detach llcd %p from ctxt %p\n", + CDEBUG(D_RPCTRACE, "Detach llcd %p from ctxt %p\n", llcd, ctxt); ctxt->loc_llcd = NULL; @@ -432,7 +432,7 @@ void llog_recov_thread_stop(struct llog_commit_master *lcm, int force) struct llog_canceld_ctxt *llcd; struct list_head *tmp; - CERROR("Busy llcds found (%d) on lcm %p\n", + CERROR("Busy llcds found (%d) on lcm %p\n", atomic_read(&lcm->lcm_count) == 0, lcm); spin_lock(&lcm->lcm_lock); @@ -442,7 +442,7 @@ void llog_recov_thread_stop(struct llog_commit_master *lcm, int force) llcd_print(llcd, __FUNCTION__, __LINE__); } spin_unlock(&lcm->lcm_lock); - + /* * No point to go further with busy llcds at this point * as this is clear bug. It might mean we got hanging @@ -681,8 +681,8 @@ int llog_obd_repl_sync(struct llog_ctxt *ctxt, struct obd_export *exp) int rc = 0; ENTRY; - /* - * Flush any remaining llcd. + /* + * Flush any remaining llcd. */ mutex_down(&ctxt->loc_sem); if (exp && (ctxt->loc_imp == exp->exp_imp_reverse)) { @@ -694,10 +694,10 @@ int llog_obd_repl_sync(struct llog_ctxt *ctxt, struct obd_export *exp) llcd_put(ctxt); mutex_up(&ctxt->loc_sem); } else { - /* + /* * This is either llog_sync() from generic llog code or sync * on client disconnect. In either way let's do it and send - * llcds to the target with waiting for completion. + * llcds to the target with waiting for completion. */ CDEBUG(D_RPCTRACE, "Sync cached llcd\n"); mutex_up(&ctxt->loc_sem); -- 1.8.3.1