From 164231a7a29b7203cd27ab624eddd5fb7c557b90 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Wed, 13 Jun 2018 22:33:55 +0800 Subject: [PATCH] LU-11079 llite: control concurrent statahead instances It is found that if there are too many concurrent statahead instances, then related statahead RPCs may accumulate on the client import (for MDT) RPC lists (imp_sending_list/imp_delayed_list/imp_unreplied_lis), as to seriously affect the efficiency of spin_lock under the case of MDT overloaded or in recovery. Be as the temporarily solution, restrict the concurrent statahead instances. If want to support more concurrent statahead instances, please consider to decentralize the RPC lists attached on related import. Signed-off-by: Fan Yong Change-Id: I7251cc536f11d184f768e3d3704ba6717644541e Reviewed-on: https://review.whamcloud.com/32690 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Lai Siyao Reviewed-by: Oleg Drokin --- lustre/llite/llite_internal.h | 11 ++++++++++- lustre/llite/llite_lib.c | 1 + lustre/llite/lproc_llite.c | 37 +++++++++++++++++++++++++++++++++++++ lustre/llite/statahead.c | 23 ++++++++++++++++------- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 5c46e34..f0b839f 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -524,6 +524,8 @@ struct ll_sb_info { int ll_rw_stats_on; /* metadata stat-ahead */ + unsigned int ll_sa_running_max;/* max concurrent + * statahead instances */ unsigned int ll_sa_max; /* max statahead RPCs */ atomic_t ll_sa_total; /* statahead thread started * count */ @@ -1176,7 +1178,14 @@ void ll_ra_stats_inc(struct inode *inode, enum ra_stat which); #define LL_SA_RPC_MIN 2 #define LL_SA_RPC_DEF 32 -#define LL_SA_RPC_MAX 8192 +#define LL_SA_RPC_MAX 512 + +/* XXX: If want to support more concurrent statahead instances, + * please consider to decentralize the RPC lists attached + * on related import, such as imp_{sending,delayed}_list. + * LU-11079 */ +#define LL_SA_RUNNING_MAX 256 +#define LL_SA_RUNNING_DEF 16 #define LL_SA_CACHE_BIT 5 #define LL_SA_CACHE_SIZE (1 << LL_SA_CACHE_BIT) diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 5d06edc..90b1290 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -123,6 +123,7 @@ static struct ll_sb_info *ll_init_sbi(void) } /* metadata statahead is enabled by default */ + sbi->ll_sa_running_max = LL_SA_RUNNING_DEF; sbi->ll_sa_max = LL_SA_RPC_DEF; atomic_set(&sbi->ll_sa_total, 0); atomic_set(&sbi->ll_sa_wrong, 0); diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c index 91f003c..ee96bac 100644 --- a/lustre/llite/lproc_llite.c +++ b/lustre/llite/lproc_llite.c @@ -691,6 +691,42 @@ static ssize_t ll_track_gid_seq_write(struct file *file, } LPROC_SEQ_FOPS(ll_track_gid); +static ssize_t statahead_running_max_show(struct kobject *kobj, + struct attribute *attr, + char *buf) +{ + struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, + ll_kset.kobj); + + return snprintf(buf, 16, "%u\n", sbi->ll_sa_running_max); +} + +static ssize_t statahead_running_max_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t count) +{ + struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, + ll_kset.kobj); + unsigned long val; + int rc; + + rc = kstrtoul(buffer, 0, &val); + if (rc) + return rc; + + if (val <= LL_SA_RUNNING_MAX) { + sbi->ll_sa_running_max = val; + return count; + } + + CERROR("Bad statahead_running_max value %lu. Valid values " + "are in the range [0, %d]\n", val, LL_SA_RUNNING_MAX); + + return -ERANGE; +} +LUSTRE_RW_ATTR(statahead_running_max); + static int ll_statahead_max_seq_show(struct seq_file *m, void *v) { struct super_block *sb = m->private; @@ -1200,6 +1236,7 @@ struct lprocfs_vars lprocfs_llite_obd_vars[] = { #define MAX_STRING_SIZE 128 static struct attribute *llite_attrs[] = { + &lustre_attr_statahead_running_max.attr, NULL, }; diff --git a/lustre/llite/statahead.c b/lustre/llite/statahead.c index 1c98b8c..22203ce 100644 --- a/lustre/llite/statahead.c +++ b/lustre/llite/statahead.c @@ -1554,20 +1554,30 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry) struct ptlrpc_thread *thread; struct l_wait_info lwi = { 0 }; struct task_struct *task; - int rc; + struct ll_sb_info *sbi = ll_i2sbi(parent->d_inode); + int first = LS_FIRST_DE; + int rc = 0; ENTRY; /* I am the "lli_opendir_pid" owner, only me can set "lli_sai". */ - rc = is_first_dirent(dir, dentry); - if (rc == LS_NOT_FIRST_DE) + first = is_first_dirent(dir, dentry); + if (first == LS_NOT_FIRST_DE) /* It is not "ls -{a}l" operation, no need statahead for it. */ GOTO(out, rc = -EFAULT); + if (unlikely(atomic_inc_return(&sbi->ll_sa_running) > + sbi->ll_sa_running_max)) { + CDEBUG(D_READA, + "Too many concurrent statahead instances, " + "avoid new statahead instance temporarily.\n"); + GOTO(out, rc = -EMFILE); + } + sai = ll_sai_alloc(parent); if (sai == NULL) GOTO(out, rc = -ENOMEM); - sai->sai_ls_all = (rc == LS_FIRST_DOT_DE); + sai->sai_ls_all = (first == LS_FIRST_DOT_DE); /* if current lli_opendir_key was deauthorized, or dir re-opened by * another process, don't start statahead, otherwise the newly spawned @@ -1582,8 +1592,6 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry) lli->lli_sai = sai; spin_unlock(&lli->lli_sa_lock); - atomic_inc(&ll_i2sbi(parent->d_inode)->ll_sa_running); - CDEBUG(D_READA, "start statahead thread: [pid %d] [parent %.*s]\n", current_pid(), parent->d_name.len, parent->d_name.name); @@ -1594,7 +1602,6 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry) spin_lock(&lli->lli_sa_lock); lli->lli_sai = NULL; spin_unlock(&lli->lli_sa_lock); - atomic_dec(&ll_i2sbi(parent->d_inode)->ll_sa_running); rc = PTR_ERR(task); CERROR("can't start ll_sa thread, rc: %d\n", rc); GOTO(out, rc); @@ -1621,6 +1628,8 @@ out: if (sai != NULL) ll_sai_free(sai); + if (first != LS_NOT_FIRST_DE) + atomic_dec(&sbi->ll_sa_running); RETURN(rc); } -- 1.8.3.1