LU-11079 llite: control concurrent statahead instances

It is found that if there are too many concurrent statahead
instances, then related statahead RPCs may accumulate on the
client import (for MDT) RPC lists
(imp_sending_list/imp_delayed_list/imp_unreplied_lis), as to
seriously affect the efficiency of spin_lock under the case
of MDT overloaded or in recovery. Be as the temporarily solution,
restrict the concurrent statahead instances.

If want to support more concurrent statahead instances, please
consider to decentralize the RPC lists attached on related import.

Lustre-change: https://review.whamcloud.com/32690
Lustre-commit: 164231a7a29b7203cd27ab624eddd5fb7c557b90

Signed-off-by: Fan Yong <fan.yong@intel.com>
Change-Id: I7251cc536f11d184f768e3d3704ba6717644541e
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Lai Siyao <lai.siyao@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/32909
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>

diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h
index 1e472ca..4acb7cd 100644 (file)
--- a/lustre/llite/llite_internal.h
+++ b/lustre/llite/llite_internal.h
@@ -522,6 +522,8 @@ struct ll_sb_info {
         int                       ll_rw_stats_on;
 
        /* metadata stat-ahead */
+       unsigned int              ll_sa_running_max;/* max concurrent
+                                                    * statahead instances */
        unsigned int              ll_sa_max;     /* max statahead RPCs */
        atomic_t                  ll_sa_total;   /* statahead thread started
                                                  * count */
@@ -1158,7 +1160,14 @@ void ll_ra_stats_inc(struct inode *inode, enum ra_stat which);
 
 #define LL_SA_RPC_MIN           2
 #define LL_SA_RPC_DEF           32
-#define LL_SA_RPC_MAX           8192
+#define LL_SA_RPC_MAX           512
+
+/* XXX: If want to support more concurrent statahead instances,
+ *     please consider to decentralize the RPC lists attached
+ *     on related import, such as imp_{sending,delayed}_list.
+ *     LU-11079 */
+#define LL_SA_RUNNING_MAX      256
+#define LL_SA_RUNNING_DEF      16
 
 #define LL_SA_CACHE_BIT         5
 #define LL_SA_CACHE_SIZE        (1 << LL_SA_CACHE_BIT)

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index a032b43..b1dbb7d 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -123,6 +123,7 @@ static struct ll_sb_info *ll_init_sbi(void)
         }
 
        /* metadata statahead is enabled by default */
+       sbi->ll_sa_running_max = LL_SA_RUNNING_DEF;
        sbi->ll_sa_max = LL_SA_RPC_DEF;
        atomic_set(&sbi->ll_sa_total, 0);
        atomic_set(&sbi->ll_sa_wrong, 0);

diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c
old mode 100644 (file)
new mode 100755 (executable)
index 2f61758..7dfb36d
--- a/lustre/llite/lproc_llite.c
+++ b/lustre/llite/lproc_llite.c
@@ -673,6 +673,40 @@ static ssize_t ll_track_gid_seq_write(struct file *file,
 }
 LPROC_SEQ_FOPS(ll_track_gid);
 
+static int ll_statahead_running_max_seq_show(struct seq_file *m, void *v)
+{
+       struct super_block *sb = m->private;
+       struct ll_sb_info *sbi = ll_s2sbi(sb);
+
+       seq_printf(m, "%u\n", sbi->ll_sa_running_max);
+       return 0;
+}
+
+static ssize_t ll_statahead_running_max_seq_write(struct file *file,
+                                          const char __user *buffer,
+                                          size_t count, loff_t *off)
+{
+       struct seq_file *m = file->private_data;
+       struct super_block *sb = m->private;
+       struct ll_sb_info *sbi = ll_s2sbi(sb);
+       int rc;
+       __s64 val;
+
+       rc = lprocfs_str_to_s64(buffer, count, &val);
+       if (rc)
+               return rc;
+
+       if (val >= 0 || val <= LL_SA_RUNNING_MAX)
+               sbi->ll_sa_running_max = val;
+       else
+               CERROR("%s: bad statahead_running_max value %lld. Valid values "
+                      "are in the range [0, %u]\n", ll_get_fsname(sb, NULL, 0),
+                      val, LL_SA_RUNNING_MAX);
+
+       return count;
+}
+LPROC_SEQ_FOPS(ll_statahead_running_max);
+
 static int ll_statahead_max_seq_show(struct seq_file *m, void *v)
 {
        struct super_block *sb = m->private;
@@ -687,7 +721,8 @@ static ssize_t ll_statahead_max_seq_write(struct file *file,
                                          size_t count, loff_t *off)
 {
        struct seq_file *m = file->private_data;
-       struct ll_sb_info *sbi = ll_s2sbi((struct super_block *)m->private);
+       struct super_block *sb = m->private;
+       struct ll_sb_info *sbi = ll_s2sbi(sb);
        int rc;
        __s64 val;
 
@@ -698,8 +733,9 @@ static ssize_t ll_statahead_max_seq_write(struct file *file,
        if (val >= 0 && val <= LL_SA_RPC_MAX)
                sbi->ll_sa_max = val;
        else
-               CERROR("Bad statahead_max value %lld. Valid values are in "
-                      "the range [0, %d]\n", val, LL_SA_RPC_MAX);
+               CERROR("%s: bad statahead_max value %lld. Valid values are in "
+                      "are in the range [0, %u]\n", ll_get_fsname(sb, NULL, 0),
+                      val, LL_SA_RPC_MAX);
 
        return count;
 }
@@ -1119,6 +1155,8 @@ struct lprocfs_vars lprocfs_llite_obd_vars[] = {
          .fops =       &ll_track_gid_fops                      },
        { .name =       "statahead_max",
          .fops =       &ll_statahead_max_fops                  },
+       { .name =       "statahead_running_max",
+         .fops =       &ll_statahead_running_max_fops          },
        { .name =       "statahead_agl",
          .fops =       &ll_statahead_agl_fops                  },
        { .name =       "statahead_stats",

diff --git a/lustre/llite/statahead.c b/lustre/llite/statahead.c
index 5d961c2..5b2af02 100644 (file)
--- a/lustre/llite/statahead.c
+++ b/lustre/llite/statahead.c
@@ -1550,20 +1550,30 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry)
        struct ptlrpc_thread *thread;
        struct l_wait_info lwi = { 0 };
        struct task_struct *task;
-       int rc;
+       struct ll_sb_info *sbi = ll_i2sbi(parent->d_inode);
+       int first = LS_FIRST_DE;
+       int rc = 0;
        ENTRY;
 
        /* I am the "lli_opendir_pid" owner, only me can set "lli_sai". */
-       rc = is_first_dirent(dir, dentry);
-       if (rc == LS_NOT_FIRST_DE)
+       first = is_first_dirent(dir, dentry);
+       if (first == LS_NOT_FIRST_DE)
                /* It is not "ls -{a}l" operation, no need statahead for it. */
                GOTO(out, rc = -EFAULT);
 
+       if (unlikely(atomic_inc_return(&sbi->ll_sa_running) >
+                                      sbi->ll_sa_running_max)) {
+               CDEBUG(D_READA,
+                      "Too many concurrent statahead instances, "
+                      "avoid new statahead instance temporarily.\n");
+               GOTO(out, rc = -EMFILE);
+       }
+
        sai = ll_sai_alloc(parent);
        if (sai == NULL)
                GOTO(out, rc = -ENOMEM);
 
-       sai->sai_ls_all = (rc == LS_FIRST_DOT_DE);
+       sai->sai_ls_all = (first == LS_FIRST_DOT_DE);
 
        /* if current lli_opendir_key was deauthorized, or dir re-opened by
         * another process, don't start statahead, otherwise the newly spawned
@@ -1578,8 +1588,6 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry)
        lli->lli_sai = sai;
        spin_unlock(&lli->lli_sa_lock);
 
-       atomic_inc(&ll_i2sbi(parent->d_inode)->ll_sa_running);
-
        CDEBUG(D_READA, "start statahead thread: [pid %d] [parent %.*s]\n",
               current_pid(), parent->d_name.len, parent->d_name.name);
 
@@ -1617,6 +1625,8 @@ out:
 
        if (sai != NULL)
                ll_sai_free(sai);
+       if (first != LS_NOT_FIRST_DE)
+               atomic_dec(&sbi->ll_sa_running);
 
        RETURN(rc);
 }