From: Patrick Farrell <pfarrell@whamcloud.com>
Date: Wed, 13 Apr 2022 15:31:15 +0000 (-0400)
Subject: EX-5014 pcc: Limit attach queue depth
X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=e5de5a703811f3a70b5e17a82acfc32ff45c7f51;p=fs%2Flustre-release.git

EX-5014 pcc: Limit attach queue depth

The existing async attach code does not attempt to limit
the number of async attaches that can be requested at once.
This is a problem because we could theoretically create too
many kthreads and overwhelm the system.

When the attach queue depth is exceeded, we stop allowing
new items to be queued by switching over to sync attach.

Ideally we would rebuild the attach code to generate a
queue of attach requests and have the attach thread code
pull items from the queue until it's exhausted, but that's
a much more substantial change and is left for later.

NB: This patch is incomplete - there's no way to adjust the
queue depth at runtime and there's no test for it.  Both
need to be added.

Signed-off-by: Patrick Farrell <pfarrell@whamcloud.com>
Change-Id: Ib00dfb67f5245a28b722278d031ee8cdf5e190d6
Reviewed-on: https://review.whamcloud.com/47061
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Yingjin Qian <qian@ddn.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
---

diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c
index 9c9e0e5..6ed118b 100644
--- a/lustre/llite/pcc.c
+++ b/lustre/llite/pcc.c
@@ -131,6 +131,8 @@ int pcc_super_init(struct pcc_super *super)
 	super->pccs_generation = 1;
 	super->pccs_async_threshold = PCC_DEFAULT_ASYNC_THRESHOLD;
 	super->pccs_mode = S_IRUSR;
+	atomic_set(&super->pccs_attaches_queued, 0);
+	super->pccs_maximum_queued_attaches = PCCS_DEFAULT_ATTACH_QUEUE_DEPTH;
 
 	return 0;
 }
@@ -1799,6 +1801,7 @@ static struct pcc_attach_context *
 pcc_attach_context_alloc(struct file *file, struct inode *inode, __u32 id)
 {
 	struct pcc_attach_context *pccx;
+	struct pcc_super *super = ll_i2pccs(inode);
 
 	OBD_ALLOC_PTR(pccx);
 	if (!pccx)
@@ -1807,12 +1810,16 @@ pcc_attach_context_alloc(struct file *file, struct inode *inode, __u32 id)
 	pccx->pccx_file = get_file(file);
 	pccx->pccx_inode = inode;
 	pccx->pccx_attach_id = id;
+	atomic_inc(&super->pccs_attaches_queued);
 
 	return pccx;
 }
 
 static inline void pcc_attach_context_free(struct pcc_attach_context *pccx)
 {
+	struct pcc_super *super = ll_i2pccs(pccx->pccx_inode);
+
+	atomic_dec(&super->pccs_attaches_queued);
 	LASSERT(pccx->pccx_file != NULL);
 	fput(pccx->pccx_file);
 	OBD_FREE_PTR(pccx);
@@ -1935,10 +1942,24 @@ static int pcc_readonly_attach_sync(struct file *file,
 static inline int pcc_do_readonly_attach(struct file *file,
 					 struct inode *inode, __u32 roid)
 {
+	struct pcc_super *super = ll_i2pccs(inode);
+	bool async = true;
 	int rc;
 
-	if (max_t(__u64, ll_i2info(inode)->lli_lazysize, i_size_read(inode)) >=
-	    ll_i2pccs(inode)->pccs_async_threshold) {
+	/* force sync if we're over the queueing limit, so this thread can't
+	 * contribute to the queue any more.  This lets us exceed the queue
+	 * depth by $NUMTHREADS, but that should be fine - deep queues are
+	 * OK, the main thing is to avoid unbounded numbers of kthreads.
+	 */
+	if (atomic_read(&super->pccs_attaches_queued) >=
+	    super->pccs_maximum_queued_attaches)
+		async = false;
+	/* if the file size is < the async threshold, don't do async */
+	if (max_t(__u64, ll_i2info(inode)->lli_lazysize, i_size_read(inode)) <
+	    super->pccs_async_threshold)
+		async = false;
+
+	if (async) {
 		rc = pcc_readonly_attach_async(file, inode, roid);
 		if (!rc || rc == -EINPROGRESS)
 			return rc;
diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h
index 8f9a45f..062abd0 100644
--- a/lustre/llite/pcc.h
+++ b/lustre/llite/pcc.h
@@ -155,6 +155,11 @@ struct pcc_dataset {
 
 #define PCC_DEFAULT_ASYNC_THRESHOLD	(256 << 20)
 
+/* after this many attaches are queued up, fall back to sync attach.  each
+ * attach creates a kthread, so we don't allow too many at once, but sync
+ * attach is very bad for applications, so we try to be generous.
+ */
+#define PCCS_DEFAULT_ATTACH_QUEUE_DEPTH	1024
 struct pcc_super {
 	/* Protect pccs_datasets */
 	struct rw_semaphore	 pccs_rw_sem;
@@ -172,6 +177,8 @@ struct pcc_super {
 	__u64			 pccs_async_threshold;
 	bool			 pccs_async_affinity;
 	umode_t			 pccs_mode;
+	atomic_t		 pccs_attaches_queued;
+	int			 pccs_maximum_queued_attaches;
 };
 
 struct pcc_inode {