X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosc%2Fosc_page.c;h=9dae62a6aa22d86a45a99cd44a6a3f3fe4697674;hp=50d8ecd2de9c3d5c95871fa905c4ba8485641c00;hb=546993d587c5fc380e9745eae98f863e02e68575;hpb=c8c7598536daa2fb943b66681240436391217078

diff --git a/lustre/osc/osc_page.c b/lustre/osc/osc_page.c
index 50d8ecd..9dae62a 100644
--- a/lustre/osc/osc_page.c
+++ b/lustre/osc/osc_page.c
@@ -23,7 +23,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
  *
- * Copyright (c) 2011, 2016, Intel Corporation.
+ * Copyright (c) 2011, 2017, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
@@ -36,8 +36,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_OSC
+#include <lustre_osc.h>
 
-#include "osc_cl_internal.h"
+#include "osc_internal.h"
 
 static void osc_lru_del(struct client_obd *cli, struct osc_page *opg);
 static void osc_lru_use(struct client_obd *cli, struct osc_page *opg);
@@ -83,20 +84,17 @@ static void osc_page_transfer_add(const struct lu_env *env,
 {
 	struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj);
 
-	/* ops_lru and ops_inflight share the same field, so take it from LRU
-	 * first and then use it as inflight. */
 	osc_lru_use(osc_cli(obj), opg);
 }
 
-int osc_page_cache_add(const struct lu_env *env,
-			const struct cl_page_slice *slice, struct cl_io *io)
+int osc_page_cache_add(const struct lu_env *env, struct osc_page *opg,
+		       struct cl_io *io, cl_commit_cbt cb)
 {
-	struct osc_page *opg = cl2osc_page(slice);
 	int result;
 	ENTRY;
 
 	osc_page_transfer_get(opg, "transfer\0cache");
-	result = osc_queue_async_io(env, io, opg);
+	result = osc_queue_async_io(env, io, opg, cb);
 	if (result != 0)
 		osc_page_transfer_put(env, opg);
 	else
@@ -118,12 +116,12 @@ static const char *osc_list(struct list_head *head)
 	return list_empty(head) ? "-" : "+";
 }
 
-static inline cfs_time_t osc_submit_duration(struct osc_page *opg)
+static inline s64 osc_submit_duration(struct osc_page *opg)
 {
-        if (opg->ops_submit_time == 0)
-                return 0;
+	if (ktime_to_ns(opg->ops_submit_time) == 0)
+		return 0;
 
-        return (cfs_time_current() - opg->ops_submit_time);
+	return ktime_ms_delta(ktime_get(), opg->ops_submit_time);
 }
 
 static int osc_page_print(const struct lu_env *env,
@@ -138,7 +136,7 @@ static int osc_page_print(const struct lu_env *env,
 	return (*printer)(env, cookie, LUSTRE_OSC_NAME"-page@%p %lu: "
 			  "1< %#x %d %u %s %s > "
 			  "2< %lld %u %u %#x %#x | %p %p %p > "
-			  "3< %d %lu %d > "
+			  "3< %d %lld %d > "
 			  "4< %d %d %d %lu %s | %s %s %s %s > "
 			  "5< %s %s %s %s | %d %s | %d %s %s>\n",
 			  opg, osc_index(opg),
@@ -195,12 +193,17 @@ static void osc_page_delete(const struct lu_env *env,
 	osc_lru_del(osc_cli(obj), opg);
 
 	if (slice->cpl_page->cp_type == CPT_CACHEABLE) {
-		void *value;
+		void *value = NULL;
 
 		spin_lock(&obj->oo_tree_lock);
-		value = radix_tree_delete(&obj->oo_tree, osc_index(opg));
-		if (value != NULL)
-			--obj->oo_npages;
+		if (opg->ops_intree) {
+			value = radix_tree_delete(&obj->oo_tree,
+						  osc_index(opg));
+			if (value != NULL) {
+				--obj->oo_npages;
+				opg->ops_intree = 0;
+			}
+		}
 		spin_unlock(&obj->oo_tree_lock);
 
 		LASSERT(ergo(value != NULL, value == opg));
@@ -249,12 +252,22 @@ static int osc_page_flush(const struct lu_env *env,
 	RETURN(rc);
 }
 
+static void osc_page_touch(const struct lu_env *env,
+			  const struct cl_page_slice *slice, size_t to)
+{
+	struct osc_page *opg = cl2osc_page(slice);
+	struct cl_object *obj = opg->ops_cl.cpl_obj;
+
+	osc_page_touch_at(env, obj, osc_index(opg), to);
+}
+
 static const struct cl_page_operations osc_page_ops = {
 	.cpo_print         = osc_page_print,
 	.cpo_delete        = osc_page_delete,
 	.cpo_clip           = osc_page_clip,
 	.cpo_cancel         = osc_page_cancel,
-	.cpo_flush          = osc_page_flush
+	.cpo_flush          = osc_page_flush,
+	.cpo_page_touch	   = osc_page_touch,
 };
 
 int osc_page_init(const struct lu_env *env, struct cl_object *obj,
@@ -262,36 +275,47 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj,
 {
 	struct osc_object *osc = cl2osc(obj);
 	struct osc_page   *opg = cl_object_page_slice(obj, page);
+	struct osc_io *oio = osc_env_io(env);
 	int result;
 
 	opg->ops_from = 0;
 	opg->ops_to   = PAGE_SIZE;
 
+	INIT_LIST_HEAD(&opg->ops_lru);
+
 	result = osc_prep_async_page(osc, opg, page->cp_vmpage,
 				     cl_offset(obj, index));
-	if (result == 0) {
-		struct osc_io *oio = osc_env_io(env);
-		opg->ops_srvlock = osc_io_srvlock(oio);
-		cl_page_slice_add(page, &opg->ops_cl, obj, index,
-				  &osc_page_ops);
-	}
-	INIT_LIST_HEAD(&opg->ops_lru);
+	if (result != 0)
+		return result;
+
+	opg->ops_srvlock = osc_io_srvlock(oio);
+	cl_page_slice_add(page, &opg->ops_cl, obj, index,
+			  &osc_page_ops);
+
 
 	/* reserve an LRU space for this page */
-	if (page->cp_type == CPT_CACHEABLE && result == 0) {
+	if (page->cp_type == CPT_CACHEABLE) {
 		result = osc_lru_alloc(env, osc_cli(osc), opg);
 		if (result == 0) {
-			spin_lock(&osc->oo_tree_lock);
-			result = radix_tree_insert(&osc->oo_tree, index, opg);
-			if (result == 0)
-				++osc->oo_npages;
-			spin_unlock(&osc->oo_tree_lock);
-			LASSERT(result == 0);
+			result = radix_tree_preload(GFP_NOFS);
+			if (result == 0) {
+				spin_lock(&osc->oo_tree_lock);
+				result = radix_tree_insert(&osc->oo_tree,
+							   index, opg);
+				if (result == 0) {
+					++osc->oo_npages;
+					opg->ops_intree = 1;
+				}
+				spin_unlock(&osc->oo_tree_lock);
+
+				radix_tree_preload_end();
+			}
 		}
 	}
 
 	return result;
 }
+EXPORT_SYMBOL(osc_page_init);
 
 /**
  * Helper function called by osc_io_submit() for every page in an immediate
@@ -300,6 +324,7 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj,
 void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
 		     enum cl_req_type crt, int brw_flags)
 {
+	struct osc_io *oio = osc_env_io(env);
 	struct osc_async_page *oap = &opg->ops_oap;
 
 	LASSERTF(oap->oap_magic == OAP_MAGIC, "Bad oap magic: oap %p, "
@@ -312,12 +337,12 @@ void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
 	oap->oap_count     = opg->ops_to - opg->ops_from;
 	oap->oap_brw_flags = OBD_BRW_SYNC | brw_flags;
 
-	if (cfs_capable(CFS_CAP_SYS_RESOURCE)) {
+	if (oio->oi_cap_sys_resource) {
 		oap->oap_brw_flags |= OBD_BRW_NOQUOTA;
 		oap->oap_cmd |= OBD_BRW_NOQUOTA;
 	}
 
-	opg->ops_submit_time = cfs_time_current();
+	opg->ops_submit_time = ktime_get();
 	osc_page_transfer_get(opg, "transfer\0imm");
 	osc_page_transfer_add(env, opg, crt);
 }
@@ -416,7 +441,7 @@ int lru_queue_work(const struct lu_env *env, void *data)
 
 void osc_lru_add_batch(struct client_obd *cli, struct list_head *plist)
 {
-	struct list_head lru = LIST_HEAD_INIT(lru);
+	LIST_HEAD(lru);
 	struct osc_async_page *oap;
 	long npages = 0;
 
@@ -489,6 +514,8 @@ static void osc_lru_use(struct client_obd *cli, struct osc_page *opg)
 	/* If page is being transferred for the first time,
 	 * ops_lru should be empty */
 	if (opg->ops_in_lru) {
+		if (list_empty(&opg->ops_lru))
+			return;
 		spin_lock(&cli->cl_lru_list_lock);
 		if (!list_empty(&opg->ops_lru)) {
 			__osc_lru_del(cli, opg);
@@ -501,19 +528,22 @@ static void osc_lru_use(struct client_obd *cli, struct osc_page *opg)
 static void discard_pagevec(const struct lu_env *env, struct cl_io *io,
 				struct cl_page **pvec, int max_index)
 {
-        int i;
+	struct pagevec *pagevec = &osc_env_info(env)->oti_pagevec;
+	int i;
 
-        for (i = 0; i < max_index; i++) {
-                struct cl_page *page = pvec[i];
+	ll_pagevec_init(pagevec, 0);
+	for (i = 0; i < max_index; i++) {
+		struct cl_page *page = pvec[i];
 
 		LASSERT(cl_page_is_owned(page, io));
 		cl_page_delete(env, page);
 		cl_page_discard(env, io, page);
 		cl_page_disown(env, io, page);
-                cl_page_put(env, page);
+		cl_pagevec_put(env, page, pagevec);
 
-                pvec[i] = NULL;
-        }
+		pvec[i] = NULL;
+	}
+	pagevec_release(pagevec);
 }
 
 /**
@@ -573,7 +603,7 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	}
 
 	pvec = (struct cl_page **)osc_env_info(env)->oti_pvec;
-	io = &osc_env_info(env)->oti_io;
+	io = osc_env_thread_io(env);
 
 	spin_lock(&cli->cl_lru_list_lock);
 	if (force)
@@ -675,6 +705,7 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	}
 	RETURN(count > 0 ? count : rc);
 }
+EXPORT_SYMBOL(osc_lru_shrink);
 
 /**
  * Reclaim LRU pages by an IO thread. The caller wants to reclaim at least
@@ -767,6 +798,7 @@ static int osc_lru_alloc(const struct lu_env *env, struct client_obd *cli,
 	struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL);
 	struct osc_io *oio = osc_env_io(env);
 	int rc = 0;
+
 	ENTRY;
 
 	if (cli->cl_cache == NULL) /* shall not be in LRU */
@@ -865,17 +897,27 @@ void osc_lru_unreserve(struct client_obd *cli, unsigned long npages)
  * are likely from the same page zone.
  */
 static inline void unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
+					    struct osc_brw_async_args *aa,
 					    int factor)
 {
-	int page_count = desc->bd_iov_count;
+	int page_count;
 	void *zone = NULL;
 	int count = 0;
 	int i;
 
-	LASSERT(ptlrpc_is_bulk_desc_kiov(desc->bd_type));
+	if (desc != NULL) {
+		LASSERT(ptlrpc_is_bulk_desc_kiov(desc->bd_type));
+		page_count = desc->bd_iov_count;
+	} else {
+		page_count = aa->aa_page_count;
+	}
 
 	for (i = 0; i < page_count; i++) {
-		void *pz = page_zone(BD_GET_KIOV(desc, i).kiov_page);
+		void *pz;
+		if (desc)
+			pz = page_zone(BD_GET_KIOV(desc, i).kiov_page);
+		else
+			pz = page_zone(aa->aa_ppga[i]->pg);
 
 		if (likely(pz == zone)) {
 			++count;
@@ -894,14 +936,16 @@ static inline void unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
 		mod_zone_page_state(zone, NR_UNSTABLE_NFS, factor * count);
 }
 
-static inline void add_unstable_page_accounting(struct ptlrpc_bulk_desc *desc)
+static inline void add_unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
+						struct osc_brw_async_args *aa)
 {
-	unstable_page_accounting(desc, 1);
+	unstable_page_accounting(desc, aa, 1);
 }
 
-static inline void dec_unstable_page_accounting(struct ptlrpc_bulk_desc *desc)
+static inline void dec_unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
+						struct osc_brw_async_args *aa)
 {
-	unstable_page_accounting(desc, -1);
+	unstable_page_accounting(desc, aa, -1);
 }
 
 /**
@@ -918,12 +962,19 @@ static inline void dec_unstable_page_accounting(struct ptlrpc_bulk_desc *desc)
 void osc_dec_unstable_pages(struct ptlrpc_request *req)
 {
 	struct ptlrpc_bulk_desc *desc       = req->rq_bulk;
+	struct osc_brw_async_args *aa = (void *)&req->rq_async_args;
 	struct client_obd       *cli        = &req->rq_import->imp_obd->u.cli;
-	int			 page_count = desc->bd_iov_count;
+	int			 page_count;
 	long			 unstable_count;
 
+	if (desc)
+		page_count = desc->bd_iov_count;
+	else
+		page_count = aa->aa_page_count;
+
 	LASSERT(page_count >= 0);
-	dec_unstable_page_accounting(desc);
+
+	dec_unstable_page_accounting(desc, aa);
 
 	unstable_count = atomic_long_sub_return(page_count,
 						&cli->cl_unstable_count);
@@ -945,14 +996,20 @@ void osc_dec_unstable_pages(struct ptlrpc_request *req)
 void osc_inc_unstable_pages(struct ptlrpc_request *req)
 {
 	struct ptlrpc_bulk_desc *desc = req->rq_bulk;
+	struct osc_brw_async_args *aa = (void *)&req->rq_async_args;
 	struct client_obd       *cli  = &req->rq_import->imp_obd->u.cli;
-	long			 page_count = desc->bd_iov_count;
+	long			 page_count;
 
 	/* No unstable page tracking */
 	if (cli->cl_cache == NULL || !cli->cl_cache->ccc_unstable_check)
 		return;
 
-	add_unstable_page_accounting(desc);
+	if (desc)
+		page_count = desc->bd_iov_count;
+	else
+		page_count = aa->aa_page_count;
+
+	add_unstable_page_accounting(desc, aa);
 	atomic_long_add(page_count, &cli->cl_unstable_count);
 	atomic_long_add(page_count, &cli->cl_cache->ccc_unstable_nr);