static void osc_page_transfer_add(const struct lu_env *env,
struct osc_page *opg, enum cl_req_type crt)
{
- struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj);
+ struct osc_object *obj = osc_page_object(opg);
osc_lru_use(osc_cli(obj), opg);
}
-int osc_page_cache_add(const struct lu_env *env, struct osc_page *opg,
- struct cl_io *io, cl_commit_cbt cb)
+int osc_page_cache_add(const struct lu_env *env, struct osc_object *osc,
+ struct osc_page *opg, struct cl_io *io,
+ cl_commit_cbt cb)
{
int result;
ENTRY;
osc_page_transfer_get(opg, "transfer\0cache");
- result = osc_queue_async_io(env, io, opg, cb);
+ result = osc_queue_async_io(env, io, osc, opg, cb);
if (result != 0)
osc_page_transfer_put(env, opg);
else
const struct cl_object *obj, pgoff_t start, pgoff_t end)
{
memset(policy, 0, sizeof *policy);
- policy->l_extent.start = cl_offset(obj, start);
- policy->l_extent.end = cl_offset(obj, end + 1) - 1;
-}
-
-static inline s64 osc_submit_duration(struct osc_page *opg)
-{
- if (ktime_to_ns(opg->ops_submit_time) == 0)
- return 0;
-
- return ktime_ms_delta(ktime_get(), opg->ops_submit_time);
+ policy->l_extent.start = start << PAGE_SHIFT;
+ policy->l_extent.end = ((end + 1) << PAGE_SHIFT) - 1;
}
static int osc_page_print(const struct lu_env *env,
{
struct osc_page *opg = cl2osc_page(slice);
struct osc_async_page *oap = &opg->ops_oap;
- struct osc_object *obj = cl2osc(slice->cpl_obj);
+ struct osc_object *obj = osc_page_object(opg);
struct client_obd *cli = &osc_export(obj)->exp_obd->u.cli;
return (*printer)(env, cookie, LUSTRE_OSC_NAME"-page@%p %lu: "
- "1< %#x %d %c %c > "
- "2< %lld %u %u %#x %#x | %p %p %p > "
- "3< %d %lld %d > "
+ "1< %d %c %c > "
+ "2< %lld %u %u %#x %#x | %p %p > "
+ "3< %d %d > "
"4< %d %d %d %lu %c | %c %c %c %c > "
"5< %c %c %c %c | %d %c | %d %c %c>\n",
opg, osc_index(opg),
/* 1 */
- oap->oap_magic, oap->oap_cmd,
+ oap->oap_cmd,
list_empty_marker(&oap->oap_pending_item),
list_empty_marker(&oap->oap_rpc_item),
/* 2 */
oap->oap_obj_off, oap->oap_page_off, oap->oap_count,
oap->oap_async_flags, oap->oap_brw_flags,
- oap->oap_request, oap->oap_cli, obj,
+ cli, obj,
/* 3 */
opg->ops_transfer_pinned,
- osc_submit_duration(opg), opg->ops_srvlock,
+ opg->ops_srvlock,
/* 4 */
cli->cl_r_in_flight, cli->cl_w_in_flight,
cli->cl_max_rpcs_in_flight,
const struct cl_page_slice *slice)
{
struct osc_page *opg = cl2osc_page(slice);
- struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj);
+ struct osc_object *obj = osc_page_object(opg);
int rc;
ENTRY;
struct osc_page *opg = cl2osc_page(slice);
struct osc_async_page *oap = &opg->ops_oap;
+ CDEBUG(D_CACHE, "from %d, to %d\n", from, to);
+
opg->ops_from = from;
/* argument @to is exclusive, but @ops_to is inclusive */
opg->ops_to = to - 1;
- /* This isn't really necessary for transient pages, but we also don't
- * call clip on transient pages often, so it's OK.
- */
- spin_lock(&oap->oap_lock);
oap->oap_async_flags |= ASYNC_COUNT_STABLE;
- spin_unlock(&oap->oap_lock);
}
static int osc_page_flush(const struct lu_env *env,
const struct cl_page_slice *slice, size_t to)
{
struct osc_page *opg = cl2osc_page(slice);
- struct cl_object *obj = opg->ops_cl.cpl_obj;
+ struct cl_object *obj = osc2cl(osc_page_object(opg));
osc_page_touch_at(env, obj, osc_index(opg), to);
}
+static const struct cl_page_operations osc_transient_page_ops = {
+ .cpo_print = osc_page_print,
+ .cpo_delete = osc_page_delete,
+ .cpo_clip = osc_page_clip,
+};
+
static const struct cl_page_operations osc_page_ops = {
.cpo_print = osc_page_print,
.cpo_delete = osc_page_delete,
INIT_LIST_HEAD(&opg->ops_lru);
- result = osc_prep_async_page(osc, opg, cl_page, cl_offset(obj, index));
+ result = osc_prep_async_page(osc, opg, cl_page, index << PAGE_SHIFT);
if (result != 0)
return result;
opg->ops_srvlock = osc_io_srvlock(oio);
- cl_page_slice_add(cl_page, &opg->ops_cl, obj, &osc_page_ops);
- /* reserve an LRU space for this page */
- if (cl_page->cp_type == CPT_CACHEABLE) {
+ if (cl_page->cp_type == CPT_TRANSIENT) {
+ cl_page_slice_add(cl_page, &opg->ops_cl, obj,
+ &osc_transient_page_ops);
+ } else if (cl_page->cp_type == CPT_CACHEABLE) {
+ cl_page_slice_add(cl_page, &opg->ops_cl, obj, &osc_page_ops);
+ /* reserve an LRU space for this page */
result = osc_lru_alloc(env, osc_cli(osc), opg);
if (result == 0) {
result = radix_tree_preload(GFP_NOFS);
* transfer (i.e., transferred synchronously).
*/
void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
- enum cl_req_type crt, int brw_flags, ktime_t submit_time)
+ enum cl_req_type crt, int brw_flags)
{
struct osc_io *oio = osc_env_io(env);
struct osc_async_page *oap = &opg->ops_oap;
- LASSERTF(oap->oap_magic == OAP_MAGIC, "Bad oap magic: oap %p, "
- "magic 0x%x\n", oap, oap->oap_magic);
LASSERT(oap->oap_async_flags & ASYNC_READY);
LASSERT(oap->oap_async_flags & ASYNC_COUNT_STABLE);
oap->oap_count = opg->ops_to - opg->ops_from + 1;
oap->oap_brw_flags = OBD_BRW_SYNC | brw_flags;
- if (oio->oi_cap_sys_resource) {
+ if (oio->oi_cap_sys_resource)
oap->oap_brw_flags |= OBD_BRW_SYS_RESOURCE;
- oap->oap_cmd |= OBD_BRW_SYS_RESOURCE;
- }
- opg->ops_submit_time = submit_time;
osc_page_transfer_get(opg, "transfer\0imm");
osc_page_transfer_add(env, opg, crt);
}
unsigned long budget;
LASSERT(cache != NULL);
- budget = cache->ccc_lru_max / (atomic_read(&cache->ccc_users) - 2);
+ budget = cache->ccc_lru_max / (refcount_read(&cache->ccc_users) - 2);
/* if it's going to run out LRU slots, we should free some, but not
* too much to maintain faireness among OSCs. */
}
}
-static void discard_pagevec(const struct lu_env *env, struct cl_io *io,
- struct cl_page **pvec, int max_index)
+static void discard_cl_pages(const struct lu_env *env, struct cl_io *io,
+ struct cl_page **pvec, int max_index)
{
- struct pagevec *pagevec = &osc_env_info(env)->oti_pagevec;
+ struct folio_batch *fbatch = &osc_env_info(env)->oti_fbatch;
int i;
- ll_pagevec_init(pagevec, 0);
+ ll_folio_batch_init(fbatch, 0);
for (i = 0; i < max_index; i++) {
struct cl_page *page = pvec[i];
+ LASSERT(page->cp_type != CPT_TRANSIENT);
LASSERT(cl_page_is_owned(page, io));
- cl_page_delete(env, page);
cl_page_discard(env, io, page);
cl_page_disown(env, io, page);
- cl_pagevec_put(env, page, pagevec);
+ cl_batch_put(env, page, fbatch);
pvec[i] = NULL;
}
- pagevec_release(pagevec);
+ folio_batch_release(fbatch);
}
/**
spin_unlock(&cli->cl_lru_list_lock);
if (clobj != NULL) {
- discard_pagevec(env, io, pvec, index);
+ discard_cl_pages(env, io, pvec, index);
index = 0;
cl_io_fini(env, io);
cl_object_put(env, clobj);
clobj = NULL;
+ cond_resched();
}
clobj = tmp;
pvec[index++] = page;
if (unlikely(index == OTI_PVEC_SIZE)) {
spin_unlock(&cli->cl_lru_list_lock);
- discard_pagevec(env, io, pvec, index);
+ discard_cl_pages(env, io, pvec, index);
index = 0;
spin_lock(&cli->cl_lru_list_lock);
spin_unlock(&cli->cl_lru_list_lock);
if (clobj != NULL) {
- discard_pagevec(env, io, pvec, index);
+ discard_cl_pages(env, io, pvec, index);
cl_io_fini(env, io);
cl_object_put(env, clobj);
+ cond_resched();
}
atomic_dec(&cli->cl_lru_shrinkers);
cache->ccc_lru_shrinkers++;
list_move_tail(&cli->cl_lru_osc, &cache->ccc_lru);
- max_scans = atomic_read(&cache->ccc_users) - 2;
+ max_scans = refcount_read(&cache->ccc_users) - 2;
while (--max_scans > 0 &&
(scan = list_first_entry_or_null(&cache->ccc_lru,
struct client_obd,
#ifdef HAVE_NR_UNSTABLE_NFS
/* Old kernels use a separate counter for unstable pages,
* newer kernels treat them like any other writeback.
+ * (see Linux commit: v5.7-467-g8d92890bd6b8)
*/
-#define NR_WRITEBACK NR_UNSTABLE_NFS
+#define NR_ZONE_WRITE_PENDING ((enum zone_stat_item)NR_UNSTABLE_NFS)
+#elif !defined(HAVE_NR_ZONE_WRITE_PENDING)
+#define NR_ZONE_WRITE_PENDING ((enum zone_stat_item)NR_WRITEBACK)
#endif
static inline void unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
- struct osc_brw_async_args *aa,
int factor)
{
int page_count;
int count = 0;
int i;
- if (desc != NULL) {
- page_count = desc->bd_iov_count;
- } else {
- page_count = aa->aa_page_count;
- }
+ ENTRY;
+
+ page_count = desc->bd_iov_count;
+
+ CDEBUG(D_PAGE, "%s %d unstable pages\n",
+ factor == 1 ? "adding" : "removing", page_count);
for (i = 0; i < page_count; i++) {
- void *pz;
- if (desc)
- pz = page_zone(desc->bd_vec[i].bv_page);
- else
- pz = page_zone(aa->aa_ppga[i]->pg);
+ void *pz = page_zone(desc->bd_vec[i].bv_page);
if (likely(pz == zone)) {
++count;
}
if (count > 0) {
- mod_zone_page_state(zone, NR_WRITEBACK,
+ mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING,
factor * count);
count = 0;
}
++count;
}
if (count > 0)
- mod_zone_page_state(zone, NR_WRITEBACK, factor * count);
+ mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING,
+ factor * count);
+
+ EXIT;
}
-static inline void add_unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
- struct osc_brw_async_args *aa)
+static inline void add_unstable_pages(struct ptlrpc_bulk_desc *desc)
{
- unstable_page_accounting(desc, aa, 1);
+ unstable_page_accounting(desc, 1);
}
-static inline void dec_unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
- struct osc_brw_async_args *aa)
+static inline void dec_unstable_pages(struct ptlrpc_bulk_desc *desc)
{
- unstable_page_accounting(desc, aa, -1);
+ unstable_page_accounting(desc, -1);
}
/**
void osc_dec_unstable_pages(struct ptlrpc_request *req)
{
struct ptlrpc_bulk_desc *desc = req->rq_bulk;
- struct osc_brw_async_args *aa = (void *)&req->rq_async_args;
struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
int page_count;
long unstable_count;
- if (desc)
- page_count = desc->bd_iov_count;
- else
- page_count = aa->aa_page_count;
+ /* no desc means short io, which doesn't have separate unstable pages,
+ * it's just using space inside the RPC itself
+ */
+ if (!desc)
+ return;
+
+ page_count = desc->bd_iov_count;
LASSERT(page_count >= 0);
- dec_unstable_page_accounting(desc, aa);
+ dec_unstable_pages(desc);
unstable_count = atomic_long_sub_return(page_count,
&cli->cl_unstable_count);
unstable_count = atomic_long_sub_return(page_count,
&cli->cl_cache->ccc_unstable_nr);
LASSERT(unstable_count >= 0);
- if (unstable_count == 0)
- wake_up(&cli->cl_cache->ccc_unstable_waitq);
if (waitqueue_active(&osc_lru_waitq))
(void)ptlrpcd_queue_work(cli->cl_lru_work);
void osc_inc_unstable_pages(struct ptlrpc_request *req)
{
struct ptlrpc_bulk_desc *desc = req->rq_bulk;
- struct osc_brw_async_args *aa = (void *)&req->rq_async_args;
struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
long page_count;
if (cli->cl_cache == NULL || !cli->cl_cache->ccc_unstable_check)
return;
- if (desc)
- page_count = desc->bd_iov_count;
- else
- page_count = aa->aa_page_count;
+ /* no desc means short io, which doesn't have separate unstable pages,
+ * it's just using space inside the RPC itself
+ */
+ if (!desc)
+ return;
- add_unstable_page_accounting(desc, aa);
+ page_count = desc->bd_iov_count;
+
+ add_unstable_pages(desc);
atomic_long_add(page_count, &cli->cl_unstable_count);
atomic_long_add(page_count, &cli->cl_cache->ccc_unstable_nr);
struct client_obd *cli;
unsigned long cached = 0;
+ if (!osc_page_cache_shrink_enabled)
+ return 0;
+
spin_lock(&osc_shrink_lock);
list_for_each_entry(cli, &osc_shrink_list, cl_shrink_list)
cached += atomic_long_read(&cli->cl_lru_in_list);
git://git.whamcloud.com / fs / lustre-release.git / blobdiff