X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fldlm%2Fldlm_pool.c;h=5289a4e1ee764300f6526496c6bbac5b737bcdb5;hp=4acf35606071963b4fd2dc82c3e3d9bd8e85c0fa;hb=ff5b59d60a28fe22ef011ab92ff6b806b8e3c552;hpb=fbe4b504f24870fd800b103293f1cfc7192448b4 diff --git a/lustre/ldlm/ldlm_pool.c b/lustre/ldlm/ldlm_pool.c index 4acf356..5289a4e 100644 --- a/lustre/ldlm/ldlm_pool.c +++ b/lustre/ldlm/ldlm_pool.c @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -26,8 +24,10 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2010, 2012, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -130,7 +130,7 @@ * This controls the speed of reaching LDLM_POOL_MAX_GSP * with increasing thread period. */ -#define LDLM_POOL_GSP_STEP (4) +#define LDLM_POOL_GSP_STEP_SHIFT (2) /* * LDLM_POOL_GSP% of all locks is default GP. @@ -142,18 +142,18 @@ */ #define LDLM_POOL_MAX_AGE (36000) +/* + * The granularity of SLV calculation. + */ +#define LDLM_POOL_SLV_SHIFT (10) + #ifdef __KERNEL__ extern cfs_proc_dir_entry_t *ldlm_ns_proc_dir; #endif -#define avg(src, add) \ - ((src) = ((src) + (add)) / 2) - -static inline __u64 dru(__u64 val, __u32 div) +static inline __u64 dru(__u64 val, __u32 shift, int round_up) { - __u64 ret = val + (div - 1); - do_div(ret, div); - return ret; + return (val + (round_up ? (1 << shift) - 1 : 0)) >> shift; } static inline __u64 ldlm_pool_slv_max(__u32 L) @@ -162,7 +162,7 @@ static inline __u64 ldlm_pool_slv_max(__u32 L) * Allow to have all locks for 1 client for 10 hrs. * Formula is the following: limit * 10h / 1 client. */ - __u64 lim = L * LDLM_POOL_MAX_AGE / 1; + __u64 lim = (__u64)L * LDLM_POOL_MAX_AGE / 1; return lim; } @@ -196,18 +196,18 @@ static inline struct ldlm_namespace *ldlm_pl2ns(struct ldlm_pool *pl) * Calculates suggested grant_step in % of available locks for passed * \a period. This is later used in grant_plan calculations. */ -static inline int ldlm_pool_t2gsp(int t) +static inline int ldlm_pool_t2gsp(unsigned int t) { /* - * This yeilds 1% grant step for anything below LDLM_POOL_GSP_STEP + * This yields 1% grant step for anything below LDLM_POOL_GSP_STEP * and up to 30% for anything higher than LDLM_POOL_GSP_STEP. * * How this will affect execution is the following: * - * - for thread peroid 1s we will have grant_step 1% which good from + * - for thread period 1s we will have grant_step 1% which good from * pov of taking some load off from server and push it out to clients. * This is like that because 1% for grant_step means that server will - * not allow clients to get lots of locks inshort period of time and + * not allow clients to get lots of locks in short period of time and * keep all old locks in their caches. Clients will always have to * get some locks back if they want to take some new; * @@ -219,8 +219,8 @@ static inline int ldlm_pool_t2gsp(int t) * plan is reached. */ return LDLM_POOL_MAX_GSP - - (LDLM_POOL_MAX_GSP - LDLM_POOL_MIN_GSP) / - (1 << (t / LDLM_POOL_GSP_STEP)); + ((LDLM_POOL_MAX_GSP - LDLM_POOL_MIN_GSP) >> + (t >> LDLM_POOL_GSP_STEP_SHIFT)); } /** @@ -228,16 +228,19 @@ static inline int ldlm_pool_t2gsp(int t) * * \pre ->pl_lock is locked. */ -static inline void ldlm_pool_recalc_grant_plan(struct ldlm_pool *pl) +static void ldlm_pool_recalc_grant_plan(struct ldlm_pool *pl) { int granted, grant_step, limit; limit = ldlm_pool_get_limit(pl); - granted = atomic_read(&pl->pl_granted); + granted = cfs_atomic_read(&pl->pl_granted); grant_step = ldlm_pool_t2gsp(pl->pl_recalc_period); grant_step = ((limit - granted) * grant_step) / 100; pl->pl_grant_plan = granted + grant_step; + limit = (limit * 5) >> 2; + if (pl->pl_grant_plan > limit) + pl->pl_grant_plan = limit; } /** @@ -245,36 +248,36 @@ static inline void ldlm_pool_recalc_grant_plan(struct ldlm_pool *pl) * * \pre ->pl_lock is locked. */ -static inline void ldlm_pool_recalc_slv(struct ldlm_pool *pl) +static void ldlm_pool_recalc_slv(struct ldlm_pool *pl) { - int grant_usage, granted, grant_plan; - __u64 slv, slv_factor; + int granted; + int grant_plan; + int round_up; + __u64 slv; + __u64 slv_factor; + __u64 grant_usage; __u32 limit; slv = pl->pl_server_lock_volume; grant_plan = pl->pl_grant_plan; limit = ldlm_pool_get_limit(pl); - granted = atomic_read(&pl->pl_granted); + granted = cfs_atomic_read(&pl->pl_granted); + round_up = granted < limit; - grant_usage = limit - (granted - grant_plan); - if (grant_usage <= 0) - grant_usage = 1; + grant_usage = max_t(int, limit - (granted - grant_plan), 1); /* * Find out SLV change factor which is the ratio of grant usage * from limit. SLV changes as fast as the ratio of grant plan - * consumtion. The more locks from grant plan are not consumed + * consumption. The more locks from grant plan are not consumed * by clients in last interval (idle time), the faster grows * SLV. And the opposite, the more grant plan is over-consumed * (load time) the faster drops SLV. */ - slv_factor = (grant_usage * 100) / limit; - if (2 * abs(granted - limit) > limit) { - slv_factor *= slv_factor; - slv_factor = dru(slv_factor, 100); - } + slv_factor = (grant_usage << LDLM_POOL_SLV_SHIFT); + do_div(slv_factor, limit); slv = slv * slv_factor; - slv = dru(slv, 100); + slv = dru(slv, LDLM_POOL_SLV_SHIFT, round_up); if (slv > ldlm_pool_slv_max(limit)) { slv = ldlm_pool_slv_max(limit); @@ -290,13 +293,13 @@ static inline void ldlm_pool_recalc_slv(struct ldlm_pool *pl) * * \pre ->pl_lock is locked. */ -static inline void ldlm_pool_recalc_stats(struct ldlm_pool *pl) +static void ldlm_pool_recalc_stats(struct ldlm_pool *pl) { int grant_plan = pl->pl_grant_plan; __u64 slv = pl->pl_server_lock_volume; - int granted = atomic_read(&pl->pl_granted); - int grant_rate = atomic_read(&pl->pl_grant_rate); - int cancel_rate = atomic_read(&pl->pl_cancel_rate); + int granted = cfs_atomic_read(&pl->pl_granted); + int grant_rate = cfs_atomic_read(&pl->pl_grant_rate); + int cancel_rate = cfs_atomic_read(&pl->pl_cancel_rate); lprocfs_counter_add(pl->pl_stats, LDLM_POOL_SLV_STAT, slv); @@ -326,9 +329,9 @@ static void ldlm_srv_pool_push_slv(struct ldlm_pool *pl) */ obd = ldlm_pl2ns(pl)->ns_obd; LASSERT(obd != NULL); - write_lock(&obd->obd_pool_lock); + write_lock(&obd->obd_pool_lock); obd->obd_pool_slv = pl->pl_server_lock_volume; - write_unlock(&obd->obd_pool_lock); + write_unlock(&obd->obd_pool_lock); } /** @@ -341,37 +344,42 @@ static int ldlm_srv_pool_recalc(struct ldlm_pool *pl) time_t recalc_interval_sec; ENTRY; - spin_lock(&pl->pl_lock); recalc_interval_sec = cfs_time_current_sec() - pl->pl_recalc_time; - if (recalc_interval_sec >= pl->pl_recalc_period) { - /* - * Recalc SLV after last period. This should be done - * _before_ recalculating new grant plan. - */ - ldlm_pool_recalc_slv(pl); + if (recalc_interval_sec < pl->pl_recalc_period) + RETURN(0); - /* - * Make sure that pool informed obd of last SLV changes. - */ - ldlm_srv_pool_push_slv(pl); + spin_lock(&pl->pl_lock); + recalc_interval_sec = cfs_time_current_sec() - pl->pl_recalc_time; + if (recalc_interval_sec < pl->pl_recalc_period) { + spin_unlock(&pl->pl_lock); + RETURN(0); + } + /* + * Recalc SLV after last period. This should be done + * _before_ recalculating new grant plan. + */ + ldlm_pool_recalc_slv(pl); - /* - * Update grant_plan for new period. - */ - ldlm_pool_recalc_grant_plan(pl); + /* + * Make sure that pool informed obd of last SLV changes. + */ + ldlm_srv_pool_push_slv(pl); - pl->pl_recalc_time = cfs_time_current_sec(); - lprocfs_counter_add(pl->pl_stats, LDLM_POOL_TIMING_STAT, - recalc_interval_sec); - } + /* + * Update grant_plan for new period. + */ + ldlm_pool_recalc_grant_plan(pl); - spin_unlock(&pl->pl_lock); - RETURN(0); + pl->pl_recalc_time = cfs_time_current_sec(); + lprocfs_counter_add(pl->pl_stats, LDLM_POOL_TIMING_STAT, + recalc_interval_sec); + spin_unlock(&pl->pl_lock); + RETURN(0); } /** * This function is used on server side as main entry point for memory - * preasure handling. It decreases SLV on \a pl according to passed + * pressure handling. It decreases SLV on \a pl according to passed * \a nr and \a gfp_mask. * * Our goal here is to decrease SLV such a way that clients hold \a nr @@ -386,22 +394,22 @@ static int ldlm_srv_pool_shrink(struct ldlm_pool *pl, * VM is asking how many entries may be potentially freed. */ if (nr == 0) - return atomic_read(&pl->pl_granted); + return cfs_atomic_read(&pl->pl_granted); /* * Client already canceled locks but server is already in shrinker * and can't cancel anything. Let's catch this race. */ - if (atomic_read(&pl->pl_granted) == 0) + if (cfs_atomic_read(&pl->pl_granted) == 0) RETURN(0); - spin_lock(&pl->pl_lock); + spin_lock(&pl->pl_lock); /* - * We want shrinker to possibly cause cancelation of @nr locks from + * We want shrinker to possibly cause cancellation of @nr locks from * clients or grant approximately @nr locks smaller next intervals. * - * This is why we decresed SLV by @nr. This effect will only be as + * This is why we decreased SLV by @nr. This effect will only be as * long as one re-calc interval (1s these days) and this should be * enough to pass this decreased SLV to all clients. On next recalc * interval pool will either increase SLV if locks load is not high @@ -420,13 +428,13 @@ static int ldlm_srv_pool_shrink(struct ldlm_pool *pl, * Make sure that pool informed obd of last SLV changes. */ ldlm_srv_pool_push_slv(pl); - spin_unlock(&pl->pl_lock); + spin_unlock(&pl->pl_lock); - /* - * We did not really free any memory here so far, it only will be - * freed later may be, so that we return 0 to not confuse VM. - */ - return 0; + /* + * We did not really free any memory here so far, it only will be + * freed later may be, so that we return 0 to not confuse VM. + */ + return 0; } /** @@ -435,17 +443,16 @@ static int ldlm_srv_pool_shrink(struct ldlm_pool *pl, static int ldlm_srv_pool_setup(struct ldlm_pool *pl, int limit) { struct obd_device *obd; - ENTRY; obd = ldlm_pl2ns(pl)->ns_obd; LASSERT(obd != NULL && obd != LP_POISON); LASSERT(obd->obd_type != LP_POISON); - write_lock(&obd->obd_pool_lock); + write_lock(&obd->obd_pool_lock); obd->obd_pool_limit = limit; - write_unlock(&obd->obd_pool_lock); + write_unlock(&obd->obd_pool_lock); ldlm_pool_set_limit(pl, limit); - RETURN(0); + return 0; } /** @@ -456,32 +463,36 @@ static void ldlm_cli_pool_pop_slv(struct ldlm_pool *pl) struct obd_device *obd; /* - * Get new SLV and Limit from obd which is updated with comming + * Get new SLV and Limit from obd which is updated with coming * RPCs. */ obd = ldlm_pl2ns(pl)->ns_obd; LASSERT(obd != NULL); - read_lock(&obd->obd_pool_lock); + read_lock(&obd->obd_pool_lock); pl->pl_server_lock_volume = obd->obd_pool_slv; ldlm_pool_set_limit(pl, obd->obd_pool_limit); - read_unlock(&obd->obd_pool_lock); + read_unlock(&obd->obd_pool_lock); } /** - * Recalculates client sise pool \a pl according to current SLV and Limit. + * Recalculates client size pool \a pl according to current SLV and Limit. */ static int ldlm_cli_pool_recalc(struct ldlm_pool *pl) { time_t recalc_interval_sec; ENTRY; - spin_lock(&pl->pl_lock); - /* - * Check if we need to recalc lists now. - */ recalc_interval_sec = cfs_time_current_sec() - pl->pl_recalc_time; - if (recalc_interval_sec < pl->pl_recalc_period) { - spin_unlock(&pl->pl_lock); + if (recalc_interval_sec < pl->pl_recalc_period) + RETURN(0); + + spin_lock(&pl->pl_lock); + /* + * Check if we need to recalc lists now. + */ + recalc_interval_sec = cfs_time_current_sec() - pl->pl_recalc_time; + if (recalc_interval_sec < pl->pl_recalc_period) { + spin_unlock(&pl->pl_lock); RETURN(0); } @@ -493,7 +504,7 @@ static int ldlm_cli_pool_recalc(struct ldlm_pool *pl) pl->pl_recalc_time = cfs_time_current_sec(); lprocfs_counter_add(pl->pl_stats, LDLM_POOL_TIMING_STAT, recalc_interval_sec); - spin_unlock(&pl->pl_lock); + spin_unlock(&pl->pl_lock); /* * Do not cancel locks in case lru resize is disabled for this ns. @@ -507,14 +518,14 @@ static int ldlm_cli_pool_recalc(struct ldlm_pool *pl) * It may be called when SLV has changed much, this is why we do not * take into account pl->pl_recalc_time here. */ - RETURN(ldlm_cancel_lru(ldlm_pl2ns(pl), 0, LDLM_SYNC, - LDLM_CANCEL_LRUR)); + RETURN(ldlm_cancel_lru(ldlm_pl2ns(pl), 0, LCF_ASYNC, + LDLM_CANCEL_LRUR)); } /** - * This function is main entry point for memory preasure handling on client side. - * Main goal of this function is to cancel some number of locks on passed \a pl - * according to \a nr and \a gfp_mask. + * This function is main entry point for memory pressure handling on client + * side. Main goal of this function is to cancel some number of locks on + * passed \a pl according to \a nr and \a gfp_mask. */ static int ldlm_cli_pool_shrink(struct ldlm_pool *pl, int nr, unsigned int gfp_mask) @@ -535,17 +546,17 @@ static int ldlm_cli_pool_shrink(struct ldlm_pool *pl, */ ldlm_cli_pool_pop_slv(pl); - spin_lock(&ns->ns_unused_lock); - unused = ns->ns_nr_unused; - spin_unlock(&ns->ns_unused_lock); - + spin_lock(&ns->ns_lock); + unused = ns->ns_nr_unused; + spin_unlock(&ns->ns_lock); + if (nr) { - canceled = ldlm_cancel_lru(ns, nr, LDLM_SYNC, - LDLM_CANCEL_SHRINK); + canceled = ldlm_cancel_lru(ns, nr, LCF_ASYNC, + LDLM_CANCEL_SHRINK); } #ifdef __KERNEL__ /* - * Retrun the number of potentially reclaimable locks. + * Return the number of potentially reclaimable locks. */ return ((unused - canceled) / 100) * sysctl_vfs_cache_pressure; #else @@ -573,8 +584,11 @@ int ldlm_pool_recalc(struct ldlm_pool *pl) time_t recalc_interval_sec; int count; - spin_lock(&pl->pl_lock); recalc_interval_sec = cfs_time_current_sec() - pl->pl_recalc_time; + if (recalc_interval_sec <= 0) + goto recalc; + + spin_lock(&pl->pl_lock); if (recalc_interval_sec > 0) { /* * Update pool statistics every 1s. @@ -584,22 +598,22 @@ int ldlm_pool_recalc(struct ldlm_pool *pl) /* * Zero out all rates and speed for the last period. */ - atomic_set(&pl->pl_grant_rate, 0); - atomic_set(&pl->pl_cancel_rate, 0); - atomic_set(&pl->pl_grant_speed, 0); + cfs_atomic_set(&pl->pl_grant_rate, 0); + cfs_atomic_set(&pl->pl_cancel_rate, 0); } - spin_unlock(&pl->pl_lock); + spin_unlock(&pl->pl_lock); + recalc: if (pl->pl_ops->po_recalc != NULL) { count = pl->pl_ops->po_recalc(pl); lprocfs_counter_add(pl->pl_stats, LDLM_POOL_RECALC_STAT, count); - return count; } + recalc_interval_sec = pl->pl_recalc_time - cfs_time_current_sec() + + pl->pl_recalc_period; - return 0; + return recalc_interval_sec; } -EXPORT_SYMBOL(ldlm_pool_recalc); /** * Pool shrink wrapper. Will call either client or server pool recalc callback @@ -635,10 +649,9 @@ EXPORT_SYMBOL(ldlm_pool_shrink); */ int ldlm_pool_setup(struct ldlm_pool *pl, int limit) { - ENTRY; if (pl->pl_ops->po_setup != NULL) - RETURN(pl->pl_ops->po_setup(pl, limit)); - RETURN(0); + return(pl->pl_ops->po_setup(pl, limit)); + return 0; } EXPORT_SYMBOL(ldlm_pool_setup); @@ -652,18 +665,18 @@ static int lprocfs_rd_pool_state(char *page, char **start, off_t off, __u64 slv, clv; __u32 limit; - spin_lock(&pl->pl_lock); + spin_lock(&pl->pl_lock); slv = pl->pl_server_lock_volume; clv = pl->pl_client_lock_volume; limit = ldlm_pool_get_limit(pl); grant_plan = pl->pl_grant_plan; - granted = atomic_read(&pl->pl_granted); - grant_rate = atomic_read(&pl->pl_grant_rate); - lvf = atomic_read(&pl->pl_lock_volume_factor); - grant_speed = atomic_read(&pl->pl_grant_speed); - cancel_rate = atomic_read(&pl->pl_cancel_rate); + granted = cfs_atomic_read(&pl->pl_granted); + grant_rate = cfs_atomic_read(&pl->pl_grant_rate); + cancel_rate = cfs_atomic_read(&pl->pl_cancel_rate); + grant_speed = grant_rate - cancel_rate; + lvf = cfs_atomic_read(&pl->pl_lock_volume_factor); grant_step = ldlm_pool_t2gsp(pl->pl_recalc_period); - spin_unlock(&pl->pl_lock); + spin_unlock(&pl->pl_lock); nr += snprintf(page + nr, count - nr, "LDLM pool state (%s):\n", pl->pl_name); @@ -690,6 +703,20 @@ static int lprocfs_rd_pool_state(char *page, char **start, off_t off, return nr; } +static int lprocfs_rd_grant_speed(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct ldlm_pool *pl = data; + int grant_speed; + + spin_lock(&pl->pl_lock); + /* serialize with ldlm_pool_recalc */ + grant_speed = cfs_atomic_read(&pl->pl_grant_rate) - + cfs_atomic_read(&pl->pl_cancel_rate); + spin_unlock(&pl->pl_lock); + return lprocfs_rd_uint(page, start, off, count, eof, &grant_speed); +} + LDLM_POOL_PROC_READER(grant_plan, int); LDLM_POOL_PROC_READER(recalc_period, int); LDLM_POOL_PROC_WRITER(recalc_period, int); @@ -707,10 +734,11 @@ static int ldlm_pool_proc_init(struct ldlm_pool *pl) if (!var_name) RETURN(-ENOMEM); - parent_ns_proc = lprocfs_srch(ldlm_ns_proc_dir, ns->ns_name); + parent_ns_proc = lprocfs_srch(ldlm_ns_proc_dir, + ldlm_ns_name(ns)); if (parent_ns_proc == NULL) { CERROR("%s: proc entry is not initialized\n", - ns->ns_name); + ldlm_ns_name(ns)); GOTO(out_free_name, rc = -EINVAL); } pl->pl_proc_dir = lprocfs_register("pool", parent_ns_proc, @@ -742,8 +770,8 @@ static int ldlm_pool_proc_init(struct ldlm_pool *pl) lprocfs_add_vars(pl->pl_proc_dir, pool_vars, 0); snprintf(var_name, MAX_STRING_SIZE, "grant_speed"); - pool_vars[0].data = &pl->pl_grant_speed; - pool_vars[0].read_fptr = lprocfs_rd_atomic; + pool_vars[0].data = pl; + pool_vars[0].read_fptr = lprocfs_rd_grant_speed; lprocfs_add_vars(pl->pl_proc_dir, pool_vars, 0); snprintf(var_name, MAX_STRING_SIZE, "cancel_rate"); @@ -816,7 +844,7 @@ static int ldlm_pool_proc_init(struct ldlm_pool *pl) lprocfs_counter_init(pl->pl_stats, LDLM_POOL_TIMING_STAT, LPROCFS_CNTR_AVGMINMAX | LPROCFS_CNTR_STDDEV, "recalc_timing", "sec"); - lprocfs_register_stats(pl->pl_proc_dir, "stats", pl->pl_stats); + rc = lprocfs_register_stats(pl->pl_proc_dir, "stats", pl->pl_stats); EXIT; out_free_name: @@ -843,21 +871,20 @@ static void ldlm_pool_proc_fini(struct ldlm_pool *pl) int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns, int idx, ldlm_side_t client) { - int rc; - ENTRY; + int rc; + ENTRY; - spin_lock_init(&pl->pl_lock); - atomic_set(&pl->pl_granted, 0); + spin_lock_init(&pl->pl_lock); + cfs_atomic_set(&pl->pl_granted, 0); pl->pl_recalc_time = cfs_time_current_sec(); - atomic_set(&pl->pl_lock_volume_factor, 1); + cfs_atomic_set(&pl->pl_lock_volume_factor, 1); - atomic_set(&pl->pl_grant_rate, 0); - atomic_set(&pl->pl_cancel_rate, 0); - atomic_set(&pl->pl_grant_speed, 0); + cfs_atomic_set(&pl->pl_grant_rate, 0); + cfs_atomic_set(&pl->pl_cancel_rate, 0); pl->pl_grant_plan = LDLM_POOL_GP(LDLM_POOL_HOST_L); snprintf(pl->pl_name, sizeof(pl->pl_name), "ldlm-pool-%s-%d", - ns->ns_name, idx); + ldlm_ns_name(ns), idx); if (client == LDLM_NAMESPACE_SERVER) { pl->pl_ops = &ldlm_srv_pool_ops; @@ -866,7 +893,7 @@ int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns, pl->pl_server_lock_volume = ldlm_pool_slv_max(LDLM_POOL_HOST_L); } else { ldlm_pool_set_limit(pl, 1); - pl->pl_server_lock_volume = 1; + pl->pl_server_lock_volume = 0; pl->pl_ops = &ldlm_cli_pool_ops; pl->pl_recalc_period = LDLM_POOL_CLI_DEF_RECALC_PERIOD; } @@ -909,13 +936,9 @@ void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock) */ if (lock->l_resource->lr_type == LDLM_FLOCK) return; - ENTRY; - - LDLM_DEBUG(lock, "add lock to pool"); - atomic_inc(&pl->pl_granted); - atomic_inc(&pl->pl_grant_rate); - atomic_inc(&pl->pl_grant_speed); + cfs_atomic_inc(&pl->pl_granted); + cfs_atomic_inc(&pl->pl_grant_rate); lprocfs_counter_incr(pl->pl_stats, LDLM_POOL_GRANT_STAT); /* * Do not do pool recalc for client side as all locks which @@ -925,7 +948,6 @@ void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock) */ if (ns_is_server(ldlm_pl2ns(pl))) ldlm_pool_recalc(pl); - EXIT; } EXPORT_SYMBOL(ldlm_pool_add); @@ -939,19 +961,15 @@ void ldlm_pool_del(struct ldlm_pool *pl, struct ldlm_lock *lock) */ if (lock->l_resource->lr_type == LDLM_FLOCK) return; - ENTRY; - LDLM_DEBUG(lock, "del lock from pool"); - LASSERT(atomic_read(&pl->pl_granted) > 0); - atomic_dec(&pl->pl_granted); - atomic_inc(&pl->pl_cancel_rate); - atomic_dec(&pl->pl_grant_speed); + LASSERT(cfs_atomic_read(&pl->pl_granted) > 0); + cfs_atomic_dec(&pl->pl_granted); + cfs_atomic_inc(&pl->pl_cancel_rate); lprocfs_counter_incr(pl->pl_stats, LDLM_POOL_CANCEL_STAT); if (ns_is_server(ldlm_pl2ns(pl))) ldlm_pool_recalc(pl); - EXIT; } EXPORT_SYMBOL(ldlm_pool_del); @@ -962,11 +980,11 @@ EXPORT_SYMBOL(ldlm_pool_del); */ __u64 ldlm_pool_get_slv(struct ldlm_pool *pl) { - __u64 slv; - spin_lock(&pl->pl_lock); - slv = pl->pl_server_lock_volume; - spin_unlock(&pl->pl_lock); - return slv; + __u64 slv; + spin_lock(&pl->pl_lock); + slv = pl->pl_server_lock_volume; + spin_unlock(&pl->pl_lock); + return slv; } EXPORT_SYMBOL(ldlm_pool_get_slv); @@ -977,9 +995,9 @@ EXPORT_SYMBOL(ldlm_pool_get_slv); */ void ldlm_pool_set_slv(struct ldlm_pool *pl, __u64 slv) { - spin_lock(&pl->pl_lock); - pl->pl_server_lock_volume = slv; - spin_unlock(&pl->pl_lock); + spin_lock(&pl->pl_lock); + pl->pl_server_lock_volume = slv; + spin_unlock(&pl->pl_lock); } EXPORT_SYMBOL(ldlm_pool_set_slv); @@ -990,11 +1008,11 @@ EXPORT_SYMBOL(ldlm_pool_set_slv); */ __u64 ldlm_pool_get_clv(struct ldlm_pool *pl) { - __u64 slv; - spin_lock(&pl->pl_lock); - slv = pl->pl_client_lock_volume; - spin_unlock(&pl->pl_lock); - return slv; + __u64 slv; + spin_lock(&pl->pl_lock); + slv = pl->pl_client_lock_volume; + spin_unlock(&pl->pl_lock); + return slv; } EXPORT_SYMBOL(ldlm_pool_get_clv); @@ -1005,9 +1023,9 @@ EXPORT_SYMBOL(ldlm_pool_get_clv); */ void ldlm_pool_set_clv(struct ldlm_pool *pl, __u64 clv) { - spin_lock(&pl->pl_lock); - pl->pl_client_lock_volume = clv; - spin_unlock(&pl->pl_lock); + spin_lock(&pl->pl_lock); + pl->pl_client_lock_volume = clv; + spin_unlock(&pl->pl_lock); } EXPORT_SYMBOL(ldlm_pool_set_clv); @@ -1016,7 +1034,7 @@ EXPORT_SYMBOL(ldlm_pool_set_clv); */ __u32 ldlm_pool_get_limit(struct ldlm_pool *pl) { - return atomic_read(&pl->pl_limit); + return cfs_atomic_read(&pl->pl_limit); } EXPORT_SYMBOL(ldlm_pool_get_limit); @@ -1025,7 +1043,7 @@ EXPORT_SYMBOL(ldlm_pool_get_limit); */ void ldlm_pool_set_limit(struct ldlm_pool *pl, __u32 limit) { - atomic_set(&pl->pl_limit, limit); + cfs_atomic_set(&pl->pl_limit, limit); } EXPORT_SYMBOL(ldlm_pool_set_limit); @@ -1034,19 +1052,19 @@ EXPORT_SYMBOL(ldlm_pool_set_limit); */ __u32 ldlm_pool_get_lvf(struct ldlm_pool *pl) { - return atomic_read(&pl->pl_lock_volume_factor); + return cfs_atomic_read(&pl->pl_lock_volume_factor); } EXPORT_SYMBOL(ldlm_pool_get_lvf); #ifdef __KERNEL__ static int ldlm_pool_granted(struct ldlm_pool *pl) { - return atomic_read(&pl->pl_granted); + return cfs_atomic_read(&pl->pl_granted); } static struct ptlrpc_thread *ldlm_pools_thread; -static struct shrinker *ldlm_pools_srv_shrinker; -static struct shrinker *ldlm_pools_cli_shrinker; +static struct cfs_shrinker *ldlm_pools_srv_shrinker; +static struct cfs_shrinker *ldlm_pools_cli_shrinker; static struct completion ldlm_pools_comp; /* @@ -1059,9 +1077,11 @@ static int ldlm_pools_shrink(ldlm_side_t client, int nr, { int total = 0, cached = 0, nr_ns; struct ldlm_namespace *ns; + struct ldlm_namespace *ns_old = NULL; /* loop detection */ void *cookie; - if (nr != 0 && !(gfp_mask & __GFP_FS)) + if (client == LDLM_NAMESPACE_CLIENT && nr != 0 && + !(gfp_mask & __GFP_FS)) return -1; CDEBUG(D_DLMTRACE, "Request to shrink %d %s locks from all pools\n", @@ -1072,21 +1092,36 @@ static int ldlm_pools_shrink(ldlm_side_t client, int nr, /* * Find out how many resources we may release. */ - for (nr_ns = atomic_read(ldlm_namespace_nr(client)); - nr_ns > 0; nr_ns--) + for (nr_ns = ldlm_namespace_nr_read(client); + nr_ns > 0; nr_ns--) { - mutex_down(ldlm_namespace_lock(client)); - if (list_empty(ldlm_namespace_list(client))) { - mutex_up(ldlm_namespace_lock(client)); + mutex_lock(ldlm_namespace_lock(client)); + if (cfs_list_empty(ldlm_namespace_list(client))) { + mutex_unlock(ldlm_namespace_lock(client)); cl_env_reexit(cookie); return 0; } ns = ldlm_namespace_first_locked(client); + + if (ns == ns_old) { + mutex_unlock(ldlm_namespace_lock(client)); + break; + } + + if (ldlm_ns_empty(ns)) { + ldlm_namespace_move_to_inactive_locked(ns, client); + mutex_unlock(ldlm_namespace_lock(client)); + continue; + } + + if (ns_old == NULL) + ns_old = ns; + ldlm_namespace_get(ns); - ldlm_namespace_move_locked(ns, client); - mutex_up(ldlm_namespace_lock(client)); + ldlm_namespace_move_to_active_locked(ns, client); + mutex_unlock(ldlm_namespace_lock(client)); total += ldlm_pool_shrink(&ns->ns_pool, 0, gfp_mask); - ldlm_namespace_put(ns, 1); + ldlm_namespace_put(ns); } if (nr == 0 || total == 0) { @@ -1097,17 +1132,17 @@ static int ldlm_pools_shrink(ldlm_side_t client, int nr, /* * Shrink at least ldlm_namespace_nr(client) namespaces. */ - for (nr_ns = atomic_read(ldlm_namespace_nr(client)); - nr_ns > 0; nr_ns--) + for (nr_ns = ldlm_namespace_nr_read(client) - nr_ns; + nr_ns > 0; nr_ns--) { int cancel, nr_locks; /* * Do not call shrink under ldlm_namespace_lock(client) */ - mutex_down(ldlm_namespace_lock(client)); - if (list_empty(ldlm_namespace_list(client))) { - mutex_up(ldlm_namespace_lock(client)); + mutex_lock(ldlm_namespace_lock(client)); + if (cfs_list_empty(ldlm_namespace_list(client))) { + mutex_unlock(ldlm_namespace_lock(client)); /* * If list is empty, we can't return any @cached > 0, * that probably would cause needless shrinker @@ -1118,34 +1153,42 @@ static int ldlm_pools_shrink(ldlm_side_t client, int nr, } ns = ldlm_namespace_first_locked(client); ldlm_namespace_get(ns); - ldlm_namespace_move_locked(ns, client); - mutex_up(ldlm_namespace_lock(client)); + ldlm_namespace_move_to_active_locked(ns, client); + mutex_unlock(ldlm_namespace_lock(client)); nr_locks = ldlm_pool_granted(&ns->ns_pool); cancel = 1 + nr_locks * nr / total; ldlm_pool_shrink(&ns->ns_pool, cancel, gfp_mask); cached += ldlm_pool_granted(&ns->ns_pool); - ldlm_namespace_put(ns, 1); + ldlm_namespace_put(ns); } cl_env_reexit(cookie); - return cached; + /* we only decrease the SLV in server pools shrinker, return -1 to + * kernel to avoid needless loop. LU-1128 */ + return (client == LDLM_NAMESPACE_SERVER) ? -1 : cached; } -static int ldlm_pools_srv_shrink(int nr, unsigned int gfp_mask) +static int ldlm_pools_srv_shrink(SHRINKER_ARGS(sc, nr_to_scan, gfp_mask)) { - return ldlm_pools_shrink(LDLM_NAMESPACE_SERVER, nr, gfp_mask); + return ldlm_pools_shrink(LDLM_NAMESPACE_SERVER, + shrink_param(sc, nr_to_scan), + shrink_param(sc, gfp_mask)); } -static int ldlm_pools_cli_shrink(int nr, unsigned int gfp_mask) +static int ldlm_pools_cli_shrink(SHRINKER_ARGS(sc, nr_to_scan, gfp_mask)) { - return ldlm_pools_shrink(LDLM_NAMESPACE_CLIENT, nr, gfp_mask); + return ldlm_pools_shrink(LDLM_NAMESPACE_CLIENT, + shrink_param(sc, nr_to_scan), + shrink_param(sc, gfp_mask)); } -void ldlm_pools_recalc(ldlm_side_t client) +int ldlm_pools_recalc(ldlm_side_t client) { __u32 nr_l = 0, nr_p = 0, l; struct ldlm_namespace *ns; + struct ldlm_namespace *ns_old = NULL; int nr, equal = 0; + int time = 50; /* seconds of sleep if no active namespaces */ /* * No need to setup pool limit for client pools. @@ -1154,9 +1197,9 @@ void ldlm_pools_recalc(ldlm_side_t client) /* * Check all modest namespaces first. */ - mutex_down(ldlm_namespace_lock(client)); - list_for_each_entry(ns, ldlm_namespace_list(client), - ns_list_chain) + mutex_lock(ldlm_namespace_lock(client)); + cfs_list_for_each_entry(ns, ldlm_namespace_list(client), + ns_list_chain) { if (ns->ns_appetite != LDLM_NAMESPACE_MODEST) continue; @@ -1167,9 +1210,9 @@ void ldlm_pools_recalc(ldlm_side_t client) /* * Set the modest pools limit equal to their avg granted - * locks + 5%. + * locks + ~6%. */ - l += dru(l * LDLM_POOLS_MODEST_MARGIN, 100); + l += dru(l, LDLM_POOLS_MODEST_MARGIN_SHIFT, 0); ldlm_pool_setup(&ns->ns_pool, l); nr_l += l; nr_p++; @@ -1190,8 +1233,8 @@ void ldlm_pools_recalc(ldlm_side_t client) /* * The rest is given to greedy namespaces. */ - list_for_each_entry(ns, ldlm_namespace_list(client), - ns_list_chain) + cfs_list_for_each_entry(ns, ldlm_namespace_list(client), + ns_list_chain) { if (!equal && ns->ns_appetite != LDLM_NAMESPACE_GREEDY) continue; @@ -1203,25 +1246,25 @@ void ldlm_pools_recalc(ldlm_side_t client) * for _all_ pools. */ l = LDLM_POOL_HOST_L / - atomic_read(ldlm_namespace_nr(client)); + ldlm_namespace_nr_read(client); } else { /* * All the rest of greedy pools will have * all locks in equal parts. */ l = (LDLM_POOL_HOST_L - nr_l) / - (atomic_read(ldlm_namespace_nr(client)) - + (ldlm_namespace_nr_read(client) - nr_p); } ldlm_pool_setup(&ns->ns_pool, l); } - mutex_up(ldlm_namespace_lock(client)); + mutex_unlock(ldlm_namespace_lock(client)); } /* * Recalc at least ldlm_namespace_nr(client) namespaces. */ - for (nr = atomic_read(ldlm_namespace_nr(client)); nr > 0; nr--) { + for (nr = ldlm_namespace_nr_read(client); nr > 0; nr--) { int skip; /* * Lock the list, get first @ns in the list, getref, move it @@ -1230,37 +1273,66 @@ void ldlm_pools_recalc(ldlm_side_t client) * rid of potential deadlock on client nodes when canceling * locks synchronously. */ - mutex_down(ldlm_namespace_lock(client)); - if (list_empty(ldlm_namespace_list(client))) { - mutex_up(ldlm_namespace_lock(client)); - break; - } - ns = ldlm_namespace_first_locked(client); - - spin_lock(&ns->ns_hash_lock); - /* - * skip ns which is being freed, and we don't want to increase - * its refcount again, not even temporarily. bz21519. - */ - if (ns->ns_refcount == 0) { - skip = 1; - } else { - skip = 0; - ldlm_namespace_get_locked(ns); - } - spin_unlock(&ns->ns_hash_lock); - - ldlm_namespace_move_locked(ns, client); - mutex_up(ldlm_namespace_lock(client)); - - /* - * After setup is done - recalc the pool. - */ - if (!skip) { - ldlm_pool_recalc(&ns->ns_pool); - ldlm_namespace_put(ns, 1); - } + mutex_lock(ldlm_namespace_lock(client)); + if (cfs_list_empty(ldlm_namespace_list(client))) { + mutex_unlock(ldlm_namespace_lock(client)); + break; + } + ns = ldlm_namespace_first_locked(client); + + if (ns_old == ns) { /* Full pass complete */ + mutex_unlock(ldlm_namespace_lock(client)); + break; + } + + /* We got an empty namespace, need to move it back to inactive + * list. + * The race with parallel resource creation is fine: + * - If they do namespace_get before our check, we fail the + * check and they move this item to the end of the list anyway + * - If we do the check and then they do namespace_get, then + * we move the namespace to inactive and they will move + * it back to active (synchronised by the lock, so no clash + * there). + */ + if (ldlm_ns_empty(ns)) { + ldlm_namespace_move_to_inactive_locked(ns, client); + mutex_unlock(ldlm_namespace_lock(client)); + continue; + } + + if (ns_old == NULL) + ns_old = ns; + + spin_lock(&ns->ns_lock); + /* + * skip ns which is being freed, and we don't want to increase + * its refcount again, not even temporarily. bz21519 & LU-499. + */ + if (ns->ns_stopping) { + skip = 1; + } else { + skip = 0; + ldlm_namespace_get(ns); + } + spin_unlock(&ns->ns_lock); + + ldlm_namespace_move_to_active_locked(ns, client); + mutex_unlock(ldlm_namespace_lock(client)); + + /* + * After setup is done - recalc the pool. + */ + if (!skip) { + int ttime = ldlm_pool_recalc(&ns->ns_pool); + + if (ttime < time) + time = ttime; + + ldlm_namespace_put(ns); + } } + return time; } EXPORT_SYMBOL(ldlm_pools_recalc); @@ -1268,10 +1340,11 @@ static int ldlm_pools_thread_main(void *arg) { struct ptlrpc_thread *thread = (struct ptlrpc_thread *)arg; char *t_name = "ldlm_poold"; + int s_time, c_time; ENTRY; cfs_daemonize(t_name); - thread->t_flags = SVC_RUNNING; + thread_set_flags(thread, SVC_RUNNING); cfs_waitq_signal(&thread->t_ctl_waitq); CDEBUG(D_DLMTRACE, "%s: pool thread starting, process %d\n", @@ -1280,37 +1353,36 @@ static int ldlm_pools_thread_main(void *arg) while (1) { struct l_wait_info lwi; - /* - * Recal all pools on this tick. - */ - ldlm_pools_recalc(LDLM_NAMESPACE_SERVER); - ldlm_pools_recalc(LDLM_NAMESPACE_CLIENT); - - /* - * Wait until the next check time, or until we're - * stopped. - */ - lwi = LWI_TIMEOUT(cfs_time_seconds(LDLM_POOLS_THREAD_PERIOD), - NULL, NULL); - l_wait_event(thread->t_ctl_waitq, (thread->t_flags & - (SVC_STOPPING|SVC_EVENT)), + /* + * Recal all pools on this tick. + */ + s_time = ldlm_pools_recalc(LDLM_NAMESPACE_SERVER); + c_time = ldlm_pools_recalc(LDLM_NAMESPACE_CLIENT); + + /* + * Wait until the next check time, or until we're + * stopped. + */ + lwi = LWI_TIMEOUT(cfs_time_seconds(min(s_time, c_time)), + NULL, NULL); + l_wait_event(thread->t_ctl_waitq, + thread_is_stopping(thread) || + thread_is_event(thread), &lwi); - if (thread->t_flags & SVC_STOPPING) { - thread->t_flags &= ~SVC_STOPPING; + if (thread_test_and_clear_flags(thread, SVC_STOPPING)) break; - } else if (thread->t_flags & SVC_EVENT) { - thread->t_flags &= ~SVC_EVENT; - } + else + thread_test_and_clear_flags(thread, SVC_EVENT); } - thread->t_flags = SVC_STOPPED; + thread_set_flags(thread, SVC_STOPPED); cfs_waitq_signal(&thread->t_ctl_waitq); CDEBUG(D_DLMTRACE, "%s: pool thread exiting, process %d\n", t_name, cfs_curproc_pid()); - complete_and_exit(&ldlm_pools_comp, 0); + complete_and_exit(&ldlm_pools_comp, 0); } static int ldlm_pools_thread_start(void) @@ -1326,15 +1398,15 @@ static int ldlm_pools_thread_start(void) if (ldlm_pools_thread == NULL) RETURN(-ENOMEM); - init_completion(&ldlm_pools_comp); + init_completion(&ldlm_pools_comp); cfs_waitq_init(&ldlm_pools_thread->t_ctl_waitq); /* * CLONE_VM and CLONE_FILES just avoid a needless copy, because we * just drop the VM and FILES in cfs_daemonize() right away. */ - rc = cfs_kernel_thread(ldlm_pools_thread_main, ldlm_pools_thread, - CLONE_VM | CLONE_FILES); + rc = cfs_create_thread(ldlm_pools_thread_main, ldlm_pools_thread, + CFS_DAEMON_FLAGS); if (rc < 0) { CERROR("Can't start pool thread, error %d\n", rc); @@ -1343,7 +1415,7 @@ static int ldlm_pools_thread_start(void) RETURN(rc); } l_wait_event(ldlm_pools_thread->t_ctl_waitq, - (ldlm_pools_thread->t_flags & SVC_RUNNING), &lwi); + thread_is_running(ldlm_pools_thread), &lwi); RETURN(0); } @@ -1356,7 +1428,7 @@ static void ldlm_pools_thread_stop(void) return; } - ldlm_pools_thread->t_flags = SVC_STOPPING; + thread_set_flags(ldlm_pools_thread, SVC_STOPPING); cfs_waitq_signal(&ldlm_pools_thread->t_ctl_waitq); /* @@ -1364,7 +1436,7 @@ static void ldlm_pools_thread_stop(void) * This fixes possible race and oops due to accessing freed memory * in pools thread. */ - wait_for_completion(&ldlm_pools_comp); + wait_for_completion(&ldlm_pools_comp); OBD_FREE_PTR(ldlm_pools_thread); ldlm_pools_thread = NULL; EXIT; @@ -1377,10 +1449,12 @@ int ldlm_pools_init(void) rc = ldlm_pools_thread_start(); if (rc == 0) { - ldlm_pools_srv_shrinker = set_shrinker(DEFAULT_SEEKS, - ldlm_pools_srv_shrink); - ldlm_pools_cli_shrinker = set_shrinker(DEFAULT_SEEKS, - ldlm_pools_cli_shrink); + ldlm_pools_srv_shrinker = + cfs_set_shrinker(CFS_DEFAULT_SEEKS, + ldlm_pools_srv_shrink); + ldlm_pools_cli_shrinker = + cfs_set_shrinker(CFS_DEFAULT_SEEKS, + ldlm_pools_cli_shrink); } RETURN(rc); } @@ -1389,11 +1463,11 @@ EXPORT_SYMBOL(ldlm_pools_init); void ldlm_pools_fini(void) { if (ldlm_pools_srv_shrinker != NULL) { - remove_shrinker(ldlm_pools_srv_shrinker); + cfs_remove_shrinker(ldlm_pools_srv_shrinker); ldlm_pools_srv_shrinker = NULL; } if (ldlm_pools_cli_shrinker != NULL) { - remove_shrinker(ldlm_pools_cli_shrinker); + cfs_remove_shrinker(ldlm_pools_cli_shrinker); ldlm_pools_cli_shrinker = NULL; } ldlm_pools_thread_stop(); @@ -1500,9 +1574,9 @@ void ldlm_pools_fini(void) } EXPORT_SYMBOL(ldlm_pools_fini); -void ldlm_pools_recalc(ldlm_side_t client) +int ldlm_pools_recalc(ldlm_side_t client) { - return; + return 0; } EXPORT_SYMBOL(ldlm_pools_recalc); #endif /* HAVE_LRU_RESIZE_SUPPORT */