From ec295cad998d4d5e7fc915491da5ac646dbc6af6 Mon Sep 17 00:00:00 2001 From: yury Date: Mon, 5 Nov 2007 16:15:04 +0000 Subject: [PATCH 1/1] b=13872 r=adilger,nathan - lru resize and ldlm cleanups and fixes. --- lustre/include/lustre_dlm.h | 32 ++++++++++++++++++++++++++------ lustre/include/lustre_export.h | 10 ++++++---- lustre/ldlm/l_lock.c | 4 ++-- lustre/ldlm/ldlm_flock.c | 2 +- lustre/ldlm/ldlm_lib.c | 6 +++--- lustre/ldlm/ldlm_lock.c | 12 ++++++------ lustre/ldlm/ldlm_lockd.c | 18 +++++++++--------- lustre/ldlm/ldlm_pool.c | 23 +++++++++++------------ lustre/ldlm/ldlm_request.c | 8 ++++---- lustre/ldlm/ldlm_resource.c | 22 ++++++++++------------ 10 files changed, 78 insertions(+), 59 deletions(-) diff --git a/lustre/include/lustre_dlm.h b/lustre/include/lustre_dlm.h index b48efef..393f966 100644 --- a/lustre/include/lustre_dlm.h +++ b/lustre/include/lustre_dlm.h @@ -43,8 +43,8 @@ typedef enum { } ldlm_error_t; typedef enum { - LDLM_NAMESPACE_SERVER = 0, - LDLM_NAMESPACE_CLIENT = 1 + LDLM_NAMESPACE_SERVER = 1 << 0, + LDLM_NAMESPACE_CLIENT = 1 << 1 } ldlm_side_t; #define LDLM_FL_LOCK_CHANGED 0x000001 /* extent, mode, or resource changed */ @@ -324,10 +324,30 @@ struct ldlm_namespace { ldlm_appetite_t ns_appetite; }; +static inline int ns_is_client(struct ldlm_namespace *ns) +{ + LASSERT(ns != NULL); + LASSERT(!(ns->ns_client & ~(LDLM_NAMESPACE_CLIENT | + LDLM_NAMESPACE_SERVER))); + LASSERT(ns->ns_client == LDLM_NAMESPACE_CLIENT || + ns->ns_client == LDLM_NAMESPACE_SERVER); + return ns->ns_client == LDLM_NAMESPACE_CLIENT; +} + +static inline int ns_is_server(struct ldlm_namespace *ns) +{ + LASSERT(ns != NULL); + LASSERT(!(ns->ns_client & ~(LDLM_NAMESPACE_CLIENT | + LDLM_NAMESPACE_SERVER))); + LASSERT(ns->ns_client == LDLM_NAMESPACE_CLIENT || + ns->ns_client == LDLM_NAMESPACE_SERVER); + return ns->ns_client == LDLM_NAMESPACE_SERVER; +} + static inline int ns_connect_lru_resize(struct ldlm_namespace *ns) { LASSERT(ns != NULL); - return ns->ns_connect_flags & OBD_CONNECT_LRU_RESIZE; + return !!(ns->ns_connect_flags & OBD_CONNECT_LRU_RESIZE); } /* @@ -559,8 +579,8 @@ int ldlm_request_cancel(struct ptlrpc_request *req, int ldlm_del_waiting_lock(struct ldlm_lock *lock); int ldlm_refresh_waiting_lock(struct ldlm_lock *lock); void ldlm_revoke_export_locks(struct obd_export *exp); -int ldlm_get_ref(ldlm_side_t client); -void ldlm_put_ref(ldlm_side_t client, int force); +int ldlm_get_ref(void); +void ldlm_put_ref(int force); /* ldlm_lock.c */ ldlm_processing_policy ldlm_get_processing_policy(struct ldlm_resource *res); @@ -772,8 +792,8 @@ struct ldlm_resource * lock_res_and_lock(struct ldlm_lock *lock); void unlock_res_and_lock(struct ldlm_lock *lock); /* ldlm_pool.c */ -int ldlm_pools_init(ldlm_side_t client); void ldlm_pools_recalc(ldlm_side_t client); +int ldlm_pools_init(void); void ldlm_pools_fini(void); void ldlm_pools_wakeup(void); diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h index cb47281..787094c 100644 --- a/lustre/include/lustre_export.h +++ b/lustre/include/lustre_export.h @@ -126,20 +126,22 @@ struct obd_export { static inline int exp_connect_cancelset(struct obd_export *exp) { - return exp ? exp->exp_connect_flags & OBD_CONNECT_CANCELSET : 0; + return exp ? !!(exp->exp_connect_flags & OBD_CONNECT_CANCELSET) : 0; } static inline int exp_connect_lru_resize(struct obd_export *exp) { LASSERT(exp != NULL); - return exp->exp_connect_flags & OBD_CONNECT_LRU_RESIZE; + return !!(exp->exp_connect_flags & OBD_CONNECT_LRU_RESIZE); } static inline int imp_connect_lru_resize(struct obd_import *imp) { + struct obd_connect_data *ocd; + LASSERT(imp != NULL); - return imp->imp_connect_data.ocd_connect_flags & - OBD_CONNECT_LRU_RESIZE; + ocd = &imp->imp_connect_data; + return !!(ocd->ocd_connect_flags & OBD_CONNECT_LRU_RESIZE); } extern struct obd_export *class_conn2export(struct lustre_handle *conn); diff --git a/lustre/ldlm/l_lock.c b/lustre/ldlm/l_lock.c index fdb8548..e23a755 100644 --- a/lustre/ldlm/l_lock.c +++ b/lustre/ldlm/l_lock.c @@ -43,7 +43,7 @@ struct ldlm_resource * lock_res_and_lock(struct ldlm_lock *lock) { struct ldlm_resource *res = lock->l_resource; - if (!res->lr_namespace->ns_client) { + if (ns_is_server(res->lr_namespace)) { /* on server-side resource of lock doesn't change */ lock_res(res); return res; @@ -59,7 +59,7 @@ void unlock_res_and_lock(struct ldlm_lock *lock) { struct ldlm_resource *res = lock->l_resource; - if (!res->lr_namespace->ns_client) { + if (ns_is_server(res->lr_namespace)) { /* on server-side resource of lock doesn't change */ unlock_res(res); return; diff --git a/lustre/ldlm/ldlm_flock.c b/lustre/ldlm/ldlm_flock.c index 6ca050b..6cb71e7 100644 --- a/lustre/ldlm/ldlm_flock.c +++ b/lustre/ldlm/ldlm_flock.c @@ -134,7 +134,7 @@ ldlm_process_flock_lock(struct ldlm_lock *req, int *flags, int first_enq, struct ldlm_lock *new = req; struct ldlm_lock *new2 = NULL; ldlm_mode_t mode = req->l_req_mode; - int local = ns->ns_client; + int local = ns_is_client(ns); int added = (mode == LCK_NL); int overlaps = 0; ENTRY; diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index f8d8145..b2f8c44 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -296,7 +296,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) cli->cl_max_rpcs_in_flight = OSC_MAX_RIF_DEFAULT; } - rc = ldlm_get_ref(LDLM_NAMESPACE_CLIENT); + rc = ldlm_get_ref(); if (rc) { CERROR("ldlm_get_ref failed: %d\n", rc); GOTO(err, rc); @@ -346,7 +346,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) err_import: class_destroy_import(imp); err_ldlm: - ldlm_put_ref(LDLM_NAMESPACE_CLIENT, 0); + ldlm_put_ref(0); err: RETURN(rc); @@ -355,7 +355,7 @@ err: int client_obd_cleanup(struct obd_device *obddev) { ENTRY; - ldlm_put_ref(LDLM_NAMESPACE_CLIENT, obddev->obd_force); + ldlm_put_ref(obddev->obd_force); RETURN(0); } diff --git a/lustre/ldlm/ldlm_lock.c b/lustre/ldlm/ldlm_lock.c index e422d98..e83d396 100644 --- a/lustre/ldlm/ldlm_lock.c +++ b/lustre/ldlm/ldlm_lock.c @@ -358,7 +358,7 @@ int ldlm_lock_change_resource(struct ldlm_namespace *ns, struct ldlm_lock *lock, int type; ENTRY; - LASSERT(ns->ns_client != 0); + LASSERT(ns_is_client(ns)); lock_res_and_lock(lock); if (memcmp(new_resid, &lock->l_resource->lr_name, @@ -609,7 +609,7 @@ void ldlm_lock_decref_internal(struct ldlm_lock *lock, __u32 mode) (lock->l_flags & LDLM_FL_CBPENDING)) { /* If we received a blocked AST and this was the last reference, * run the callback. */ - if (ns->ns_client == LDLM_NAMESPACE_SERVER && lock->l_export) + if (ns_is_server(ns) && lock->l_export) CERROR("FL_CBPENDING set on non-local lock--just a " "warning\n"); @@ -621,7 +621,7 @@ void ldlm_lock_decref_internal(struct ldlm_lock *lock, __u32 mode) if ((lock->l_flags & LDLM_FL_ATOMIC_CB) || ldlm_bl_to_thread_lock(ns, NULL, lock) != 0) ldlm_handle_bl_callback(ns, NULL, lock); - } else if (ns->ns_client == LDLM_NAMESPACE_CLIENT && + } else if (ns_is_client(ns) && !lock->l_readers && !lock->l_writers && !(lock->l_flags & LDLM_FL_NO_LRU)) { /* If this is a client-side namespace and this was the last @@ -1152,7 +1152,7 @@ ldlm_error_t ldlm_lock_enqueue(struct ldlm_namespace *ns, { struct ldlm_lock *lock = *lockp; struct ldlm_resource *res = lock->l_resource; - int local = res->lr_namespace->ns_client; + int local = ns_is_client(res->lr_namespace); ldlm_processing_policy policy; ldlm_error_t rc = ELDLM_OK; ENTRY; @@ -1392,7 +1392,7 @@ void ldlm_reprocess_all(struct ldlm_resource *res) ENTRY; /* Local lock trees don't get reprocessed. */ - if (res->lr_namespace->ns_client) { + if (ns_is_client(res->lr_namespace)) { EXIT; return; } @@ -1619,7 +1619,7 @@ struct ldlm_resource *ldlm_lock_convert(struct ldlm_lock *lock, int new_mode, ldlm_resource_unlink_lock(lock); /* If this is a local resource, put it on the appropriate list. */ - if (res->lr_namespace->ns_client) { + if (ns_is_client(res->lr_namespace)) { if (*flags & (LDLM_FL_BLOCK_CONV | LDLM_FL_BLOCK_GRANTED)) { ldlm_resource_add_lock(res, &res->lr_converting, lock); } else { diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c index df34b4e..08b2ead 100644 --- a/lustre/ldlm/ldlm_lockd.c +++ b/lustre/ldlm/ldlm_lockd.c @@ -1761,16 +1761,16 @@ static int ldlm_bl_thread_main(void *arg) #endif -static int ldlm_setup(ldlm_side_t client); -static int ldlm_cleanup(ldlm_side_t client, int force); +static int ldlm_setup(void); +static int ldlm_cleanup(int force); -int ldlm_get_ref(ldlm_side_t client) +int ldlm_get_ref(void) { int rc = 0; ENTRY; mutex_down(&ldlm_ref_sem); if (++ldlm_refcount == 1) { - rc = ldlm_setup(client); + rc = ldlm_setup(); if (rc) ldlm_refcount--; } @@ -1779,12 +1779,12 @@ int ldlm_get_ref(ldlm_side_t client) RETURN(rc); } -void ldlm_put_ref(ldlm_side_t client, int force) +void ldlm_put_ref(int force) { ENTRY; mutex_down(&ldlm_ref_sem); if (ldlm_refcount == 1) { - int rc = ldlm_cleanup(client, force); + int rc = ldlm_cleanup(force); if (rc) CERROR("ldlm_cleanup failed: %d\n", rc); else @@ -1797,7 +1797,7 @@ void ldlm_put_ref(ldlm_side_t client, int force) EXIT; } -static int ldlm_setup(ldlm_side_t client) +static int ldlm_setup(void) { struct ldlm_bl_pool *blp; int rc = 0; @@ -1902,7 +1902,7 @@ static int ldlm_setup(ldlm_side_t client) #endif #ifdef __KERNEL__ - rc = ldlm_pools_init(client); + rc = ldlm_pools_init(); if (rc) GOTO(out_thread, rc); #endif @@ -1924,7 +1924,7 @@ static int ldlm_setup(ldlm_side_t client) return rc; } -static int ldlm_cleanup(ldlm_side_t client, int force) +static int ldlm_cleanup(int force) { #ifdef __KERNEL__ struct ldlm_bl_pool *blp = ldlm_state->ldlm_bl_pool; diff --git a/lustre/ldlm/ldlm_pool.c b/lustre/ldlm/ldlm_pool.c index 70c6eb9..704b4cd 100644 --- a/lustre/ldlm/ldlm_pool.c +++ b/lustre/ldlm/ldlm_pool.c @@ -337,7 +337,7 @@ EXPORT_SYMBOL(ldlm_pool_shrink); int ldlm_pool_setup(struct ldlm_pool *pl, __u32 limit) { ENTRY; - if (ldlm_pl2ns(pl)->ns_client == LDLM_NAMESPACE_SERVER) + if (ns_is_server(ldlm_pl2ns(pl))) ldlm_pool_set_limit(pl, limit); RETURN(0); } @@ -368,7 +368,7 @@ static int lprocfs_rd_pool_state(char *page, char **start, off_t off, pl->pl_name); nr += snprintf(page + nr, count - nr, " SLV: "LPU64"\n", slv); - if (ldlm_pl2ns(pl)->ns_client == LDLM_NAMESPACE_CLIENT) { + if (ns_is_client(ldlm_pl2ns(pl))) { nr += snprintf(page + nr, count - nr, " LVF: %d\n", atomic_read(&pl->pl_lock_volume_factor)); } @@ -465,11 +465,11 @@ static int ldlm_pool_proc_init(struct ldlm_pool *pl) snprintf(var_name, MAX_STRING_SIZE, "grant_step"); pool_vars[0].data = &pl->pl_grant_step; pool_vars[0].read_fptr = lprocfs_rd_atomic; - if (ns->ns_client == LDLM_NAMESPACE_SERVER) + if (ns_is_server(ns)) pool_vars[0].write_fptr = lprocfs_wr_atomic; lprocfs_add_vars(pl->pl_proc_dir, pool_vars, 0); - if (ns->ns_client == LDLM_NAMESPACE_CLIENT) { + if (ns_is_client(ns)) { snprintf(var_name, MAX_STRING_SIZE, "lock_volume_factor"); pool_vars[0].data = &pl->pl_lock_volume_factor; pool_vars[0].read_fptr = lprocfs_rd_uint; @@ -589,7 +589,7 @@ void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock) /* No need to recalc client pools here as this is already done * on enqueue/cancel and locks to cancel already packed to the * rpc. */ - if (ldlm_pl2ns(pl)->ns_client == LDLM_NAMESPACE_SERVER) + if (ns_is_server(ldlm_pl2ns(pl))) ldlm_pool_recalc(pl); EXIT; } @@ -604,7 +604,7 @@ void ldlm_pool_del(struct ldlm_pool *pl, struct ldlm_lock *lock) atomic_dec(&pl->pl_grant_speed); /* Same as in ldlm_pool_add() */ - if (ldlm_pl2ns(pl)->ns_client == LDLM_NAMESPACE_SERVER) + if (ns_is_server(ldlm_pl2ns(pl))) ldlm_pool_recalc(pl); EXIT; } @@ -825,8 +825,8 @@ static int ldlm_pools_thread_main(void *arg) struct l_wait_info lwi; /* Recal all pools on this tick. */ - ldlm_pools_recalc(LDLM_NAMESPACE_CLIENT); ldlm_pools_recalc(LDLM_NAMESPACE_SERVER); + ldlm_pools_recalc(LDLM_NAMESPACE_CLIENT); /* Wait until the next check time, or until we're * stopped. */ @@ -853,7 +853,7 @@ static int ldlm_pools_thread_main(void *arg) complete_and_exit(&ldlm_pools_comp, 0); } -static int ldlm_pools_thread_start(ldlm_side_t client) +static int ldlm_pools_thread_start(void) { struct l_wait_info lwi = { 0 }; int rc; @@ -866,7 +866,6 @@ static int ldlm_pools_thread_start(ldlm_side_t client) if (ldlm_pools_thread == NULL) RETURN(-ENOMEM); - ldlm_pools_thread->t_id = client; init_completion(&ldlm_pools_comp); cfs_waitq_init(&ldlm_pools_thread->t_ctl_waitq); @@ -907,12 +906,12 @@ static void ldlm_pools_thread_stop(void) EXIT; } -int ldlm_pools_init(ldlm_side_t client) +int ldlm_pools_init(void) { int rc; ENTRY; - rc = ldlm_pools_thread_start(client); + rc = ldlm_pools_thread_start(); if (rc == 0) { ldlm_pools_srv_shrinker = set_shrinker(DEFAULT_SEEKS, ldlm_pools_srv_shrink); @@ -1007,7 +1006,7 @@ void ldlm_pool_set_limit(struct ldlm_pool *pl, __u32 limit) } EXPORT_SYMBOL(ldlm_pool_set_limit); -int ldlm_pools_init(ldlm_side_t client) +int ldlm_pools_init(void) { return 0; } diff --git a/lustre/ldlm/ldlm_request.c b/lustre/ldlm/ldlm_request.c index c40d2e8..8aff2f6 100644 --- a/lustre/ldlm/ldlm_request.c +++ b/lustre/ldlm/ldlm_request.c @@ -245,7 +245,7 @@ int ldlm_cli_enqueue_local(struct ldlm_namespace *ns, ENTRY; LASSERT(!(*flags & LDLM_FL_REPLAY)); - if (unlikely(ns->ns_client)) { + if (unlikely(ns_is_client(ns))) { CERROR("Trying to enqueue local lock in a shadow namespace\n"); LBUG(); } @@ -686,7 +686,7 @@ static int ldlm_cli_convert_local(struct ldlm_lock *lock, int new_mode, struct ldlm_resource *res; int rc; ENTRY; - if (lock->l_resource->lr_namespace->ns_client) { + if (ns_is_client(lock->l_resource->lr_namespace)) { CERROR("Trying to cancel local lock\n"); LBUG(); } @@ -812,7 +812,7 @@ static int ldlm_cli_cancel_local(struct ldlm_lock *lock) } ldlm_lock_cancel(lock); } else { - if (lock->l_resource->lr_namespace->ns_client) { + if (ns_is_client(lock->l_resource->lr_namespace)) { LDLM_ERROR(lock, "Trying to cancel local lock"); LBUG(); } @@ -1441,7 +1441,7 @@ int ldlm_cli_join_lru(struct ldlm_namespace *ns, int count = 0; ENTRY; - LASSERT(ns->ns_client == LDLM_NAMESPACE_CLIENT); + LASSERT(ns_is_client(ns)); res = ldlm_resource_get(ns, NULL, res_id, LDLM_EXTENT, 0); if (res == NULL) diff --git a/lustre/ldlm/ldlm_resource.c b/lustre/ldlm/ldlm_resource.c index 5c12ea3..4f68aed 100644 --- a/lustre/ldlm/ldlm_resource.c +++ b/lustre/ldlm/ldlm_resource.c @@ -214,7 +214,7 @@ void ldlm_proc_namespace(struct ldlm_namespace *ns) lock_vars[0].read_fptr = lprocfs_rd_atomic; lprocfs_add_vars(ldlm_ns_proc_dir, lock_vars, 0); - if (ns->ns_client) { + if (ns_is_client(ns)) { snprintf(lock_name, MAX_STRING_SIZE, "%s/lock_unused_count", ns->ns_name); lock_vars[0].data = &ns->ns_nr_unused; @@ -250,7 +250,7 @@ struct ldlm_namespace *ldlm_namespace_new(char *name, ldlm_side_t client, int rc, idx, namelen; ENTRY; - rc = ldlm_get_ref(client); + rc = ldlm_get_ref(); if (rc) { CERROR("ldlm_get_ref failed: %d\n", rc); RETURN(NULL); @@ -313,7 +313,7 @@ out_hash: out_ns: OBD_FREE_PTR(ns); out_ref: - ldlm_put_ref(client, 0); + ldlm_put_ref(0); RETURN(NULL); } @@ -328,7 +328,7 @@ static void cleanup_resource(struct ldlm_resource *res, struct list_head *q, int flags) { struct list_head *tmp; - int rc = 0, client = res->lr_namespace->ns_client; + int rc = 0, client = ns_is_client(res->lr_namespace); int local_only = (flags & LDLM_FL_LOCAL_ONLY); ENTRY; @@ -445,17 +445,15 @@ int ldlm_namespace_cleanup(struct ldlm_namespace *ns, int flags) /* Cleanup, but also free, the namespace */ int ldlm_namespace_free(struct ldlm_namespace *ns, int force) { - ldlm_side_t client; ENTRY; if (!ns) RETURN(ELDLM_OK); - client = ns->ns_client; - mutex_down(ldlm_namespace_lock(client)); + mutex_down(ldlm_namespace_lock(ns->ns_client)); list_del(&ns->ns_list_chain); atomic_dec(ldlm_namespace_nr(ns->ns_client)); ldlm_pool_fini(&ns->ns_pool); - mutex_up(ldlm_namespace_lock(client)); + mutex_up(ldlm_namespace_lock(ns->ns_client)); /* At shutdown time, don't call the cancellation callback */ ldlm_namespace_cleanup(ns, 0); @@ -493,12 +491,11 @@ int ldlm_namespace_free(struct ldlm_namespace *ns, int force) "dlm namespace %s free done waiting\n", ns->ns_name); } - POISON(ns->ns_hash, 0x5a, sizeof(*ns->ns_hash) * RES_HASH_SIZE); OBD_VFREE(ns->ns_hash, sizeof(*ns->ns_hash) * RES_HASH_SIZE); OBD_FREE(ns->ns_name, strlen(ns->ns_name) + 1); OBD_FREE_PTR(ns); - ldlm_put_ref(client, force); + ldlm_put_ref(force); RETURN(ELDLM_OK); } @@ -883,8 +880,9 @@ void ldlm_namespace_dump(int level, struct ldlm_namespace *ns) if (!((libcfs_debug | D_ERROR) & level)) return; - CDEBUG(level, "--- Namespace: %s (rc: %d, client: %d)\n", - ns->ns_name, ns->ns_refcount, ns->ns_client); + CDEBUG(level, "--- Namespace: %s (rc: %d, side: %s)\n", + ns->ns_name, ns->ns_refcount, + ns_is_client(ns) ? "client" : "server"); if (cfs_time_before(cfs_time_current(), ns->ns_next_dump)) return; -- 1.8.3.1