From bef2828d5dbe56fd0a59c2a260794771f89795b8 Mon Sep 17 00:00:00 2001 From: Jinshan Xiong Date: Thu, 25 Aug 2011 13:32:02 -0700 Subject: [PATCH] LU-641 obdclass: use rwlock for obd_dev_lock In target_handle_connect() it grabs obd_dev_lock to find obd_device by target uuid. This causes lock contention if there are two many connection requests on the same time. Based on obd_dev_lock is a read-mostly lock, it is much reasonable to revise it as a rwlock. Change-Id: I7f936d6182df9e7ed46b8a1b6fb259226188acbc Signed-off-by: Jinshan Xiong Reviewed-on: http://review.whamcloud.com/1294 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/obd_class.h | 2 +- lustre/obdclass/class_obd.c | 21 +---------- lustre/obdclass/genops.c | 67 ++++++++++++++++++++++++------------ lustre/obdclass/linux/linux-module.c | 8 ++--- 4 files changed, 51 insertions(+), 47 deletions(-) diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 198b10a..df7cd84 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -63,7 +63,7 @@ /* OBD Device Declarations */ extern struct obd_device *obd_devs[MAX_OBD_DEVICES]; -extern cfs_spinlock_t obd_dev_lock; +extern cfs_rwlock_t obd_dev_lock; /* OBD Operations Declarations */ extern struct obd_device *class_conn2obd(struct lustre_handle *); diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c index 42255a5..e344148 100644 --- a/lustre/obdclass/class_obd.c +++ b/lustre/obdclass/class_obd.c @@ -60,7 +60,7 @@ cfs_atomic_t libcfs_kmemory = {0}; struct obd_device *obd_devs[MAX_OBD_DEVICES]; cfs_list_t obd_types; -cfs_spinlock_t obd_dev_lock = CFS_SPIN_LOCK_UNLOCKED; +cfs_rwlock_t obd_dev_lock = CFS_RW_LOCK_UNLOCKED; #ifndef __KERNEL__ __u64 obd_max_pages = 0; @@ -368,24 +368,6 @@ EXPORT_SYMBOL(ptlrpc_put_connection_superhack); EXPORT_SYMBOL(proc_lustre_root); -EXPORT_SYMBOL(class_register_type); -EXPORT_SYMBOL(class_unregister_type); -EXPORT_SYMBOL(class_get_type); -EXPORT_SYMBOL(class_put_type); -EXPORT_SYMBOL(class_name2dev); -EXPORT_SYMBOL(class_name2obd); -EXPORT_SYMBOL(class_uuid2dev); -EXPORT_SYMBOL(class_uuid2obd); -EXPORT_SYMBOL(class_find_client_obd); -EXPORT_SYMBOL(class_devices_in_group); -EXPORT_SYMBOL(class_conn2export); -EXPORT_SYMBOL(class_exp2obd); -EXPORT_SYMBOL(class_conn2obd); -EXPORT_SYMBOL(class_exp2cliimp); -EXPORT_SYMBOL(class_conn2cliimp); -EXPORT_SYMBOL(class_disconnect); -EXPORT_SYMBOL(class_num2obd); - /* uuid.c */ EXPORT_SYMBOL(class_uuid_unparse); EXPORT_SYMBOL(lustre_uuid_to_peer); @@ -541,7 +523,6 @@ int init_obdclass(void) if (err) return err; - cfs_spin_lock_init(&obd_dev_lock); CFS_INIT_LIST_HEAD(&obd_types); err = cfs_psdev_register(&obd_psdev); diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 55435b3..36d8b44 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -139,6 +139,7 @@ struct obd_type *class_get_type(const char *name) } return type; } +EXPORT_SYMBOL(class_get_type); void class_put_type(struct obd_type *type) { @@ -148,6 +149,7 @@ void class_put_type(struct obd_type *type) cfs_module_put(type->typ_dt_ops->o_owner); cfs_spin_unlock(&type->obd_type_lock); } +EXPORT_SYMBOL(class_put_type); #define CLASS_MAX_NAME 1024 @@ -220,6 +222,7 @@ int class_register_type(struct obd_ops *dt_ops, struct md_ops *md_ops, OBD_FREE(type, sizeof(*type)); RETURN(rc); } +EXPORT_SYMBOL(class_register_type); int class_unregister_type(const char *name) { @@ -258,6 +261,7 @@ int class_unregister_type(const char *name) OBD_FREE(type, sizeof(*type)); RETURN(0); } /* class_unregister_type */ +EXPORT_SYMBOL(class_unregister_type); /** * Create a new obd device. @@ -296,9 +300,10 @@ struct obd_device *class_newdev(const char *type_name, const char *name) } LASSERT(newdev->obd_magic == OBD_DEVICE_MAGIC); - cfs_spin_lock(&obd_dev_lock); + cfs_write_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); + if (obd && obd->obd_name && (strcmp(name, obd->obd_name) == 0)) { CERROR("Device %s already exists, won't add\n", name); @@ -326,7 +331,7 @@ struct obd_device *class_newdev(const char *type_name, const char *name) obd_devs[i] = result; } } - cfs_spin_unlock(&obd_dev_lock); + cfs_write_unlock(&obd_dev_lock); if (result == NULL && i >= class_devno_max()) { CERROR("all %u OBD devices used, increase MAX_OBD_DEVICES\n", @@ -357,9 +362,9 @@ void class_release_dev(struct obd_device *obd) CDEBUG(D_INFO, "Release obd device %s obd_type name =%s\n", obd->obd_name,obd->obd_type->typ_name); - cfs_spin_lock(&obd_dev_lock); + cfs_write_lock(&obd_dev_lock); obd_devs[obd->obd_minor] = NULL; - cfs_spin_unlock(&obd_dev_lock); + cfs_write_unlock(&obd_dev_lock); obd_device_free(obd); class_put_type(obd_type); @@ -372,24 +377,26 @@ int class_name2dev(const char *name) if (!name) return -1; - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); + if (obd && obd->obd_name && strcmp(name, obd->obd_name) == 0) { /* Make sure we finished attaching before we give out any references */ LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC); if (obd->obd_attached) { - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return i; } break; } } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return -1; } +EXPORT_SYMBOL(class_name2dev); struct obd_device *class_name2obd(const char *name) { @@ -399,24 +406,27 @@ struct obd_device *class_name2obd(const char *name) return NULL; return class_num2obd(dev); } +EXPORT_SYMBOL(class_name2obd); int class_uuid2dev(struct obd_uuid *uuid) { int i; - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); + if (obd && obd_uuid_equals(uuid, &obd->obd_uuid)) { LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC); - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return i; } } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return -1; } +EXPORT_SYMBOL(class_uuid2dev); struct obd_device *class_uuid2obd(struct obd_uuid *uuid) { @@ -425,6 +435,7 @@ struct obd_device *class_uuid2obd(struct obd_uuid *uuid) return NULL; return class_num2obd(dev); } +EXPORT_SYMBOL(class_uuid2obd); /** * Get obd device from ::obd_devs[] @@ -453,15 +464,17 @@ struct obd_device *class_num2obd(int num) return obd; } +EXPORT_SYMBOL(class_num2obd); void class_obd_list(void) { char *status; int i; - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); + if (obd == NULL) continue; if (obd->obd_stopping) @@ -477,7 +490,7 @@ void class_obd_list(void) obd->obd_name, obd->obd_uuid.uuid, cfs_atomic_read(&obd->obd_refcount)); } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return; } @@ -490,9 +503,10 @@ struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid, { int i; - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); + if (obd == NULL) continue; if ((strncmp(obd->obd_type->typ_name, typ_name, @@ -501,15 +515,16 @@ struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid, &obd->u.cli.cl_target_uuid) && ((grp_uuid)? obd_uuid_equals(grp_uuid, &obd->obd_uuid) : 1)) { - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return obd; } } } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return NULL; } +EXPORT_SYMBOL(class_find_client_obd); /* Iterate the obd_device list looking devices have grp_uuid. Start searching at *next, and if a device is found, the next index to look @@ -526,22 +541,24 @@ struct obd_device * class_devices_in_group(struct obd_uuid *grp_uuid, int *next) else return NULL; - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); + if (obd == NULL) continue; if (obd_uuid_equals(grp_uuid, &obd->obd_uuid)) { if (next != NULL) *next = i+1; - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return obd; } } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return NULL; } +EXPORT_SYMBOL(class_devices_in_group); /** * to notify sptlrpc log for \a fsname has changed, let every relevant OBD @@ -555,7 +572,7 @@ int class_notify_sptlrpc_conf(const char *fsname, int namelen) LASSERT(namelen > 0); - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { obd = class_num2obd(i); @@ -574,15 +591,15 @@ int class_notify_sptlrpc_conf(const char *fsname, int namelen) continue; class_incref(obd, __FUNCTION__, obd); - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); rc2 = obd_set_info_async(obd->obd_self_export, sizeof(KEY_SPTLRPC_CONF), KEY_SPTLRPC_CONF, 0, NULL, NULL); rc = rc ? rc : rc2; class_decref(obd, __FUNCTION__, obd); - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); return rc; } EXPORT_SYMBOL(class_notify_sptlrpc_conf); @@ -672,6 +689,7 @@ struct obd_export *class_conn2export(struct lustre_handle *conn) export = class_handle2object(conn->cookie); RETURN(export); } +EXPORT_SYMBOL(class_conn2export); struct obd_device *class_exp2obd(struct obd_export *exp) { @@ -679,6 +697,7 @@ struct obd_device *class_exp2obd(struct obd_export *exp) return exp->exp_obd; return NULL; } +EXPORT_SYMBOL(class_exp2obd); struct obd_device *class_conn2obd(struct lustre_handle *conn) { @@ -691,6 +710,7 @@ struct obd_device *class_conn2obd(struct lustre_handle *conn) } return NULL; } +EXPORT_SYMBOL(class_conn2obd); struct obd_import *class_exp2cliimp(struct obd_export *exp) { @@ -699,6 +719,7 @@ struct obd_import *class_exp2cliimp(struct obd_export *exp) return NULL; return obd->u.cli.cl_import; } +EXPORT_SYMBOL(class_exp2cliimp); struct obd_import *class_conn2cliimp(struct lustre_handle *conn) { @@ -707,6 +728,7 @@ struct obd_import *class_conn2cliimp(struct lustre_handle *conn) return NULL; return obd->u.cli.cl_import; } +EXPORT_SYMBOL(class_conn2cliimp); /* Export management functions */ static void class_export_destroy(struct obd_export *exp) @@ -1154,6 +1176,7 @@ no_disconn: class_export_put(export); RETURN(0); } +EXPORT_SYMBOL(class_disconnect); /* Return non-zero for a fully connected export */ int class_connected_export(struct obd_export *exp) diff --git a/lustre/obdclass/linux/linux-module.c b/lustre/obdclass/linux/linux-module.c index d893f16..f8b71955 100644 --- a/lustre/obdclass/linux/linux-module.c +++ b/lustre/obdclass/linux/linux-module.c @@ -290,7 +290,7 @@ static int obd_proc_read_health(char *page, char **start, off_t off, if (libcfs_catastrophe) rc += snprintf(page + rc, count - rc, "LBUG\n"); - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd; @@ -303,7 +303,7 @@ static int obd_proc_read_health(char *page, char **start, off_t off, continue; class_incref(obd, __FUNCTION__, cfs_current()); - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); if (obd_health_check(obd)) { rc += snprintf(page + rc, count - rc, @@ -311,9 +311,9 @@ static int obd_proc_read_health(char *page, char **start, off_t off, obd->obd_name); } class_decref(obd, __FUNCTION__, cfs_current()); - cfs_spin_lock(&obd_dev_lock); + cfs_read_lock(&obd_dev_lock); } - cfs_spin_unlock(&obd_dev_lock); + cfs_read_unlock(&obd_dev_lock); if (rc == 0) return snprintf(page, count, "healthy\n"); -- 1.8.3.1