LU-641 obdclass: use rwlock for obd_dev_lock

In target_handle_connect() it grabs obd_dev_lock to find obd_device
by target uuid. This causes lock contention if there are two many connection
requests on the same time. Based on obd_dev_lock is a read-mostly lock, it
is much reasonable to revise it as a rwlock.

Change-Id: I7f936d6182df9e7ed46b8a1b6fb259226188acbc
Signed-off-by: Jinshan Xiong <jay@whamcloud.com>
Reviewed-on: http://review.whamcloud.com/1294
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index 198b10a..df7cd84 100644 (file)
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -63,7 +63,7 @@
 
 /* OBD Device Declarations */
 extern struct obd_device *obd_devs[MAX_OBD_DEVICES];
-extern cfs_spinlock_t obd_dev_lock;
+extern cfs_rwlock_t obd_dev_lock;
 
 /* OBD Operations Declarations */
 extern struct obd_device *class_conn2obd(struct lustre_handle *);

diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c
index 42255a5..e344148 100644 (file)
--- a/lustre/obdclass/class_obd.c
+++ b/lustre/obdclass/class_obd.c
@@ -60,7 +60,7 @@ cfs_atomic_t libcfs_kmemory = {0};
 
 struct obd_device *obd_devs[MAX_OBD_DEVICES];
 cfs_list_t obd_types;
-cfs_spinlock_t obd_dev_lock = CFS_SPIN_LOCK_UNLOCKED;
+cfs_rwlock_t obd_dev_lock = CFS_RW_LOCK_UNLOCKED;
 
 #ifndef __KERNEL__
 __u64 obd_max_pages = 0;
@@ -368,24 +368,6 @@ EXPORT_SYMBOL(ptlrpc_put_connection_superhack);
 
 EXPORT_SYMBOL(proc_lustre_root);
 
-EXPORT_SYMBOL(class_register_type);
-EXPORT_SYMBOL(class_unregister_type);
-EXPORT_SYMBOL(class_get_type);
-EXPORT_SYMBOL(class_put_type);
-EXPORT_SYMBOL(class_name2dev);
-EXPORT_SYMBOL(class_name2obd);
-EXPORT_SYMBOL(class_uuid2dev);
-EXPORT_SYMBOL(class_uuid2obd);
-EXPORT_SYMBOL(class_find_client_obd);
-EXPORT_SYMBOL(class_devices_in_group);
-EXPORT_SYMBOL(class_conn2export);
-EXPORT_SYMBOL(class_exp2obd);
-EXPORT_SYMBOL(class_conn2obd);
-EXPORT_SYMBOL(class_exp2cliimp);
-EXPORT_SYMBOL(class_conn2cliimp);
-EXPORT_SYMBOL(class_disconnect);
-EXPORT_SYMBOL(class_num2obd);
-
 /* uuid.c */
 EXPORT_SYMBOL(class_uuid_unparse);
 EXPORT_SYMBOL(lustre_uuid_to_peer);
@@ -541,7 +523,6 @@ int init_obdclass(void)
         if (err)
                 return err;
 
-        cfs_spin_lock_init(&obd_dev_lock);
         CFS_INIT_LIST_HEAD(&obd_types);
 
         err = cfs_psdev_register(&obd_psdev);

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 55435b3..36d8b44 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -139,6 +139,7 @@ struct obd_type *class_get_type(const char *name)
         }
         return type;
 }
+EXPORT_SYMBOL(class_get_type);
 
 void class_put_type(struct obd_type *type)
 {
@@ -148,6 +149,7 @@ void class_put_type(struct obd_type *type)
         cfs_module_put(type->typ_dt_ops->o_owner);
         cfs_spin_unlock(&type->obd_type_lock);
 }
+EXPORT_SYMBOL(class_put_type);
 
 #define CLASS_MAX_NAME 1024
 
@@ -220,6 +222,7 @@ int class_register_type(struct obd_ops *dt_ops, struct md_ops *md_ops,
         OBD_FREE(type, sizeof(*type));
         RETURN(rc);
 }
+EXPORT_SYMBOL(class_register_type);
 
 int class_unregister_type(const char *name)
 {
@@ -258,6 +261,7 @@ int class_unregister_type(const char *name)
         OBD_FREE(type, sizeof(*type));
         RETURN(0);
 } /* class_unregister_type */
+EXPORT_SYMBOL(class_unregister_type);
 
 /**
  * Create a new obd device.
@@ -296,9 +300,10 @@ struct obd_device *class_newdev(const char *type_name, const char *name)
         }
         LASSERT(newdev->obd_magic == OBD_DEVICE_MAGIC);
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_write_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
+
                 if (obd && obd->obd_name &&
                     (strcmp(name, obd->obd_name) == 0)) {
                         CERROR("Device %s already exists, won't add\n", name);
@@ -326,7 +331,7 @@ struct obd_device *class_newdev(const char *type_name, const char *name)
                         obd_devs[i] = result;
                 }
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_write_unlock(&obd_dev_lock);
 
         if (result == NULL && i >= class_devno_max()) {
                 CERROR("all %u OBD devices used, increase MAX_OBD_DEVICES\n",
@@ -357,9 +362,9 @@ void class_release_dev(struct obd_device *obd)
         CDEBUG(D_INFO, "Release obd device %s obd_type name =%s\n",
                obd->obd_name,obd->obd_type->typ_name);
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_write_lock(&obd_dev_lock);
         obd_devs[obd->obd_minor] = NULL;
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_write_unlock(&obd_dev_lock);
         obd_device_free(obd);
 
         class_put_type(obd_type);
@@ -372,24 +377,26 @@ int class_name2dev(const char *name)
         if (!name)
                 return -1;
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
+
                 if (obd && obd->obd_name && strcmp(name, obd->obd_name) == 0) {
                         /* Make sure we finished attaching before we give
                            out any references */
                         LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC);
                         if (obd->obd_attached) {
-                                cfs_spin_unlock(&obd_dev_lock);
+                                cfs_read_unlock(&obd_dev_lock);
                                 return i;
                         }
                         break;
                 }
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
 
         return -1;
 }
+EXPORT_SYMBOL(class_name2dev);
 
 struct obd_device *class_name2obd(const char *name)
 {
@@ -399,24 +406,27 @@ struct obd_device *class_name2obd(const char *name)
                 return NULL;
         return class_num2obd(dev);
 }
+EXPORT_SYMBOL(class_name2obd);
 
 int class_uuid2dev(struct obd_uuid *uuid)
 {
         int i;
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
+
                 if (obd && obd_uuid_equals(uuid, &obd->obd_uuid)) {
                         LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC);
-                        cfs_spin_unlock(&obd_dev_lock);
+                        cfs_read_unlock(&obd_dev_lock);
                         return i;
                 }
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
 
         return -1;
 }
+EXPORT_SYMBOL(class_uuid2dev);
 
 struct obd_device *class_uuid2obd(struct obd_uuid *uuid)
 {
@@ -425,6 +435,7 @@ struct obd_device *class_uuid2obd(struct obd_uuid *uuid)
                 return NULL;
         return class_num2obd(dev);
 }
+EXPORT_SYMBOL(class_uuid2obd);
 
 /**
  * Get obd device from ::obd_devs[]
@@ -453,15 +464,17 @@ struct obd_device *class_num2obd(int num)
 
         return obd;
 }
+EXPORT_SYMBOL(class_num2obd);
 
 void class_obd_list(void)
 {
         char *status;
         int i;
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
+
                 if (obd == NULL)
                         continue;
                 if (obd->obd_stopping)
@@ -477,7 +490,7 @@ void class_obd_list(void)
                          obd->obd_name, obd->obd_uuid.uuid,
                          cfs_atomic_read(&obd->obd_refcount));
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
         return;
 }
 
@@ -490,9 +503,10 @@ struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid,
 {
         int i;
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
+
                 if (obd == NULL)
                         continue;
                 if ((strncmp(obd->obd_type->typ_name, typ_name,
@@ -501,15 +515,16 @@ struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid,
                                             &obd->u.cli.cl_target_uuid) &&
                             ((grp_uuid)? obd_uuid_equals(grp_uuid,
                                                          &obd->obd_uuid) : 1)) {
-                                cfs_spin_unlock(&obd_dev_lock);
+                                cfs_read_unlock(&obd_dev_lock);
                                 return obd;
                         }
                 }
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
 
         return NULL;
 }
+EXPORT_SYMBOL(class_find_client_obd);
 
 /* Iterate the obd_device list looking devices have grp_uuid. Start
    searching at *next, and if a device is found, the next index to look
@@ -526,22 +541,24 @@ struct obd_device * class_devices_in_group(struct obd_uuid *grp_uuid, int *next)
         else
                 return NULL;
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
+
                 if (obd == NULL)
                         continue;
                 if (obd_uuid_equals(grp_uuid, &obd->obd_uuid)) {
                         if (next != NULL)
                                 *next = i+1;
-                        cfs_spin_unlock(&obd_dev_lock);
+                        cfs_read_unlock(&obd_dev_lock);
                         return obd;
                 }
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
 
         return NULL;
 }
+EXPORT_SYMBOL(class_devices_in_group);
 
 /**
  * to notify sptlrpc log for \a fsname has changed, let every relevant OBD
@@ -555,7 +572,7 @@ int class_notify_sptlrpc_conf(const char *fsname, int namelen)
 
         LASSERT(namelen > 0);
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 obd = class_num2obd(i);
 
@@ -574,15 +591,15 @@ int class_notify_sptlrpc_conf(const char *fsname, int namelen)
                         continue;
 
                 class_incref(obd, __FUNCTION__, obd);
-                cfs_spin_unlock(&obd_dev_lock);
+                cfs_read_unlock(&obd_dev_lock);
                 rc2 = obd_set_info_async(obd->obd_self_export,
                                          sizeof(KEY_SPTLRPC_CONF),
                                          KEY_SPTLRPC_CONF, 0, NULL, NULL);
                 rc = rc ? rc : rc2;
                 class_decref(obd, __FUNCTION__, obd);
-                cfs_spin_lock(&obd_dev_lock);
+                cfs_read_lock(&obd_dev_lock);
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
         return rc;
 }
 EXPORT_SYMBOL(class_notify_sptlrpc_conf);
@@ -672,6 +689,7 @@ struct obd_export *class_conn2export(struct lustre_handle *conn)
         export = class_handle2object(conn->cookie);
         RETURN(export);
 }
+EXPORT_SYMBOL(class_conn2export);
 
 struct obd_device *class_exp2obd(struct obd_export *exp)
 {
@@ -679,6 +697,7 @@ struct obd_device *class_exp2obd(struct obd_export *exp)
                 return exp->exp_obd;
         return NULL;
 }
+EXPORT_SYMBOL(class_exp2obd);
 
 struct obd_device *class_conn2obd(struct lustre_handle *conn)
 {
@@ -691,6 +710,7 @@ struct obd_device *class_conn2obd(struct lustre_handle *conn)
         }
         return NULL;
 }
+EXPORT_SYMBOL(class_conn2obd);
 
 struct obd_import *class_exp2cliimp(struct obd_export *exp)
 {
@@ -699,6 +719,7 @@ struct obd_import *class_exp2cliimp(struct obd_export *exp)
                 return NULL;
         return obd->u.cli.cl_import;
 }
+EXPORT_SYMBOL(class_exp2cliimp);
 
 struct obd_import *class_conn2cliimp(struct lustre_handle *conn)
 {
@@ -707,6 +728,7 @@ struct obd_import *class_conn2cliimp(struct lustre_handle *conn)
                 return NULL;
         return obd->u.cli.cl_import;
 }
+EXPORT_SYMBOL(class_conn2cliimp);
 
 /* Export management functions */
 static void class_export_destroy(struct obd_export *exp)
@@ -1154,6 +1176,7 @@ no_disconn:
         class_export_put(export);
         RETURN(0);
 }
+EXPORT_SYMBOL(class_disconnect);
 
 /* Return non-zero for a fully connected export */
 int class_connected_export(struct obd_export *exp)

diff --git a/lustre/obdclass/linux/linux-module.c b/lustre/obdclass/linux/linux-module.c
index d893f16..f8b7195 100644 (file)
--- a/lustre/obdclass/linux/linux-module.c
+++ b/lustre/obdclass/linux/linux-module.c
@@ -290,7 +290,7 @@ static int obd_proc_read_health(char *page, char **start, off_t off,
         if (libcfs_catastrophe)
                 rc += snprintf(page + rc, count - rc, "LBUG\n");
 
-        cfs_spin_lock(&obd_dev_lock);
+        cfs_read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd;
 
@@ -303,7 +303,7 @@ static int obd_proc_read_health(char *page, char **start, off_t off,
                         continue;
 
                 class_incref(obd, __FUNCTION__, cfs_current());
-                cfs_spin_unlock(&obd_dev_lock);
+                cfs_read_unlock(&obd_dev_lock);
 
                 if (obd_health_check(obd)) {
                         rc += snprintf(page + rc, count - rc,
@@ -311,9 +311,9 @@ static int obd_proc_read_health(char *page, char **start, off_t off,
                                        obd->obd_name);
                 }
                 class_decref(obd, __FUNCTION__, cfs_current());
-                cfs_spin_lock(&obd_dev_lock);
+                cfs_read_lock(&obd_dev_lock);
         }
-        cfs_spin_unlock(&obd_dev_lock);
+        cfs_read_unlock(&obd_dev_lock);
 
         if (rc == 0)
                 return snprintf(page, count, "healthy\n");