Whamcloud - gitweb
LU-4134 obdclass: obd_device improvement 45/8045/30
authorAlexander Boyko <alexander.boyko@seagate.com>
Thu, 15 Jun 2017 14:25:58 +0000 (17:25 +0300)
committerOleg Drokin <oleg.drokin@intel.com>
Tue, 24 Oct 2017 07:17:53 +0000 (07:17 +0000)
The patch removes self exports from obd's reference counting which
allows to avoid freeing of self exports by zombie thread.
A pair of functions class_register_device()/class_unregister_device()
is to make sure that an obd can not be referenced again once its
refcount reached 0. For target_handle_connect() take a reference for
obd_device during finding it by name.

Fix grant mismatch message
"tot_granted 4194304 != fo_tot_granted".

Signed-off-by: Vladimir Saveliev <vladimir.saveliev@seagate.com>
Signed-off-by: Alexey Lyashkov <alexey.lyashkov@seagate.com>
Signed-off-by: Alexander Boyko <alexander.boyko@seagate.com>
Seagate-bug-id: MRP-2139 MRP-3267
Change-Id: I9cc6860431c6bb7db6983e0d15a5d3d2b564265e
Reviewed-on: https://review.whamcloud.com/8045
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
lustre/include/obd_class.h
lustre/ldlm/ldlm_lib.c
lustre/obdclass/genops.c
lustre/obdclass/obd_config.c
lustre/obdclass/obd_mount.c
lustre/target/tgt_grant.c

index 92d1273..88b3546 100644 (file)
@@ -68,9 +68,13 @@ int class_register_type(struct obd_ops *, struct md_ops *, bool enable_proc,
                        const char *nm, struct lu_device_type *ldt);
 int class_unregister_type(const char *nm);
 
-struct obd_device *class_newdev(const char *type_name, const char *name);
-void class_release_dev(struct obd_device *obd);
+struct obd_device *class_newdev(const char *type_name, const char *name,
+                               const char *uuid);
+int class_register_device(struct obd_device *obd);
+void class_unregister_device(struct obd_device *obd);
+void class_free_dev(struct obd_device *obd);
 
+struct obd_device *class_dev_by_str(const char *str);
 int class_name2dev(const char *name);
 struct obd_device *class_name2obd(const char *name);
 int class_uuid2dev(struct obd_uuid *uuid);
@@ -318,6 +322,8 @@ struct obd_export *class_export_get(struct obd_export *exp);
 void class_export_put(struct obd_export *exp);
 struct obd_export *class_new_export(struct obd_device *obddev,
                                     struct obd_uuid *cluuid);
+struct obd_export *class_new_export_self(struct obd_device *obd,
+                                        struct obd_uuid *uuid);
 void class_unlink_export(struct obd_export *exp);
 
 struct obd_import *class_import_get(struct obd_import *);
index 2f368bb..429bf4c 100644 (file)
@@ -951,7 +951,6 @@ int target_handle_connect(struct ptlrpc_request *req)
         * reconnect case */
        struct lustre_handle conn;
        struct lustre_handle *tmp;
-        struct obd_uuid tgtuuid;
         struct obd_uuid cluuid;
         char *str;
         int rc = 0;
@@ -960,7 +959,6 @@ int target_handle_connect(struct ptlrpc_request *req)
        bool     mds_conn = false, lw_client = false, initial_conn = false;
        bool     mds_mds_conn = false;
        bool     new_mds_mds_conn = false;
-       bool     target_referenced = false;
         struct obd_connect_data *data, *tmpdata;
         int size, tmpsize;
         lnet_nid_t *client_nid = NULL;
@@ -974,11 +972,7 @@ int target_handle_connect(struct ptlrpc_request *req)
                 GOTO(out, rc = -EINVAL);
         }
 
-        obd_str2uuid(&tgtuuid, str);
-        target = class_uuid2obd(&tgtuuid);
-        if (!target)
-                target = class_name2obd(str);
-
+       target = class_dev_by_str(str);
        if (!target) {
                deuuidify(str, NULL, &target_start, &target_len);
                LCONSOLE_ERROR_MSG(0x137, "%s: not available for connect "
@@ -990,6 +984,9 @@ int target_handle_connect(struct ptlrpc_request *req)
        }
 
        spin_lock(&target->obd_dev_lock);
+
+       target->obd_conn_inprogress++;
+
        if (target->obd_stopping || !target->obd_set_up) {
                spin_unlock(&target->obd_dev_lock);
 
@@ -1011,13 +1008,6 @@ int target_handle_connect(struct ptlrpc_request *req)
                GOTO(out, rc = -EAGAIN);
        }
 
-       /* Make sure the target isn't cleaned up while we're here. Yes,
-        * there's still a race between the above check and our incref here.
-        * Really, class_uuid2obd should take the ref. */
-       class_incref(target, __func__, current);
-       target_referenced = true;
-
-       target->obd_conn_inprogress++;
        spin_unlock(&target->obd_dev_lock);
 
         str = req_capsule_client_get(&req->rq_pill, &RMF_CLUUID);
@@ -1442,12 +1432,11 @@ out:
 
                class_export_put(export);
        }
-       if (target_referenced == true && target != NULL) {
+       if (target != NULL) {
                spin_lock(&target->obd_dev_lock);
                target->obd_conn_inprogress--;
                spin_unlock(&target->obd_dev_lock);
-
-               class_decref(target, __func__, current);
+               class_decref(target, "find", current);
        }
        req->rq_status = rc;
        RETURN(rc);
index a398ee5..175e391 100644 (file)
@@ -40,6 +40,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/kthread.h>
 #include <obd_class.h>
+#include <lustre_log.h>
 #include <lprocfs_status.h>
 #include <lustre_disk.h>
 #include <lustre_kernelcomm.h>
@@ -302,21 +303,20 @@ EXPORT_SYMBOL(class_unregister_type);
 /**
  * Create a new obd device.
  *
- * Find an empty slot in ::obd_devs[], create a new obd device in it.
+ * Allocate the new obd_device and initialize it.
  *
  * \param[in] type_name obd device type string.
  * \param[in] name      obd device name.
+ * \param[in] uuid      obd device UUID
  *
- * \retval NULL if create fails, otherwise return the obd device
- *         pointer created.
+ * \retval newdev         pointer to created obd_device
+ * \retval ERR_PTR(errno) on error
  */
-struct obd_device *class_newdev(const char *type_name, const char *name)
+struct obd_device *class_newdev(const char *type_name, const char *name,
+                               const char *uuid)
 {
-        struct obd_device *result = NULL;
         struct obd_device *newdev;
         struct obd_type *type = NULL;
-        int i;
-        int new_obd_minor = 0;
         ENTRY;
 
         if (strlen(name) >= MAX_OBD_NAME) {
@@ -331,93 +331,184 @@ struct obd_device *class_newdev(const char *type_name, const char *name)
         }
 
         newdev = obd_device_alloc();
-       if (newdev == NULL)
-               GOTO(out_type, result = ERR_PTR(-ENOMEM));
-
+       if (newdev == NULL) {
+               class_put_type(type);
+               RETURN(ERR_PTR(-ENOMEM));
+       }
         LASSERT(newdev->obd_magic == OBD_DEVICE_MAGIC);
+       strncpy(newdev->obd_name, name, sizeof(newdev->obd_name) - 1);
+       newdev->obd_type = type;
+       newdev->obd_minor = -1;
+
+       rwlock_init(&newdev->obd_pool_lock);
+       newdev->obd_pool_limit = 0;
+       newdev->obd_pool_slv = 0;
+
+       INIT_LIST_HEAD(&newdev->obd_exports);
+       INIT_LIST_HEAD(&newdev->obd_unlinked_exports);
+       INIT_LIST_HEAD(&newdev->obd_delayed_exports);
+       INIT_LIST_HEAD(&newdev->obd_exports_timed);
+       INIT_LIST_HEAD(&newdev->obd_nid_stats);
+       spin_lock_init(&newdev->obd_nid_lock);
+       spin_lock_init(&newdev->obd_dev_lock);
+       mutex_init(&newdev->obd_dev_mutex);
+       spin_lock_init(&newdev->obd_osfs_lock);
+       /* newdev->obd_osfs_age must be set to a value in the distant
+        * past to guarantee a fresh statfs is fetched on mount. */
+       newdev->obd_osfs_age = cfs_time_shift_64(-1000);
+
+       /* XXX belongs in setup not attach  */
+       init_rwsem(&newdev->obd_observer_link_sem);
+       /* recovery data */
+       init_timer(&newdev->obd_recovery_timer);
+       spin_lock_init(&newdev->obd_recovery_task_lock);
+       init_waitqueue_head(&newdev->obd_next_transno_waitq);
+       init_waitqueue_head(&newdev->obd_evict_inprogress_waitq);
+       INIT_LIST_HEAD(&newdev->obd_req_replay_queue);
+       INIT_LIST_HEAD(&newdev->obd_lock_replay_queue);
+       INIT_LIST_HEAD(&newdev->obd_final_req_queue);
+       INIT_LIST_HEAD(&newdev->obd_evict_list);
+       INIT_LIST_HEAD(&newdev->obd_lwp_list);
+
+       llog_group_init(&newdev->obd_olg);
+       /* Detach drops this */
+       atomic_set(&newdev->obd_refcount, 1);
+       lu_ref_init(&newdev->obd_reference);
+       lu_ref_add(&newdev->obd_reference, "newdev", newdev);
+
+       newdev->obd_conn_inprogress = 0;
+
+       strncpy(newdev->obd_uuid.uuid, uuid, strlen(uuid));
+
+       CDEBUG(D_IOCTL, "Allocate new device %s (%p)\n",
+              newdev->obd_name, newdev);
+
+       return newdev;
+}
 
-       write_lock(&obd_dev_lock);
-        for (i = 0; i < class_devno_max(); i++) {
-                struct obd_device *obd = class_num2obd(i);
+/**
+ * Free obd device.
+ *
+ * \param[in] obd obd_device to be freed
+ *
+ * \retval none
+ */
+void class_free_dev(struct obd_device *obd)
+{
+       struct obd_type *obd_type = obd->obd_type;
 
-               if (obd && (strcmp(name, obd->obd_name) == 0)) {
-                        CERROR("Device %s already exists at %d, won't add\n",
-                               name, i);
-                        if (result) {
-                                LASSERTF(result->obd_magic == OBD_DEVICE_MAGIC,
-                                         "%p obd_magic %08x != %08x\n", result,
-                                         result->obd_magic, OBD_DEVICE_MAGIC);
-                                LASSERTF(result->obd_minor == new_obd_minor,
-                                         "%p obd_minor %d != %d\n", result,
-                                         result->obd_minor, new_obd_minor);
-
-                                obd_devs[result->obd_minor] = NULL;
-                                result->obd_name[0]='\0';
-                         }
-                        result = ERR_PTR(-EEXIST);
-                        break;
-                }
-                if (!result && !obd) {
-                        result = newdev;
-                        result->obd_minor = i;
-                        new_obd_minor = i;
-                        result->obd_type = type;
-                        strncpy(result->obd_name, name,
-                                sizeof(result->obd_name) - 1);
-                        obd_devs[i] = result;
-                }
-        }
-       write_unlock(&obd_dev_lock);
+       LASSERTF(obd->obd_magic == OBD_DEVICE_MAGIC, "%p obd_magic %08x "
+                "!= %08x\n", obd, obd->obd_magic, OBD_DEVICE_MAGIC);
+       LASSERTF(obd->obd_minor == -1 || obd_devs[obd->obd_minor] == obd,
+                "obd %p != obd_devs[%d] %p\n",
+                obd, obd->obd_minor, obd_devs[obd->obd_minor]);
+       LASSERTF(atomic_read(&obd->obd_refcount) == 0,
+                "obd_refcount should be 0, not %d\n",
+                atomic_read(&obd->obd_refcount));
+       LASSERT(obd_type != NULL);
 
-        if (result == NULL && i >= class_devno_max()) {
-                CERROR("all %u OBD devices used, increase MAX_OBD_DEVICES\n",
-                       class_devno_max());
-               GOTO(out, result = ERR_PTR(-EOVERFLOW));
-        }
+       CDEBUG(D_INFO, "Release obd device %s obd_type name = %s\n",
+              obd->obd_name, obd->obd_type->typ_name);
 
-       if (IS_ERR(result))
-               GOTO(out, result);
+       CDEBUG(D_CONFIG, "finishing cleanup of obd %s (%s)\n",
+                        obd->obd_name, obd->obd_uuid.uuid);
+       if (obd->obd_stopping) {
+               int err;
 
-       CDEBUG(D_IOCTL, "Adding new device %s (%p)\n",
-              result->obd_name, result);
+               /* If we're not stopping, we were never set up */
+               err = obd_cleanup(obd);
+               if (err)
+                       CERROR("Cleanup %s returned %d\n",
+                               obd->obd_name, err);
+       }
 
-       RETURN(result);
-out:
-       obd_device_free(newdev);
-out_type:
-       class_put_type(type);
-       return result;
+       obd_device_free(obd);
+
+       class_put_type(obd_type);
 }
 
-void class_release_dev(struct obd_device *obd)
+/**
+ * Unregister obd device.
+ *
+ * Free slot in obd_dev[] used by \a obd.
+ *
+ * \param[in] new_obd obd_device to be unregistered
+ *
+ * \retval none
+ */
+void class_unregister_device(struct obd_device *obd)
 {
-        struct obd_type *obd_type = obd->obd_type;
-
-        LASSERTF(obd->obd_magic == OBD_DEVICE_MAGIC, "%p obd_magic %08x != %08x\n",
-                 obd, obd->obd_magic, OBD_DEVICE_MAGIC);
-        LASSERTF(obd == obd_devs[obd->obd_minor], "obd %p != obd_devs[%d] %p\n",
-                 obd, obd->obd_minor, obd_devs[obd->obd_minor]);
-        LASSERT(obd_type != NULL);
+       write_lock(&obd_dev_lock);
+       if (obd->obd_minor >= 0) {
+               LASSERT(obd_devs[obd->obd_minor] == obd);
+               obd_devs[obd->obd_minor] = NULL;
+               obd->obd_minor = -1;
+       }
+       write_unlock(&obd_dev_lock);
+}
 
-        CDEBUG(D_INFO, "Release obd device %s at %d obd_type name =%s\n",
-               obd->obd_name, obd->obd_minor, obd->obd_type->typ_name);
+/**
+ * Register obd device.
+ *
+ * Find free slot in obd_devs[], fills it with \a new_obd.
+ *
+ * \param[in] new_obd obd_device to be registered
+ *
+ * \retval 0          success
+ * \retval -EEXIST    device with this name is registered
+ * \retval -EOVERFLOW obd_devs[] is full
+ */
+int class_register_device(struct obd_device *new_obd)
+{
+       int ret = 0;
+       int i;
+       int new_obd_minor = 0;
+       bool minor_assign = false;
 
        write_lock(&obd_dev_lock);
-        obd_devs[obd->obd_minor] = NULL;
+       for (i = 0; i < class_devno_max(); i++) {
+               struct obd_device *obd = class_num2obd(i);
+
+               if (obd != NULL &&
+                   (strcmp(new_obd->obd_name, obd->obd_name) == 0)) {
+                       CERROR("%s: already exists, won't add\n",
+                              obd->obd_name);
+                       /* in case we found a free slot before duplicate */
+                       minor_assign = false;
+                       ret = -EEXIST;
+                       break;
+               }
+               if (!minor_assign && obd == NULL) {
+                       new_obd_minor = i;
+                       minor_assign = true;
+               }
+       }
+
+       if (minor_assign) {
+               new_obd->obd_minor = new_obd_minor;
+               LASSERTF(obd_devs[new_obd_minor] == NULL, "obd_devs[%d] "
+                        "%p\n", new_obd_minor, obd_devs[new_obd_minor]);
+               obd_devs[new_obd_minor] = new_obd;
+       } else {
+               if (ret == 0) {
+                       ret = -EOVERFLOW;
+                       CERROR("%s: all %u/%u devices used, increase "
+                              "MAX_OBD_DEVICES: rc = %d\n", new_obd->obd_name,
+                              i, class_devno_max(), ret);
+               }
+       }
        write_unlock(&obd_dev_lock);
-        obd_device_free(obd);
 
-        class_put_type(obd_type);
+       RETURN(ret);
 }
 
-int class_name2dev(const char *name)
+static int class_name2dev_nolock(const char *name)
 {
         int i;
 
         if (!name)
                 return -1;
 
-       read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
 
@@ -426,17 +517,30 @@ int class_name2dev(const char *name)
                            out any references */
                         LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC);
                         if (obd->obd_attached) {
-                               read_unlock(&obd_dev_lock);
                                 return i;
                         }
                         break;
                 }
         }
-       read_unlock(&obd_dev_lock);
 
         return -1;
 }
 
+int class_name2dev(const char *name)
+{
+       int i;
+
+       if (!name)
+               return -1;
+
+       read_lock(&obd_dev_lock);
+       i = class_name2dev_nolock(name);
+       read_unlock(&obd_dev_lock);
+
+       return i;
+}
+EXPORT_SYMBOL(class_name2dev);
+
 struct obd_device *class_name2obd(const char *name)
 {
         int dev = class_name2dev(name);
@@ -447,25 +551,34 @@ struct obd_device *class_name2obd(const char *name)
 }
 EXPORT_SYMBOL(class_name2obd);
 
-int class_uuid2dev(struct obd_uuid *uuid)
+int class_uuid2dev_nolock(struct obd_uuid *uuid)
 {
         int i;
 
-       read_lock(&obd_dev_lock);
         for (i = 0; i < class_devno_max(); i++) {
                 struct obd_device *obd = class_num2obd(i);
 
                 if (obd && obd_uuid_equals(uuid, &obd->obd_uuid)) {
                         LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC);
-                       read_unlock(&obd_dev_lock);
                         return i;
                 }
         }
-       read_unlock(&obd_dev_lock);
 
         return -1;
 }
 
+int class_uuid2dev(struct obd_uuid *uuid)
+{
+       int i;
+
+       read_lock(&obd_dev_lock);
+       i = class_uuid2dev_nolock(uuid);
+       read_unlock(&obd_dev_lock);
+
+       return i;
+}
+EXPORT_SYMBOL(class_uuid2dev);
+
 struct obd_device *class_uuid2obd(struct obd_uuid *uuid)
 {
         int dev = class_uuid2dev(uuid);
@@ -504,6 +617,40 @@ struct obd_device *class_num2obd(int num)
 }
 
 /**
+ * Find obd in obd_dev[] by name or uuid.
+ *
+ * Increment obd's refcount if found.
+ *
+ * \param[in] str obd name or uuid
+ *
+ * \retval NULL    if not found
+ * \retval target  pointer to found obd_device
+ */
+struct obd_device *class_dev_by_str(const char *str)
+{
+       struct obd_device *target = NULL;
+       struct obd_uuid tgtuuid;
+       int rc;
+
+       obd_str2uuid(&tgtuuid, str);
+
+       read_lock(&obd_dev_lock);
+       rc = class_uuid2dev_nolock(&tgtuuid);
+       if (rc < 0)
+               rc = class_name2dev_nolock(str);
+
+       if (rc >= 0)
+               target = class_num2obd(rc);
+
+       if (target != NULL)
+               class_incref(target, "find", current);
+       read_unlock(&obd_dev_lock);
+
+       RETURN(target);
+}
+EXPORT_SYMBOL(class_dev_by_str);
+
+/**
  * Get obd devices count. Device in any
  *    state are counted
  * \retval obd device count
@@ -795,7 +942,10 @@ static void class_export_destroy(struct obd_export *exp)
        LASSERT(list_empty(&exp->exp_req_replay_queue));
        LASSERT(list_empty(&exp->exp_hp_rpcs));
         obd_destroy_export(exp);
-        class_decref(obd, "export", exp);
+       /* self export doesn't hold a reference to an obd, although it
+        * exists until freeing of the obd */
+       if (exp != obd->obd_self_export)
+               class_decref(obd, "export", exp);
 
         OBD_FREE_RCU(exp, sizeof(*exp), &exp->exp_handle);
         EXIT;
@@ -828,24 +978,37 @@ void class_export_put(struct obd_export *exp)
               atomic_read(&exp->exp_refcount) - 1);
 
        if (atomic_dec_and_test(&exp->exp_refcount)) {
-               LASSERT(!list_empty(&exp->exp_obd_chain));
-               LASSERT(list_empty(&exp->exp_stale_list));
+               struct obd_device *obd = exp->exp_obd;
+
                CDEBUG(D_IOCTL, "final put %p/%s\n",
                       exp, exp->exp_client_uuid.uuid);
 
                /* release nid stat refererence */
                lprocfs_exp_cleanup(exp);
 
-               obd_zombie_export_add(exp);
+               if (exp == obd->obd_self_export) {
+                       /* self export should be destroyed without
+                        * zombie thread as it doesn't hold a
+                        * reference to obd and doesn't hold any
+                        * resources */
+                       class_export_destroy(exp);
+                       /* self export is destroyed, no class
+                        * references exist and it is safe to free
+                        * obd */
+                       class_free_dev(obd);
+               } else {
+                       LASSERT(!list_empty(&exp->exp_obd_chain));
+                       obd_zombie_export_add(exp);
+               }
+
        }
 }
 EXPORT_SYMBOL(class_export_put);
-
 /* Creates a new export, adds it to the hash table, and returns a
  * pointer to it. The refcount is 2: one for the hash reference, and
  * one for the pointer returned by this function. */
-struct obd_export *class_new_export(struct obd_device *obd,
-                                    struct obd_uuid *cluuid)
+struct obd_export *__class_new_export(struct obd_device *obd,
+                                     struct obd_uuid *cluuid, bool is_self)
 {
         struct obd_export *export;
        struct cfs_hash *hash = NULL;
@@ -859,6 +1022,7 @@ struct obd_export *class_new_export(struct obd_device *obd,
         export->exp_conn_cnt = 0;
         export->exp_lock_hash = NULL;
        export->exp_flock_hash = NULL;
+       /* 2 = class_handle_hash + last */
        atomic_set(&export->exp_refcount, 2);
        atomic_set(&export->exp_rpc_count, 0);
        atomic_set(&export->exp_cb_count, 0);
@@ -892,17 +1056,17 @@ struct obd_export *class_new_export(struct obd_device *obd,
        export->exp_client_uuid = *cluuid;
        obd_init_export(export);
 
-       spin_lock(&obd->obd_dev_lock);
-       /* shouldn't happen, but might race */
-       if (obd->obd_stopping)
-               GOTO(exit_unlock, rc = -ENODEV);
+       if (!obd_uuid_equals(cluuid, &obd->obd_uuid)) {
+               spin_lock(&obd->obd_dev_lock);
+               /* shouldn't happen, but might race */
+               if (obd->obd_stopping)
+                       GOTO(exit_unlock, rc = -ENODEV);
 
-       hash = cfs_hash_getref(obd->obd_uuid_hash);
-       if (hash == NULL)
-               GOTO(exit_unlock, rc = -ENODEV);
-       spin_unlock(&obd->obd_dev_lock);
+               hash = cfs_hash_getref(obd->obd_uuid_hash);
+               if (hash == NULL)
+                       GOTO(exit_unlock, rc = -ENODEV);
+               spin_unlock(&obd->obd_dev_lock);
 
-        if (!obd_uuid_equals(cluuid, &obd->obd_uuid)) {
                 rc = cfs_hash_add_unique(hash, cluuid, &export->exp_uuid_hash);
                 if (rc != 0) {
                         LCONSOLE_WARN("%s: denying duplicate export for %s, %d\n",
@@ -914,17 +1078,24 @@ struct obd_export *class_new_export(struct obd_device *obd,
        at_init(&export->exp_bl_lock_at, obd_timeout, 0);
        spin_lock(&obd->obd_dev_lock);
         if (obd->obd_stopping) {
-                cfs_hash_del(hash, cluuid, &export->exp_uuid_hash);
-                GOTO(exit_unlock, rc = -ENODEV);
+               if (hash)
+                       cfs_hash_del(hash, cluuid, &export->exp_uuid_hash);
+               GOTO(exit_unlock, rc = -ESHUTDOWN);
         }
 
-        class_incref(obd, "export", export);
-       list_add(&export->exp_obd_chain, &export->exp_obd->obd_exports);
-       list_add_tail(&export->exp_obd_chain_timed,
-                     &export->exp_obd->obd_exports_timed);
-        export->exp_obd->obd_num_exports++;
+       if (!is_self) {
+               class_incref(obd, "export", export);
+               list_add_tail(&export->exp_obd_chain_timed,
+                             &obd->obd_exports_timed);
+               list_add(&export->exp_obd_chain, &obd->obd_exports);
+               obd->obd_num_exports++;
+       } else {
+               INIT_LIST_HEAD(&export->exp_obd_chain_timed);
+               INIT_LIST_HEAD(&export->exp_obd_chain);
+       }
        spin_unlock(&obd->obd_dev_lock);
-       cfs_hash_putref(hash);
+       if (hash)
+               cfs_hash_putref(hash);
        RETURN(export);
 
 exit_unlock:
@@ -938,12 +1109,29 @@ exit_err:
         OBD_FREE_PTR(export);
         return ERR_PTR(rc);
 }
+
+struct obd_export *class_new_export(struct obd_device *obd,
+                                   struct obd_uuid *uuid)
+{
+       return __class_new_export(obd, uuid, false);
+}
 EXPORT_SYMBOL(class_new_export);
 
+struct obd_export *class_new_export_self(struct obd_device *obd,
+                                        struct obd_uuid *uuid)
+{
+       return __class_new_export(obd, uuid, true);
+}
+
 void class_unlink_export(struct obd_export *exp)
 {
        class_handle_unhash(&exp->exp_handle);
 
+       if (exp->exp_obd->obd_self_export == exp) {
+               class_export_put(exp);
+               return;
+       }
+
        spin_lock(&exp->exp_obd->obd_dev_lock);
        /* delete an uuid-export hashitem from hashtables */
        if (!hlist_unhashed(&exp->exp_uuid_hash))
index bcb9375..81754de 100644 (file)
@@ -365,6 +365,7 @@ EXPORT_SYMBOL(lustre_cfg_string);
  */
 int class_attach(struct lustre_cfg *lcfg)
 {
+       struct obd_export *exp;
         struct obd_device *obd = NULL;
         char *typename, *name, *uuid;
         int rc, len;
@@ -381,24 +382,26 @@ int class_attach(struct lustre_cfg *lcfg)
                 RETURN(-EINVAL);
         }
         name = lustre_cfg_string(lcfg, 0);
-
         if (!LUSTRE_CFG_BUFLEN(lcfg, 2)) {
                 CERROR("No UUID passed!\n");
                 RETURN(-EINVAL);
         }
-        uuid = lustre_cfg_string(lcfg, 2);
 
-        CDEBUG(D_IOCTL, "attach type %s name: %s uuid: %s\n",
-               MKSTR(typename), MKSTR(name), MKSTR(uuid));
+       uuid = lustre_cfg_string(lcfg, 2);
+       len = strlen(uuid);
+       if (len >= sizeof(obd->obd_uuid)) {
+               CERROR("%s: uuid must be < %d bytes long\n",
+                      name, (int)sizeof(obd->obd_uuid));
+               RETURN(-EINVAL);
+       }
 
-        obd = class_newdev(typename, name);
+       obd = class_newdev(typename, name, uuid);
         if (IS_ERR(obd)) {
                 /* Already exists or out of obds */
                 rc = PTR_ERR(obd);
-                obd = NULL;
                 CERROR("Cannot create device %s of type %s : %d\n",
                        name, typename, rc);
-                GOTO(out, rc);
+               RETURN(rc);
         }
         LASSERTF(obd != NULL, "Cannot get obd device %s of type %s\n",
                  name, typename);
@@ -408,64 +411,30 @@ int class_attach(struct lustre_cfg *lcfg)
         LASSERTF(strncmp(obd->obd_name, name, strlen(name)) == 0,
                  "%p obd_name %s != %s\n", obd, obd->obd_name, name);
 
-       rwlock_init(&obd->obd_pool_lock);
-       obd->obd_pool_limit = 0;
-       obd->obd_pool_slv = 0;
-
-       INIT_LIST_HEAD(&obd->obd_exports);
-       INIT_LIST_HEAD(&obd->obd_unlinked_exports);
-       INIT_LIST_HEAD(&obd->obd_delayed_exports);
-       INIT_LIST_HEAD(&obd->obd_exports_timed);
-       INIT_LIST_HEAD(&obd->obd_nid_stats);
-       spin_lock_init(&obd->obd_nid_lock);
-       spin_lock_init(&obd->obd_dev_lock);
-       mutex_init(&obd->obd_dev_mutex);
-       spin_lock_init(&obd->obd_osfs_lock);
-       /* obd->obd_osfs_age must be set to a value in the distant
-        * past to guarantee a fresh statfs is fetched on mount. */
-       obd->obd_osfs_age = cfs_time_shift_64(-1000);
-
-       /* XXX belongs in setup not attach  */
-       init_rwsem(&obd->obd_observer_link_sem);
-       /* recovery data */
-       init_timer(&obd->obd_recovery_timer);
-       spin_lock_init(&obd->obd_recovery_task_lock);
-       init_waitqueue_head(&obd->obd_next_transno_waitq);
-       init_waitqueue_head(&obd->obd_evict_inprogress_waitq);
-       INIT_LIST_HEAD(&obd->obd_req_replay_queue);
-       INIT_LIST_HEAD(&obd->obd_lock_replay_queue);
-       INIT_LIST_HEAD(&obd->obd_final_req_queue);
-       INIT_LIST_HEAD(&obd->obd_evict_list);
-       INIT_LIST_HEAD(&obd->obd_lwp_list);
-
-       llog_group_init(&obd->obd_olg);
-
-       obd->obd_conn_inprogress = 0;
-
-        len = strlen(uuid);
-        if (len >= sizeof(obd->obd_uuid)) {
-                CERROR("uuid must be < %d bytes long\n",
-                       (int)sizeof(obd->obd_uuid));
-                GOTO(out, rc = -EINVAL);
-        }
-        memcpy(obd->obd_uuid.uuid, uuid, len);
+       exp = class_new_export_self(obd, &obd->obd_uuid);
+       if (IS_ERR(exp)) {
+               /* force free */
+               GOTO(out, rc = PTR_ERR(exp));
+               RETURN(PTR_ERR(exp));
+       }
 
-        /* Detach drops this */
-       spin_lock(&obd->obd_dev_lock);
-       atomic_set(&obd->obd_refcount, 1);
-       spin_unlock(&obd->obd_dev_lock);
-        lu_ref_init(&obd->obd_reference);
-        lu_ref_add(&obd->obd_reference, "attach", obd);
+       obd->obd_self_export = exp;
+       list_del_init(&exp->exp_obd_chain_timed);
+       class_export_put(exp);
+
+       rc = class_register_device(obd);
+       if (rc != 0)
+               GOTO(out, rc);
 
-        obd->obd_attached = 1;
-        CDEBUG(D_IOCTL, "OBD: dev %d attached type %s with refcount %d\n",
+       obd->obd_attached = 1;
+       CDEBUG(D_IOCTL, "OBD: dev %d attached type %s with refcount %d\n",
               obd->obd_minor, typename, atomic_read(&obd->obd_refcount));
-        RETURN(0);
- out:
-        if (obd != NULL) {
-                class_release_dev(obd);
-        }
-        return rc;
+       RETURN(0);
+out:
+       class_decref(obd, "newdev", obd);
+       class_free_dev(obd);
+
+       RETURN(rc);
 }
 EXPORT_SYMBOL(class_attach);
 
@@ -475,7 +444,6 @@ EXPORT_SYMBOL(class_attach);
 int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
 {
         int err = 0;
-        struct obd_export *exp;
         ENTRY;
 
         LASSERT(obd != NULL);
@@ -524,7 +492,7 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
                                              CFS_HASH_MAX_THETA,
                                              &uuid_hash_ops, CFS_HASH_DEFAULT);
         if (!obd->obd_uuid_hash)
-                GOTO(err_hash, err = -ENOMEM);
+               GOTO(err_exit, err = -ENOMEM);
 
         /* create a nid-export lustre hash */
         obd->obd_nid_hash = cfs_hash_create("NID_HASH",
@@ -535,7 +503,7 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
                                             CFS_HASH_MAX_THETA,
                                             &nid_hash_ops, CFS_HASH_DEFAULT);
         if (!obd->obd_nid_hash)
-                GOTO(err_hash, err = -ENOMEM);
+               GOTO(err_exit, err = -ENOMEM);
 
         /* create a nid-stats lustre hash */
         obd->obd_nid_stats_hash = cfs_hash_create("NID_STATS",
@@ -545,8 +513,8 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
                                                   CFS_HASH_MIN_THETA,
                                                   CFS_HASH_MAX_THETA,
                                                   &nid_stat_hash_ops, CFS_HASH_DEFAULT);
-        if (!obd->obd_nid_stats_hash)
-                GOTO(err_hash, err = -ENOMEM);
+       if (!obd->obd_nid_stats_hash)
+               GOTO(err_exit, err = -ENOMEM);
 
        /* create a client_generation-export lustre hash */
        obd->obd_gen_hash = cfs_hash_create("UUID_HASH",
@@ -557,21 +525,13 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
                                            CFS_HASH_MAX_THETA,
                                            &gen_hash_ops, CFS_HASH_DEFAULT);
        if (!obd->obd_gen_hash)
-               GOTO(err_hash, err = -ENOMEM);
-
-        exp = class_new_export(obd, &obd->obd_uuid);
-        if (IS_ERR(exp))
-                GOTO(err_hash, err = PTR_ERR(exp));
-
-        obd->obd_self_export = exp;
-       list_del_init(&exp->exp_obd_chain_timed);
-        class_export_put(exp);
+               GOTO(err_exit, err = -ENOMEM);
 
-        err = obd_setup(obd, lcfg);
-        if (err)
-                GOTO(err_exp, err);
+       err = obd_setup(obd, lcfg);
+       if (err)
+               GOTO(err_exit, err);
 
-        obd->obd_set_up = 1;
+       obd->obd_set_up = 1;
 
        spin_lock(&obd->obd_dev_lock);
        /* cleanup drops this */
@@ -582,12 +542,7 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
                obd->obd_name, obd->obd_uuid.uuid);
 
         RETURN(0);
-err_exp:
-        if (obd->obd_self_export) {
-                class_unlink_export(obd->obd_self_export);
-                obd->obd_self_export = NULL;
-        }
-err_hash:
+err_exit:
         if (obd->obd_uuid_hash) {
                 cfs_hash_putref(obd->obd_uuid_hash);
                 obd->obd_uuid_hash = NULL;
@@ -631,10 +586,14 @@ int class_detach(struct obd_device *obd, struct lustre_cfg *lcfg)
        obd->obd_attached = 0;
        spin_unlock(&obd->obd_dev_lock);
 
+       /* cleanup in progress. we don't like to find this device after now */
+       class_unregister_device(obd);
+
         CDEBUG(D_IOCTL, "detach on obd %s (uuid %s)\n",
                obd->obd_name, obd->obd_uuid.uuid);
 
-        class_decref(obd, "attach", obd);
+       class_decref(obd, "newdev", obd);
+
         RETURN(0);
 }
 EXPORT_SYMBOL(class_detach);
@@ -664,6 +623,9 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
        }
        /* Leave this on forever */
        obd->obd_stopping = 1;
+       /* function can't return error after that point, so clear setup flag
+        * as early as possible to avoid finding via obd_devs / hash */
+       obd->obd_set_up = 0;
        spin_unlock(&obd->obd_dev_lock);
 
        /* wait for already-arrived-connections to finish. */
@@ -696,17 +658,11 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
 
        LASSERT(obd->obd_self_export);
 
-       /* The three references that should be remaining are the
-        * obd_self_export and the attach and setup references. */
-       if (atomic_read(&obd->obd_refcount) > 3) {
-               /* refcounf - 3 might be the number of real exports
-                  (excluding self export). But class_incref is called
-                  by other things as well, so don't count on it. */
-               CDEBUG(D_IOCTL, "%s: forcing exports to disconnect: %d\n",
-                      obd->obd_name, atomic_read(&obd->obd_refcount) - 3);
-               dump_exports(obd, 0, D_HA);
-               class_disconnect_exports(obd);
-       }
+       CDEBUG(D_IOCTL, "%s: forcing exports to disconnect: %d/%d\n",
+              obd->obd_name, obd->obd_num_exports,
+              atomic_read(&obd->obd_refcount) - 2);
+       dump_exports(obd, 0, D_HA);
+       class_disconnect_exports(obd);
 
        /* Precleanup, we must make sure all exports get destroyed. */
        err = obd_precleanup(obd);
@@ -758,43 +714,31 @@ EXPORT_SYMBOL(class_incref);
 
 void class_decref(struct obd_device *obd, const char *scope, const void *source)
 {
-       int err;
-       int refs;
+       int last;
 
-       spin_lock(&obd->obd_dev_lock);
-       atomic_dec(&obd->obd_refcount);
-       refs = atomic_read(&obd->obd_refcount);
-       spin_unlock(&obd->obd_dev_lock);
+       CDEBUG(D_INFO, "Decref %s (%p) now %d - %s\n", obd->obd_name, obd,
+              atomic_read(&obd->obd_refcount), scope);
+
+       LASSERT(obd->obd_num_exports >= 0);
+       last = atomic_dec_and_test(&obd->obd_refcount);
        lu_ref_del(&obd->obd_reference, scope, source);
 
-       CDEBUG(D_INFO, "Decref %s (%p) now %d\n", obd->obd_name, obd, refs);
+       if (last) {
+               struct obd_export *exp;
 
-       if ((refs == 1) && obd->obd_stopping) {
+               LASSERT(!obd->obd_attached);
                /* All exports have been destroyed; there should
-                  be no more in-progress ops by this point.*/
-
-               spin_lock(&obd->obd_self_export->exp_lock);
-               obd->obd_self_export->exp_flags |= exp_flags_from_obd(obd);
-               spin_unlock(&obd->obd_self_export->exp_lock);
+                * be no more in-progress ops by this point.*/
+               exp = obd->obd_self_export;
 
-                /* note that we'll recurse into class_decref again */
-                class_unlink_export(obd->obd_self_export);
-                return;
-        }
-
-        if (refs == 0) {
-                CDEBUG(D_CONFIG, "finishing cleanup of obd %s (%s)\n",
-                       obd->obd_name, obd->obd_uuid.uuid);
-                LASSERT(!obd->obd_attached);
-                if (obd->obd_stopping) {
-                        /* If we're not stopping, we were never set up */
-                        err = obd_cleanup(obd);
-                        if (err)
-                                CERROR("Cleanup %s returned %d\n",
-                                       obd->obd_name, err);
+               if (exp) {
+                       exp->exp_flags |= exp_flags_from_obd(obd);
+                       /*
+                        * note that we'll recurse into class_decref again
+                        * but it's not a problem because we was last user
+                        */
+                       class_unlink_export(exp);
                 }
-
-                class_release_dev(obd);
         }
 }
 EXPORT_SYMBOL(class_decref);
index 362d8c1..b8d3b76 100644 (file)
@@ -219,7 +219,7 @@ int lustre_start_mgc(struct super_block *sb)
         struct lustre_sb_info *lsi = s2lsi(sb);
         struct obd_device *obd;
         struct obd_export *exp;
-        struct obd_uuid *uuid;
+       struct obd_uuid *uuid = NULL;
         class_uuid_t uuidc;
         lnet_nid_t nid;
        char nidstr[LNET_NIDSTR_SIZE];
@@ -408,7 +408,6 @@ int lustre_start_mgc(struct super_block *sb)
         rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME,
                                 (char *)uuid->uuid, LUSTRE_MGS_OBDNAME,
                                 niduuid, NULL, NULL);
-        OBD_FREE_PTR(uuid);
         if (rc)
                 GOTO(out_free, rc);
 
@@ -469,7 +468,7 @@ int lustre_start_mgc(struct super_block *sb)
             lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)
                 data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV;
         data->ocd_version = LUSTRE_VERSION_CODE;
-        rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL);
+       rc = obd_connect(NULL, &exp, obd, uuid, data, NULL);
         if (rc) {
                 CERROR("connect failed %d\n", rc);
                 GOTO(out, rc);
@@ -484,6 +483,8 @@ out:
 out_free:
        mutex_unlock(&mgc_start_lock);
 
+       if (uuid)
+               OBD_FREE_PTR(uuid);
         if (data)
                 OBD_FREE_PTR(data);
         if (mgcname)
index 1caad7c..7887623 100644 (file)
@@ -138,11 +138,6 @@ static int tgt_check_export_grants(struct obd_export *exp, u64 *dirty,
        struct tg_export_data *ted = &exp->exp_target_data;
        int level = D_CACHE;
 
-       if (exp->exp_obd->obd_self_export == exp)
-               CDEBUG(D_CACHE, "%s: processing self export: %ld %ld "
-                      "%ld\n", exp->exp_obd->obd_name, ted->ted_grant,
-                      ted->ted_pending, ted->ted_dirty);
-
        if (ted->ted_grant < 0 || ted->ted_pending < 0 || ted->ted_dirty < 0)
                level = D_ERROR;
        CDEBUG_LIMIT(level, "%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
@@ -188,6 +183,7 @@ void tgt_grant_sanity_check(struct obd_device *obd, const char *func)
        struct lu_target *lut = obd->u.obt.obt_lut;
        struct tg_grants_data *tgd = &lut->lut_tgd;
        struct obd_export *exp;
+       struct tg_export_data *ted;
        u64                maxsize;
        u64                tot_dirty = 0;
        u64                tot_pending = 0;
@@ -209,6 +205,15 @@ void tgt_grant_sanity_check(struct obd_device *obd, const char *func)
 
        spin_lock(&obd->obd_dev_lock);
        spin_lock(&tgd->tgd_grant_lock);
+       exp = obd->obd_self_export;
+       ted = &exp->exp_target_data;
+       CDEBUG(D_CACHE, "%s: processing self export: %ld %ld "
+              "%ld\n", obd->obd_name, ted->ted_grant,
+              ted->ted_pending, ted->ted_dirty);
+       tot_granted += ted->ted_grant + ted->ted_pending;
+       tot_pending += ted->ted_pending;
+       tot_dirty += ted->ted_dirty;
+
        list_for_each_entry(exp, &obd->obd_exports, exp_obd_chain) {
                error = tgt_check_export_grants(exp, &tot_dirty, &tot_pending,
                                                &tot_granted, maxsize);