From 45900a7777ac02130d8bf65724c4b6cffca9d546 Mon Sep 17 00:00:00 2001 From: Alexander Boyko Date: Thu, 15 Jun 2017 17:25:58 +0300 Subject: [PATCH] LU-4134 obdclass: obd_device improvement The patch removes self exports from obd's reference counting which allows to avoid freeing of self exports by zombie thread. A pair of functions class_register_device()/class_unregister_device() is to make sure that an obd can not be referenced again once its refcount reached 0. For target_handle_connect() take a reference for obd_device during finding it by name. Fix grant mismatch message "tot_granted 4194304 != fo_tot_granted". Signed-off-by: Vladimir Saveliev Signed-off-by: Alexey Lyashkov Signed-off-by: Alexander Boyko Seagate-bug-id: MRP-2139 MRP-3267 Change-Id: I9cc6860431c6bb7db6983e0d15a5d3d2b564265e Reviewed-on: https://review.whamcloud.com/8045 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Alexey Lyashkov Reviewed-by: Oleg Drokin --- lustre/include/obd_class.h | 10 +- lustre/ldlm/ldlm_lib.c | 23 +-- lustre/obdclass/genops.c | 390 ++++++++++++++++++++++++++++++++----------- lustre/obdclass/obd_config.c | 200 ++++++++-------------- lustre/obdclass/obd_mount.c | 7 +- lustre/target/tgt_grant.c | 15 +- 6 files changed, 389 insertions(+), 256 deletions(-) diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 92d1273..88b35465 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -68,9 +68,13 @@ int class_register_type(struct obd_ops *, struct md_ops *, bool enable_proc, const char *nm, struct lu_device_type *ldt); int class_unregister_type(const char *nm); -struct obd_device *class_newdev(const char *type_name, const char *name); -void class_release_dev(struct obd_device *obd); +struct obd_device *class_newdev(const char *type_name, const char *name, + const char *uuid); +int class_register_device(struct obd_device *obd); +void class_unregister_device(struct obd_device *obd); +void class_free_dev(struct obd_device *obd); +struct obd_device *class_dev_by_str(const char *str); int class_name2dev(const char *name); struct obd_device *class_name2obd(const char *name); int class_uuid2dev(struct obd_uuid *uuid); @@ -318,6 +322,8 @@ struct obd_export *class_export_get(struct obd_export *exp); void class_export_put(struct obd_export *exp); struct obd_export *class_new_export(struct obd_device *obddev, struct obd_uuid *cluuid); +struct obd_export *class_new_export_self(struct obd_device *obd, + struct obd_uuid *uuid); void class_unlink_export(struct obd_export *exp); struct obd_import *class_import_get(struct obd_import *); diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 2f368bb..429bf4c 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -951,7 +951,6 @@ int target_handle_connect(struct ptlrpc_request *req) * reconnect case */ struct lustre_handle conn; struct lustre_handle *tmp; - struct obd_uuid tgtuuid; struct obd_uuid cluuid; char *str; int rc = 0; @@ -960,7 +959,6 @@ int target_handle_connect(struct ptlrpc_request *req) bool mds_conn = false, lw_client = false, initial_conn = false; bool mds_mds_conn = false; bool new_mds_mds_conn = false; - bool target_referenced = false; struct obd_connect_data *data, *tmpdata; int size, tmpsize; lnet_nid_t *client_nid = NULL; @@ -974,11 +972,7 @@ int target_handle_connect(struct ptlrpc_request *req) GOTO(out, rc = -EINVAL); } - obd_str2uuid(&tgtuuid, str); - target = class_uuid2obd(&tgtuuid); - if (!target) - target = class_name2obd(str); - + target = class_dev_by_str(str); if (!target) { deuuidify(str, NULL, &target_start, &target_len); LCONSOLE_ERROR_MSG(0x137, "%s: not available for connect " @@ -990,6 +984,9 @@ int target_handle_connect(struct ptlrpc_request *req) } spin_lock(&target->obd_dev_lock); + + target->obd_conn_inprogress++; + if (target->obd_stopping || !target->obd_set_up) { spin_unlock(&target->obd_dev_lock); @@ -1011,13 +1008,6 @@ int target_handle_connect(struct ptlrpc_request *req) GOTO(out, rc = -EAGAIN); } - /* Make sure the target isn't cleaned up while we're here. Yes, - * there's still a race between the above check and our incref here. - * Really, class_uuid2obd should take the ref. */ - class_incref(target, __func__, current); - target_referenced = true; - - target->obd_conn_inprogress++; spin_unlock(&target->obd_dev_lock); str = req_capsule_client_get(&req->rq_pill, &RMF_CLUUID); @@ -1442,12 +1432,11 @@ out: class_export_put(export); } - if (target_referenced == true && target != NULL) { + if (target != NULL) { spin_lock(&target->obd_dev_lock); target->obd_conn_inprogress--; spin_unlock(&target->obd_dev_lock); - - class_decref(target, __func__, current); + class_decref(target, "find", current); } req->rq_status = rc; RETURN(rc); diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index a398ee5..175e391 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -302,21 +303,20 @@ EXPORT_SYMBOL(class_unregister_type); /** * Create a new obd device. * - * Find an empty slot in ::obd_devs[], create a new obd device in it. + * Allocate the new obd_device and initialize it. * * \param[in] type_name obd device type string. * \param[in] name obd device name. + * \param[in] uuid obd device UUID * - * \retval NULL if create fails, otherwise return the obd device - * pointer created. + * \retval newdev pointer to created obd_device + * \retval ERR_PTR(errno) on error */ -struct obd_device *class_newdev(const char *type_name, const char *name) +struct obd_device *class_newdev(const char *type_name, const char *name, + const char *uuid) { - struct obd_device *result = NULL; struct obd_device *newdev; struct obd_type *type = NULL; - int i; - int new_obd_minor = 0; ENTRY; if (strlen(name) >= MAX_OBD_NAME) { @@ -331,93 +331,184 @@ struct obd_device *class_newdev(const char *type_name, const char *name) } newdev = obd_device_alloc(); - if (newdev == NULL) - GOTO(out_type, result = ERR_PTR(-ENOMEM)); - + if (newdev == NULL) { + class_put_type(type); + RETURN(ERR_PTR(-ENOMEM)); + } LASSERT(newdev->obd_magic == OBD_DEVICE_MAGIC); + strncpy(newdev->obd_name, name, sizeof(newdev->obd_name) - 1); + newdev->obd_type = type; + newdev->obd_minor = -1; + + rwlock_init(&newdev->obd_pool_lock); + newdev->obd_pool_limit = 0; + newdev->obd_pool_slv = 0; + + INIT_LIST_HEAD(&newdev->obd_exports); + INIT_LIST_HEAD(&newdev->obd_unlinked_exports); + INIT_LIST_HEAD(&newdev->obd_delayed_exports); + INIT_LIST_HEAD(&newdev->obd_exports_timed); + INIT_LIST_HEAD(&newdev->obd_nid_stats); + spin_lock_init(&newdev->obd_nid_lock); + spin_lock_init(&newdev->obd_dev_lock); + mutex_init(&newdev->obd_dev_mutex); + spin_lock_init(&newdev->obd_osfs_lock); + /* newdev->obd_osfs_age must be set to a value in the distant + * past to guarantee a fresh statfs is fetched on mount. */ + newdev->obd_osfs_age = cfs_time_shift_64(-1000); + + /* XXX belongs in setup not attach */ + init_rwsem(&newdev->obd_observer_link_sem); + /* recovery data */ + init_timer(&newdev->obd_recovery_timer); + spin_lock_init(&newdev->obd_recovery_task_lock); + init_waitqueue_head(&newdev->obd_next_transno_waitq); + init_waitqueue_head(&newdev->obd_evict_inprogress_waitq); + INIT_LIST_HEAD(&newdev->obd_req_replay_queue); + INIT_LIST_HEAD(&newdev->obd_lock_replay_queue); + INIT_LIST_HEAD(&newdev->obd_final_req_queue); + INIT_LIST_HEAD(&newdev->obd_evict_list); + INIT_LIST_HEAD(&newdev->obd_lwp_list); + + llog_group_init(&newdev->obd_olg); + /* Detach drops this */ + atomic_set(&newdev->obd_refcount, 1); + lu_ref_init(&newdev->obd_reference); + lu_ref_add(&newdev->obd_reference, "newdev", newdev); + + newdev->obd_conn_inprogress = 0; + + strncpy(newdev->obd_uuid.uuid, uuid, strlen(uuid)); + + CDEBUG(D_IOCTL, "Allocate new device %s (%p)\n", + newdev->obd_name, newdev); + + return newdev; +} - write_lock(&obd_dev_lock); - for (i = 0; i < class_devno_max(); i++) { - struct obd_device *obd = class_num2obd(i); +/** + * Free obd device. + * + * \param[in] obd obd_device to be freed + * + * \retval none + */ +void class_free_dev(struct obd_device *obd) +{ + struct obd_type *obd_type = obd->obd_type; - if (obd && (strcmp(name, obd->obd_name) == 0)) { - CERROR("Device %s already exists at %d, won't add\n", - name, i); - if (result) { - LASSERTF(result->obd_magic == OBD_DEVICE_MAGIC, - "%p obd_magic %08x != %08x\n", result, - result->obd_magic, OBD_DEVICE_MAGIC); - LASSERTF(result->obd_minor == new_obd_minor, - "%p obd_minor %d != %d\n", result, - result->obd_minor, new_obd_minor); - - obd_devs[result->obd_minor] = NULL; - result->obd_name[0]='\0'; - } - result = ERR_PTR(-EEXIST); - break; - } - if (!result && !obd) { - result = newdev; - result->obd_minor = i; - new_obd_minor = i; - result->obd_type = type; - strncpy(result->obd_name, name, - sizeof(result->obd_name) - 1); - obd_devs[i] = result; - } - } - write_unlock(&obd_dev_lock); + LASSERTF(obd->obd_magic == OBD_DEVICE_MAGIC, "%p obd_magic %08x " + "!= %08x\n", obd, obd->obd_magic, OBD_DEVICE_MAGIC); + LASSERTF(obd->obd_minor == -1 || obd_devs[obd->obd_minor] == obd, + "obd %p != obd_devs[%d] %p\n", + obd, obd->obd_minor, obd_devs[obd->obd_minor]); + LASSERTF(atomic_read(&obd->obd_refcount) == 0, + "obd_refcount should be 0, not %d\n", + atomic_read(&obd->obd_refcount)); + LASSERT(obd_type != NULL); - if (result == NULL && i >= class_devno_max()) { - CERROR("all %u OBD devices used, increase MAX_OBD_DEVICES\n", - class_devno_max()); - GOTO(out, result = ERR_PTR(-EOVERFLOW)); - } + CDEBUG(D_INFO, "Release obd device %s obd_type name = %s\n", + obd->obd_name, obd->obd_type->typ_name); - if (IS_ERR(result)) - GOTO(out, result); + CDEBUG(D_CONFIG, "finishing cleanup of obd %s (%s)\n", + obd->obd_name, obd->obd_uuid.uuid); + if (obd->obd_stopping) { + int err; - CDEBUG(D_IOCTL, "Adding new device %s (%p)\n", - result->obd_name, result); + /* If we're not stopping, we were never set up */ + err = obd_cleanup(obd); + if (err) + CERROR("Cleanup %s returned %d\n", + obd->obd_name, err); + } - RETURN(result); -out: - obd_device_free(newdev); -out_type: - class_put_type(type); - return result; + obd_device_free(obd); + + class_put_type(obd_type); } -void class_release_dev(struct obd_device *obd) +/** + * Unregister obd device. + * + * Free slot in obd_dev[] used by \a obd. + * + * \param[in] new_obd obd_device to be unregistered + * + * \retval none + */ +void class_unregister_device(struct obd_device *obd) { - struct obd_type *obd_type = obd->obd_type; - - LASSERTF(obd->obd_magic == OBD_DEVICE_MAGIC, "%p obd_magic %08x != %08x\n", - obd, obd->obd_magic, OBD_DEVICE_MAGIC); - LASSERTF(obd == obd_devs[obd->obd_minor], "obd %p != obd_devs[%d] %p\n", - obd, obd->obd_minor, obd_devs[obd->obd_minor]); - LASSERT(obd_type != NULL); + write_lock(&obd_dev_lock); + if (obd->obd_minor >= 0) { + LASSERT(obd_devs[obd->obd_minor] == obd); + obd_devs[obd->obd_minor] = NULL; + obd->obd_minor = -1; + } + write_unlock(&obd_dev_lock); +} - CDEBUG(D_INFO, "Release obd device %s at %d obd_type name =%s\n", - obd->obd_name, obd->obd_minor, obd->obd_type->typ_name); +/** + * Register obd device. + * + * Find free slot in obd_devs[], fills it with \a new_obd. + * + * \param[in] new_obd obd_device to be registered + * + * \retval 0 success + * \retval -EEXIST device with this name is registered + * \retval -EOVERFLOW obd_devs[] is full + */ +int class_register_device(struct obd_device *new_obd) +{ + int ret = 0; + int i; + int new_obd_minor = 0; + bool minor_assign = false; write_lock(&obd_dev_lock); - obd_devs[obd->obd_minor] = NULL; + for (i = 0; i < class_devno_max(); i++) { + struct obd_device *obd = class_num2obd(i); + + if (obd != NULL && + (strcmp(new_obd->obd_name, obd->obd_name) == 0)) { + CERROR("%s: already exists, won't add\n", + obd->obd_name); + /* in case we found a free slot before duplicate */ + minor_assign = false; + ret = -EEXIST; + break; + } + if (!minor_assign && obd == NULL) { + new_obd_minor = i; + minor_assign = true; + } + } + + if (minor_assign) { + new_obd->obd_minor = new_obd_minor; + LASSERTF(obd_devs[new_obd_minor] == NULL, "obd_devs[%d] " + "%p\n", new_obd_minor, obd_devs[new_obd_minor]); + obd_devs[new_obd_minor] = new_obd; + } else { + if (ret == 0) { + ret = -EOVERFLOW; + CERROR("%s: all %u/%u devices used, increase " + "MAX_OBD_DEVICES: rc = %d\n", new_obd->obd_name, + i, class_devno_max(), ret); + } + } write_unlock(&obd_dev_lock); - obd_device_free(obd); - class_put_type(obd_type); + RETURN(ret); } -int class_name2dev(const char *name) +static int class_name2dev_nolock(const char *name) { int i; if (!name) return -1; - read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); @@ -426,17 +517,30 @@ int class_name2dev(const char *name) out any references */ LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC); if (obd->obd_attached) { - read_unlock(&obd_dev_lock); return i; } break; } } - read_unlock(&obd_dev_lock); return -1; } +int class_name2dev(const char *name) +{ + int i; + + if (!name) + return -1; + + read_lock(&obd_dev_lock); + i = class_name2dev_nolock(name); + read_unlock(&obd_dev_lock); + + return i; +} +EXPORT_SYMBOL(class_name2dev); + struct obd_device *class_name2obd(const char *name) { int dev = class_name2dev(name); @@ -447,25 +551,34 @@ struct obd_device *class_name2obd(const char *name) } EXPORT_SYMBOL(class_name2obd); -int class_uuid2dev(struct obd_uuid *uuid) +int class_uuid2dev_nolock(struct obd_uuid *uuid) { int i; - read_lock(&obd_dev_lock); for (i = 0; i < class_devno_max(); i++) { struct obd_device *obd = class_num2obd(i); if (obd && obd_uuid_equals(uuid, &obd->obd_uuid)) { LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC); - read_unlock(&obd_dev_lock); return i; } } - read_unlock(&obd_dev_lock); return -1; } +int class_uuid2dev(struct obd_uuid *uuid) +{ + int i; + + read_lock(&obd_dev_lock); + i = class_uuid2dev_nolock(uuid); + read_unlock(&obd_dev_lock); + + return i; +} +EXPORT_SYMBOL(class_uuid2dev); + struct obd_device *class_uuid2obd(struct obd_uuid *uuid) { int dev = class_uuid2dev(uuid); @@ -504,6 +617,40 @@ struct obd_device *class_num2obd(int num) } /** + * Find obd in obd_dev[] by name or uuid. + * + * Increment obd's refcount if found. + * + * \param[in] str obd name or uuid + * + * \retval NULL if not found + * \retval target pointer to found obd_device + */ +struct obd_device *class_dev_by_str(const char *str) +{ + struct obd_device *target = NULL; + struct obd_uuid tgtuuid; + int rc; + + obd_str2uuid(&tgtuuid, str); + + read_lock(&obd_dev_lock); + rc = class_uuid2dev_nolock(&tgtuuid); + if (rc < 0) + rc = class_name2dev_nolock(str); + + if (rc >= 0) + target = class_num2obd(rc); + + if (target != NULL) + class_incref(target, "find", current); + read_unlock(&obd_dev_lock); + + RETURN(target); +} +EXPORT_SYMBOL(class_dev_by_str); + +/** * Get obd devices count. Device in any * state are counted * \retval obd device count @@ -795,7 +942,10 @@ static void class_export_destroy(struct obd_export *exp) LASSERT(list_empty(&exp->exp_req_replay_queue)); LASSERT(list_empty(&exp->exp_hp_rpcs)); obd_destroy_export(exp); - class_decref(obd, "export", exp); + /* self export doesn't hold a reference to an obd, although it + * exists until freeing of the obd */ + if (exp != obd->obd_self_export) + class_decref(obd, "export", exp); OBD_FREE_RCU(exp, sizeof(*exp), &exp->exp_handle); EXIT; @@ -828,24 +978,37 @@ void class_export_put(struct obd_export *exp) atomic_read(&exp->exp_refcount) - 1); if (atomic_dec_and_test(&exp->exp_refcount)) { - LASSERT(!list_empty(&exp->exp_obd_chain)); - LASSERT(list_empty(&exp->exp_stale_list)); + struct obd_device *obd = exp->exp_obd; + CDEBUG(D_IOCTL, "final put %p/%s\n", exp, exp->exp_client_uuid.uuid); /* release nid stat refererence */ lprocfs_exp_cleanup(exp); - obd_zombie_export_add(exp); + if (exp == obd->obd_self_export) { + /* self export should be destroyed without + * zombie thread as it doesn't hold a + * reference to obd and doesn't hold any + * resources */ + class_export_destroy(exp); + /* self export is destroyed, no class + * references exist and it is safe to free + * obd */ + class_free_dev(obd); + } else { + LASSERT(!list_empty(&exp->exp_obd_chain)); + obd_zombie_export_add(exp); + } + } } EXPORT_SYMBOL(class_export_put); - /* Creates a new export, adds it to the hash table, and returns a * pointer to it. The refcount is 2: one for the hash reference, and * one for the pointer returned by this function. */ -struct obd_export *class_new_export(struct obd_device *obd, - struct obd_uuid *cluuid) +struct obd_export *__class_new_export(struct obd_device *obd, + struct obd_uuid *cluuid, bool is_self) { struct obd_export *export; struct cfs_hash *hash = NULL; @@ -859,6 +1022,7 @@ struct obd_export *class_new_export(struct obd_device *obd, export->exp_conn_cnt = 0; export->exp_lock_hash = NULL; export->exp_flock_hash = NULL; + /* 2 = class_handle_hash + last */ atomic_set(&export->exp_refcount, 2); atomic_set(&export->exp_rpc_count, 0); atomic_set(&export->exp_cb_count, 0); @@ -892,17 +1056,17 @@ struct obd_export *class_new_export(struct obd_device *obd, export->exp_client_uuid = *cluuid; obd_init_export(export); - spin_lock(&obd->obd_dev_lock); - /* shouldn't happen, but might race */ - if (obd->obd_stopping) - GOTO(exit_unlock, rc = -ENODEV); + if (!obd_uuid_equals(cluuid, &obd->obd_uuid)) { + spin_lock(&obd->obd_dev_lock); + /* shouldn't happen, but might race */ + if (obd->obd_stopping) + GOTO(exit_unlock, rc = -ENODEV); - hash = cfs_hash_getref(obd->obd_uuid_hash); - if (hash == NULL) - GOTO(exit_unlock, rc = -ENODEV); - spin_unlock(&obd->obd_dev_lock); + hash = cfs_hash_getref(obd->obd_uuid_hash); + if (hash == NULL) + GOTO(exit_unlock, rc = -ENODEV); + spin_unlock(&obd->obd_dev_lock); - if (!obd_uuid_equals(cluuid, &obd->obd_uuid)) { rc = cfs_hash_add_unique(hash, cluuid, &export->exp_uuid_hash); if (rc != 0) { LCONSOLE_WARN("%s: denying duplicate export for %s, %d\n", @@ -914,17 +1078,24 @@ struct obd_export *class_new_export(struct obd_device *obd, at_init(&export->exp_bl_lock_at, obd_timeout, 0); spin_lock(&obd->obd_dev_lock); if (obd->obd_stopping) { - cfs_hash_del(hash, cluuid, &export->exp_uuid_hash); - GOTO(exit_unlock, rc = -ENODEV); + if (hash) + cfs_hash_del(hash, cluuid, &export->exp_uuid_hash); + GOTO(exit_unlock, rc = -ESHUTDOWN); } - class_incref(obd, "export", export); - list_add(&export->exp_obd_chain, &export->exp_obd->obd_exports); - list_add_tail(&export->exp_obd_chain_timed, - &export->exp_obd->obd_exports_timed); - export->exp_obd->obd_num_exports++; + if (!is_self) { + class_incref(obd, "export", export); + list_add_tail(&export->exp_obd_chain_timed, + &obd->obd_exports_timed); + list_add(&export->exp_obd_chain, &obd->obd_exports); + obd->obd_num_exports++; + } else { + INIT_LIST_HEAD(&export->exp_obd_chain_timed); + INIT_LIST_HEAD(&export->exp_obd_chain); + } spin_unlock(&obd->obd_dev_lock); - cfs_hash_putref(hash); + if (hash) + cfs_hash_putref(hash); RETURN(export); exit_unlock: @@ -938,12 +1109,29 @@ exit_err: OBD_FREE_PTR(export); return ERR_PTR(rc); } + +struct obd_export *class_new_export(struct obd_device *obd, + struct obd_uuid *uuid) +{ + return __class_new_export(obd, uuid, false); +} EXPORT_SYMBOL(class_new_export); +struct obd_export *class_new_export_self(struct obd_device *obd, + struct obd_uuid *uuid) +{ + return __class_new_export(obd, uuid, true); +} + void class_unlink_export(struct obd_export *exp) { class_handle_unhash(&exp->exp_handle); + if (exp->exp_obd->obd_self_export == exp) { + class_export_put(exp); + return; + } + spin_lock(&exp->exp_obd->obd_dev_lock); /* delete an uuid-export hashitem from hashtables */ if (!hlist_unhashed(&exp->exp_uuid_hash)) diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index bcb9375..81754de 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -365,6 +365,7 @@ EXPORT_SYMBOL(lustre_cfg_string); */ int class_attach(struct lustre_cfg *lcfg) { + struct obd_export *exp; struct obd_device *obd = NULL; char *typename, *name, *uuid; int rc, len; @@ -381,24 +382,26 @@ int class_attach(struct lustre_cfg *lcfg) RETURN(-EINVAL); } name = lustre_cfg_string(lcfg, 0); - if (!LUSTRE_CFG_BUFLEN(lcfg, 2)) { CERROR("No UUID passed!\n"); RETURN(-EINVAL); } - uuid = lustre_cfg_string(lcfg, 2); - CDEBUG(D_IOCTL, "attach type %s name: %s uuid: %s\n", - MKSTR(typename), MKSTR(name), MKSTR(uuid)); + uuid = lustre_cfg_string(lcfg, 2); + len = strlen(uuid); + if (len >= sizeof(obd->obd_uuid)) { + CERROR("%s: uuid must be < %d bytes long\n", + name, (int)sizeof(obd->obd_uuid)); + RETURN(-EINVAL); + } - obd = class_newdev(typename, name); + obd = class_newdev(typename, name, uuid); if (IS_ERR(obd)) { /* Already exists or out of obds */ rc = PTR_ERR(obd); - obd = NULL; CERROR("Cannot create device %s of type %s : %d\n", name, typename, rc); - GOTO(out, rc); + RETURN(rc); } LASSERTF(obd != NULL, "Cannot get obd device %s of type %s\n", name, typename); @@ -408,64 +411,30 @@ int class_attach(struct lustre_cfg *lcfg) LASSERTF(strncmp(obd->obd_name, name, strlen(name)) == 0, "%p obd_name %s != %s\n", obd, obd->obd_name, name); - rwlock_init(&obd->obd_pool_lock); - obd->obd_pool_limit = 0; - obd->obd_pool_slv = 0; - - INIT_LIST_HEAD(&obd->obd_exports); - INIT_LIST_HEAD(&obd->obd_unlinked_exports); - INIT_LIST_HEAD(&obd->obd_delayed_exports); - INIT_LIST_HEAD(&obd->obd_exports_timed); - INIT_LIST_HEAD(&obd->obd_nid_stats); - spin_lock_init(&obd->obd_nid_lock); - spin_lock_init(&obd->obd_dev_lock); - mutex_init(&obd->obd_dev_mutex); - spin_lock_init(&obd->obd_osfs_lock); - /* obd->obd_osfs_age must be set to a value in the distant - * past to guarantee a fresh statfs is fetched on mount. */ - obd->obd_osfs_age = cfs_time_shift_64(-1000); - - /* XXX belongs in setup not attach */ - init_rwsem(&obd->obd_observer_link_sem); - /* recovery data */ - init_timer(&obd->obd_recovery_timer); - spin_lock_init(&obd->obd_recovery_task_lock); - init_waitqueue_head(&obd->obd_next_transno_waitq); - init_waitqueue_head(&obd->obd_evict_inprogress_waitq); - INIT_LIST_HEAD(&obd->obd_req_replay_queue); - INIT_LIST_HEAD(&obd->obd_lock_replay_queue); - INIT_LIST_HEAD(&obd->obd_final_req_queue); - INIT_LIST_HEAD(&obd->obd_evict_list); - INIT_LIST_HEAD(&obd->obd_lwp_list); - - llog_group_init(&obd->obd_olg); - - obd->obd_conn_inprogress = 0; - - len = strlen(uuid); - if (len >= sizeof(obd->obd_uuid)) { - CERROR("uuid must be < %d bytes long\n", - (int)sizeof(obd->obd_uuid)); - GOTO(out, rc = -EINVAL); - } - memcpy(obd->obd_uuid.uuid, uuid, len); + exp = class_new_export_self(obd, &obd->obd_uuid); + if (IS_ERR(exp)) { + /* force free */ + GOTO(out, rc = PTR_ERR(exp)); + RETURN(PTR_ERR(exp)); + } - /* Detach drops this */ - spin_lock(&obd->obd_dev_lock); - atomic_set(&obd->obd_refcount, 1); - spin_unlock(&obd->obd_dev_lock); - lu_ref_init(&obd->obd_reference); - lu_ref_add(&obd->obd_reference, "attach", obd); + obd->obd_self_export = exp; + list_del_init(&exp->exp_obd_chain_timed); + class_export_put(exp); + + rc = class_register_device(obd); + if (rc != 0) + GOTO(out, rc); - obd->obd_attached = 1; - CDEBUG(D_IOCTL, "OBD: dev %d attached type %s with refcount %d\n", + obd->obd_attached = 1; + CDEBUG(D_IOCTL, "OBD: dev %d attached type %s with refcount %d\n", obd->obd_minor, typename, atomic_read(&obd->obd_refcount)); - RETURN(0); - out: - if (obd != NULL) { - class_release_dev(obd); - } - return rc; + RETURN(0); +out: + class_decref(obd, "newdev", obd); + class_free_dev(obd); + + RETURN(rc); } EXPORT_SYMBOL(class_attach); @@ -475,7 +444,6 @@ EXPORT_SYMBOL(class_attach); int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) { int err = 0; - struct obd_export *exp; ENTRY; LASSERT(obd != NULL); @@ -524,7 +492,7 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) CFS_HASH_MAX_THETA, &uuid_hash_ops, CFS_HASH_DEFAULT); if (!obd->obd_uuid_hash) - GOTO(err_hash, err = -ENOMEM); + GOTO(err_exit, err = -ENOMEM); /* create a nid-export lustre hash */ obd->obd_nid_hash = cfs_hash_create("NID_HASH", @@ -535,7 +503,7 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) CFS_HASH_MAX_THETA, &nid_hash_ops, CFS_HASH_DEFAULT); if (!obd->obd_nid_hash) - GOTO(err_hash, err = -ENOMEM); + GOTO(err_exit, err = -ENOMEM); /* create a nid-stats lustre hash */ obd->obd_nid_stats_hash = cfs_hash_create("NID_STATS", @@ -545,8 +513,8 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) CFS_HASH_MIN_THETA, CFS_HASH_MAX_THETA, &nid_stat_hash_ops, CFS_HASH_DEFAULT); - if (!obd->obd_nid_stats_hash) - GOTO(err_hash, err = -ENOMEM); + if (!obd->obd_nid_stats_hash) + GOTO(err_exit, err = -ENOMEM); /* create a client_generation-export lustre hash */ obd->obd_gen_hash = cfs_hash_create("UUID_HASH", @@ -557,21 +525,13 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) CFS_HASH_MAX_THETA, &gen_hash_ops, CFS_HASH_DEFAULT); if (!obd->obd_gen_hash) - GOTO(err_hash, err = -ENOMEM); - - exp = class_new_export(obd, &obd->obd_uuid); - if (IS_ERR(exp)) - GOTO(err_hash, err = PTR_ERR(exp)); - - obd->obd_self_export = exp; - list_del_init(&exp->exp_obd_chain_timed); - class_export_put(exp); + GOTO(err_exit, err = -ENOMEM); - err = obd_setup(obd, lcfg); - if (err) - GOTO(err_exp, err); + err = obd_setup(obd, lcfg); + if (err) + GOTO(err_exit, err); - obd->obd_set_up = 1; + obd->obd_set_up = 1; spin_lock(&obd->obd_dev_lock); /* cleanup drops this */ @@ -582,12 +542,7 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) obd->obd_name, obd->obd_uuid.uuid); RETURN(0); -err_exp: - if (obd->obd_self_export) { - class_unlink_export(obd->obd_self_export); - obd->obd_self_export = NULL; - } -err_hash: +err_exit: if (obd->obd_uuid_hash) { cfs_hash_putref(obd->obd_uuid_hash); obd->obd_uuid_hash = NULL; @@ -631,10 +586,14 @@ int class_detach(struct obd_device *obd, struct lustre_cfg *lcfg) obd->obd_attached = 0; spin_unlock(&obd->obd_dev_lock); + /* cleanup in progress. we don't like to find this device after now */ + class_unregister_device(obd); + CDEBUG(D_IOCTL, "detach on obd %s (uuid %s)\n", obd->obd_name, obd->obd_uuid.uuid); - class_decref(obd, "attach", obd); + class_decref(obd, "newdev", obd); + RETURN(0); } EXPORT_SYMBOL(class_detach); @@ -664,6 +623,9 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg) } /* Leave this on forever */ obd->obd_stopping = 1; + /* function can't return error after that point, so clear setup flag + * as early as possible to avoid finding via obd_devs / hash */ + obd->obd_set_up = 0; spin_unlock(&obd->obd_dev_lock); /* wait for already-arrived-connections to finish. */ @@ -696,17 +658,11 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg) LASSERT(obd->obd_self_export); - /* The three references that should be remaining are the - * obd_self_export and the attach and setup references. */ - if (atomic_read(&obd->obd_refcount) > 3) { - /* refcounf - 3 might be the number of real exports - (excluding self export). But class_incref is called - by other things as well, so don't count on it. */ - CDEBUG(D_IOCTL, "%s: forcing exports to disconnect: %d\n", - obd->obd_name, atomic_read(&obd->obd_refcount) - 3); - dump_exports(obd, 0, D_HA); - class_disconnect_exports(obd); - } + CDEBUG(D_IOCTL, "%s: forcing exports to disconnect: %d/%d\n", + obd->obd_name, obd->obd_num_exports, + atomic_read(&obd->obd_refcount) - 2); + dump_exports(obd, 0, D_HA); + class_disconnect_exports(obd); /* Precleanup, we must make sure all exports get destroyed. */ err = obd_precleanup(obd); @@ -758,43 +714,31 @@ EXPORT_SYMBOL(class_incref); void class_decref(struct obd_device *obd, const char *scope, const void *source) { - int err; - int refs; + int last; - spin_lock(&obd->obd_dev_lock); - atomic_dec(&obd->obd_refcount); - refs = atomic_read(&obd->obd_refcount); - spin_unlock(&obd->obd_dev_lock); + CDEBUG(D_INFO, "Decref %s (%p) now %d - %s\n", obd->obd_name, obd, + atomic_read(&obd->obd_refcount), scope); + + LASSERT(obd->obd_num_exports >= 0); + last = atomic_dec_and_test(&obd->obd_refcount); lu_ref_del(&obd->obd_reference, scope, source); - CDEBUG(D_INFO, "Decref %s (%p) now %d\n", obd->obd_name, obd, refs); + if (last) { + struct obd_export *exp; - if ((refs == 1) && obd->obd_stopping) { + LASSERT(!obd->obd_attached); /* All exports have been destroyed; there should - be no more in-progress ops by this point.*/ - - spin_lock(&obd->obd_self_export->exp_lock); - obd->obd_self_export->exp_flags |= exp_flags_from_obd(obd); - spin_unlock(&obd->obd_self_export->exp_lock); + * be no more in-progress ops by this point.*/ + exp = obd->obd_self_export; - /* note that we'll recurse into class_decref again */ - class_unlink_export(obd->obd_self_export); - return; - } - - if (refs == 0) { - CDEBUG(D_CONFIG, "finishing cleanup of obd %s (%s)\n", - obd->obd_name, obd->obd_uuid.uuid); - LASSERT(!obd->obd_attached); - if (obd->obd_stopping) { - /* If we're not stopping, we were never set up */ - err = obd_cleanup(obd); - if (err) - CERROR("Cleanup %s returned %d\n", - obd->obd_name, err); + if (exp) { + exp->exp_flags |= exp_flags_from_obd(obd); + /* + * note that we'll recurse into class_decref again + * but it's not a problem because we was last user + */ + class_unlink_export(exp); } - - class_release_dev(obd); } } EXPORT_SYMBOL(class_decref); diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 362d8c1..b8d3b76 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -219,7 +219,7 @@ int lustre_start_mgc(struct super_block *sb) struct lustre_sb_info *lsi = s2lsi(sb); struct obd_device *obd; struct obd_export *exp; - struct obd_uuid *uuid; + struct obd_uuid *uuid = NULL; class_uuid_t uuidc; lnet_nid_t nid; char nidstr[LNET_NIDSTR_SIZE]; @@ -408,7 +408,6 @@ int lustre_start_mgc(struct super_block *sb) rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME, (char *)uuid->uuid, LUSTRE_MGS_OBDNAME, niduuid, NULL, NULL); - OBD_FREE_PTR(uuid); if (rc) GOTO(out_free, rc); @@ -469,7 +468,7 @@ int lustre_start_mgc(struct super_block *sb) lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR) data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV; data->ocd_version = LUSTRE_VERSION_CODE; - rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL); + rc = obd_connect(NULL, &exp, obd, uuid, data, NULL); if (rc) { CERROR("connect failed %d\n", rc); GOTO(out, rc); @@ -484,6 +483,8 @@ out: out_free: mutex_unlock(&mgc_start_lock); + if (uuid) + OBD_FREE_PTR(uuid); if (data) OBD_FREE_PTR(data); if (mgcname) diff --git a/lustre/target/tgt_grant.c b/lustre/target/tgt_grant.c index 1caad7c..7887623 100644 --- a/lustre/target/tgt_grant.c +++ b/lustre/target/tgt_grant.c @@ -138,11 +138,6 @@ static int tgt_check_export_grants(struct obd_export *exp, u64 *dirty, struct tg_export_data *ted = &exp->exp_target_data; int level = D_CACHE; - if (exp->exp_obd->obd_self_export == exp) - CDEBUG(D_CACHE, "%s: processing self export: %ld %ld " - "%ld\n", exp->exp_obd->obd_name, ted->ted_grant, - ted->ted_pending, ted->ted_dirty); - if (ted->ted_grant < 0 || ted->ted_pending < 0 || ted->ted_dirty < 0) level = D_ERROR; CDEBUG_LIMIT(level, "%s: cli %s/%p dirty %ld pend %ld grant %ld\n", @@ -188,6 +183,7 @@ void tgt_grant_sanity_check(struct obd_device *obd, const char *func) struct lu_target *lut = obd->u.obt.obt_lut; struct tg_grants_data *tgd = &lut->lut_tgd; struct obd_export *exp; + struct tg_export_data *ted; u64 maxsize; u64 tot_dirty = 0; u64 tot_pending = 0; @@ -209,6 +205,15 @@ void tgt_grant_sanity_check(struct obd_device *obd, const char *func) spin_lock(&obd->obd_dev_lock); spin_lock(&tgd->tgd_grant_lock); + exp = obd->obd_self_export; + ted = &exp->exp_target_data; + CDEBUG(D_CACHE, "%s: processing self export: %ld %ld " + "%ld\n", obd->obd_name, ted->ted_grant, + ted->ted_pending, ted->ted_dirty); + tot_granted += ted->ted_grant + ted->ted_pending; + tot_pending += ted->ted_pending; + tot_dirty += ted->ted_dirty; + list_for_each_entry(exp, &obd->obd_exports, exp_obd_chain) { error = tgt_check_export_grants(exp, &tot_dirty, &tot_pending, &tot_granted, maxsize); -- 1.8.3.1