* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
*
- * Copyright (c) 2012, 2013, Intel Corporation.
+ * Copyright (c) 2012, 2014, Intel Corporation.
*/
/*
* This file is part of Lustre, http://www.lustre.org/
*
* lustre/osp/osp_dev.c
*
- * Lustre OST/MDT Proxy Device (OSP) is in the MDS stack, and used as
- * a proxy device to communicate with other MDTs and OSTs.
- *
- * The purpose is to export the OSD API from the remote MDT or OST's underlying
- * OSD for access and modification by the local service (typically MDD/LOD, or
- * LFSCK) as if it were a local OSD. The goal is that the OSP provides a
- * transparent interface for access to the remote OSD.
- *
* Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
* Author: Mikhail Pershin <mike.pershin@intel.com>
* Author: Di Wang <di.wang@intel.com>
*/
+/*
+ * The Object Storage Proxy (OSP) module provides an implementation of
+ * the DT API for remote MDTs and OSTs. Every local OSP device (or
+ * object) is a proxy for a remote OSD device (or object). Thus OSP
+ * converts DT operations into RPCs, which are sent to the OUT service
+ * on a remote target, converted back to DT operations, and
+ * executed. Of course there are many ways in which this description
+ * is inaccurate but it's a good enough mental model. OSP is used by
+ * the MDT stack in several ways:
+ *
+ * - OSP devices allocate FIDs for the stripe sub-objects of a striped
+ * file or directory.
+ *
+ * - OSP objects represent the remote MDT and OST objects that are
+ * the stripes of a striped object.
+ *
+ * - OSP devices log, send, and track synchronous operations (setattr
+ * and unlink) to remote targets.
+ *
+ * - OSP objects are the bottom slice of the compound LU object
+ * representing a remote MDT object: MDT/MDD/LOD/OSP.
+ *
+ * - OSP objects are used by LFSCK to represent remote OST objects
+ * during the verification of MDT-OST consistency.
+ *
+ * - OSP devices batch idempotent requests (declare_attr_get() and
+ * declare_xattr_get()) to the remote target and cache their results.
+ *
+ * In addition the OSP layer implements a subset of the OBD device API
+ * to support being a client of a remote target, connecting to other
+ * layers, and FID allocation.
+ */
#define DEBUG_SUBSYSTEM S_MDS
#include <lustre_ioctl.h>
#include <lustre_param.h>
#include <lustre_log.h>
-#include <lustre_mdc.h>
#include "osp_internal.h"
* \retval object object being created if the creation succeed.
* \retval NULL NULL if the creation failed.
*/
-struct lu_object *osp_object_alloc(const struct lu_env *env,
- const struct lu_object_header *hdr,
- struct lu_device *d)
+static struct lu_object *osp_object_alloc(const struct lu_env *env,
+ const struct lu_object_header *hdr,
+ struct lu_device *d)
{
struct lu_object_header *h = NULL;
struct osp_object *o;
RETURN(dto);
/* Get attributes of the local object. */
- rc = dt_attr_get(env, dto, attr, NULL);
+ rc = dt_attr_get(env, dto, attr);
if (rc) {
CERROR("%s: can't be initialized: rc = %d\n",
osp->opd_obd->obd_name, rc);
struct osp_thread_info *osi = osp_env_info(env);
struct lu_fid *fid = &osp->opd_last_used_fid;
struct dt_object *dto;
- int rc;
+ int rc = -EFAULT;
ENTRY;
dto = osp_find_or_create_local_file(env, osp, &osi->osi_attr,
osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
osp->opd_index);
rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
- if (rc != 0)
+ if (rc != 0 && rc != -EFAULT)
GOTO(out, rc);
- } else {
+ }
+
+ if (rc == -EFAULT) { /* fresh LAST_ID */
fid->f_oid = 0;
osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
osp->opd_index);
struct osp_thread_info *osi = osp_env_info(env);
struct lu_fid *fid = &osp->opd_last_used_fid;
struct dt_object *dto;
- int rc;
+ int rc = -EFAULT;
ENTRY;
dto = osp_find_or_create_local_file(env, osp, &osi->osi_attr,
osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
osp->opd_index);
rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
- if (rc != 0)
+ if (rc != 0 && rc != -EFAULT)
GOTO(out, rc);
- } else {
+ }
+
+ if (rc == -EFAULT) { /* fresh OSP */
fid->f_seq = 0;
osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
osp->opd_index);
rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
osi->osi_off);
+ if (rc != 0)
+ GOTO(out, rc);
}
osp->opd_last_used_seq_file = dto;
RETURN(0);
rc = osp_disconnect(d);
- osp_sync_fini(d);
-
if (!d->opd_connect_mdt) {
+ /* stop sync thread */
+ osp_sync_fini(d);
+
/* stop precreate thread */
osp_precreate_fini(d);
break;
case LCFG_PARAM:
LASSERT(obd);
- rc = class_process_proc_seq_param(PARAM_OSC, obd->obd_vars,
- lcfg, obd);
+ rc = class_process_proc_param(PARAM_OSC, obd->obd_vars,
+ lcfg, obd);
if (rc > 0)
rc = 0;
if (rc == -ENOSYS) {
unsigned long start = cfs_time_current();
ENTRY;
+ /* No Sync between MDTs yet. */
+ if (d->opd_connect_mdt)
+ RETURN(0);
+
if (unlikely(d->opd_imp_active == 0))
RETURN(-ENOTCONN);
id = d->opd_syn_last_used_id;
+ down_write(&d->opd_async_updates_rwsem);
+
+ CDEBUG(D_OTHER, "%s: async updates %d\n", d->opd_obd->obd_name,
+ atomic_read(&d->opd_async_updates_count));
+
+ /* make sure the connection is fine */
+ expire = cfs_time_shift(obd_timeout);
+ lwi = LWI_TIMEOUT(expire - cfs_time_current(), osp_sync_timeout, d);
+ rc = l_wait_event(d->opd_syn_barrier_waitq,
+ atomic_read(&d->opd_async_updates_count) == 0,
+ &lwi);
+ up_write(&d->opd_async_updates_rwsem);
+ if (rc != 0)
+ GOTO(out, rc);
- CDEBUG(D_OTHER, "%s: id: used %lu, processed %lu\n",
+ CDEBUG(D_CACHE, "%s: id: used %lu, processed %lu\n",
d->opd_obd->obd_name, id, d->opd_syn_last_processed_id);
/* wait till all-in-line are processed */
/* block new processing (barrier>0 - few callers are possible */
atomic_inc(&d->opd_syn_barrier);
- CDEBUG(D_OTHER, "%s: %u in flight\n", d->opd_obd->obd_name,
+ CDEBUG(D_CACHE, "%s: %u in flight\n", d->opd_obd->obd_name,
d->opd_syn_rpc_in_flight);
/* wait till all-in-flight are replied, so executed by the target */
GOTO(out, rc = -ETIMEDOUT);
}
- CDEBUG(D_OTHER, "%s: done in %lu\n", d->opd_obd->obd_name,
- cfs_time_current() - start);
out:
/* resume normal processing (barrier=0) */
atomic_dec(&d->opd_syn_barrier);
__osp_sync_check_for_work(d);
+ CDEBUG(D_CACHE, "%s: done in %lu: rc = %d\n", d->opd_obd->obd_name,
+ cfs_time_current() - start, rc);
+
RETURN(rc);
}
.dt_trans_create = osp_trans_create,
.dt_trans_start = osp_trans_start,
.dt_trans_stop = osp_trans_stop,
+ .dt_trans_cb_add = osp_trans_cb_add,
};
/**
}
/**
+ * Determine if the lock needs to be cancelled
+ *
+ * Determine if the unused lock should be cancelled before replay, see
+ * (ldlm_cancel_no_wait_policy()). Currently, only inode bits lock exists
+ * between MDTs.
+ *
+ * \param[in] lock lock to be checked.
+ *
+ * \retval 1 if the lock needs to be cancelled before replay.
+ * \retval 0 if the lock does not need to be cancelled before
+ * replay.
+ */
+static int osp_cancel_weight(struct ldlm_lock *lock)
+{
+ if (lock->l_resource->lr_type != LDLM_IBITS)
+ RETURN(0);
+
+ RETURN(1);
+}
+
+/**
* Initialize OSP device according to the parameters in the configuration
* log \a cfg.
*
ENTRY;
mutex_init(&osp->opd_async_requests_mutex);
+ INIT_LIST_HEAD(&osp->opd_async_updates);
+ init_rwsem(&osp->opd_async_updates_rwsem);
+ atomic_set(&osp->opd_async_updates_count, 0);
obd = class_name2obd(lustre_cfg_string(cfg, 0));
if (obd == NULL) {
* used in orphan cleanup. */
rc = osp_last_used_init(env, osp);
if (rc)
- GOTO(out_proc, rc);
+ GOTO(out_fid, rc);
/* Initialize precreation thread, it handles new
rc = osp_init_precreate(osp);
if (rc)
GOTO(out_last_used, rc);
+
+ /*
+ * Initialize synhronization mechanism taking
+ * care of propogating changes to OST in near
+ * transactional manner.
+ */
+ rc = osp_sync_init(env, osp);
+ if (rc < 0)
+ GOTO(out_precreat, rc);
}
- /*
- * Initialize synhronization mechanism taking
- * care of propogating changes to OST in near
- * transactional manner.
- */
- rc = osp_sync_init(env, osp);
- if (rc)
- GOTO(out_precreat, rc);
+ ns_register_cancel(obd->obd_namespace, osp_cancel_weight);
/*
* Initiate connect to OST
RETURN(0);
out:
- /* stop sync thread */
- osp_sync_fini(osp);
+ if (!osp->opd_connect_mdt)
+ /* stop sync thread */
+ osp_sync_fini(osp);
out_precreat:
/* stop precreate thread */
if (!osp->opd_connect_mdt)
out_last_used:
if (!osp->opd_connect_mdt)
osp_last_used_fini(env, osp);
+out_fid:
+ obd_fid_fini(osp->opd_obd);
out_proc:
ptlrpc_lprocfs_unregister_obd(obd);
lprocfs_obd_cleanup(obd);
ENTRY;
if (osp->opd_async_requests != NULL) {
- out_destroy_update_req(osp->opd_async_requests);
+ dt_update_request_destroy(osp->opd_async_requests);
osp->opd_async_requests = NULL;
}
if (rc) {
CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
GOTO(out, rc);
+ } else {
+ osp->opd_obd->u.cli.cl_seq->lcs_exp =
+ class_export_get(osp->opd_exp);
}
ptlrpc_pinger_add_import(imp);
}
/**
- * Prepare fid client.
- *
- * This function prepares the FID client for the OSP. It will check and assign
- * the export (to MDT0) for its FID client, so OSP can allocate super sequence
- * or lookup sequence in FLDB of MDT0.
- *
- * \param[in] osp OSP device
- */
-static void osp_prepare_fid_client(struct osp_device *osp)
-{
- LASSERT(osp->opd_obd->u.cli.cl_seq != NULL);
- if (osp->opd_obd->u.cli.cl_seq->lcs_exp != NULL)
- return;
-
- LASSERT(osp->opd_exp != NULL);
- osp->opd_obd->u.cli.cl_seq->lcs_exp =
- class_export_get(osp->opd_exp);
-}
-
-/**
* Implementation of obd_ops::o_import_event
*
* This function is called when some related import event happens. It will
case IMP_EVENT_ACTIVE:
d->opd_imp_active = 1;
- osp_prepare_fid_client(d);
if (d->opd_got_disconnected)
d->opd_new_connection = 1;
d->opd_imp_connected = 1;
* \param[in] key the key
* \param[out] vallen length of \a val
* \param[out] val holds the value returned by the key
- * \param[in] unused necessary for the interface but unused
*
* \retval 0 0 if getting information succeeded.
* \retval negative negative errno if getting information failed.
*/
static int osp_obd_get_info(const struct lu_env *env, struct obd_export *exp,
- __u32 keylen, void *key, __u32 *vallen, void *val,
- struct lov_stripe_md *unused)
+ __u32 keylen, void *key, __u32 *vallen, void *val)
{
int rc = -EINVAL;
* requested from seq meta server
* \retval negative negative errno if FID allocation failed.
*/
-int osp_fid_alloc(const struct lu_env *env, struct obd_export *exp,
- struct lu_fid *fid, struct md_op_data *unused)
+static int osp_fid_alloc(const struct lu_env *env, struct obd_export *exp,
+ struct lu_fid *fid, struct md_op_data *unused)
{
struct client_obd *cli = &exp->exp_obd->u.cli;
struct osp_device *osp = lu2osp_dev(exp->exp_obd->obd_lu_dev);
LASSERT(osp->opd_obd->u.cli.cl_seq != NULL);
/* Sigh, fid client is not ready yet */
- if (osp->opd_obd->u.cli.cl_seq->lcs_exp == NULL)
- RETURN(-ENOTCONN);
+ LASSERT(osp->opd_obd->u.cli.cl_seq->lcs_exp != NULL);
RETURN(seq_client_alloc_fid(env, seq, fid));
}
rc = class_register_type(&osp_obd_device_ops, NULL, true, NULL,
-#ifndef HAVE_ONLY_PROCFS_SEQ
- NULL,
-#endif
LUSTRE_OSP_NAME, &osp_device_type);
if (rc != 0) {
lu_kmem_fini(osp_caches);
}
rc = class_register_type(&lwp_obd_device_ops, NULL, true, NULL,
-#ifndef HAVE_ONLY_PROCFS_SEQ
- NULL,
-#endif
LUSTRE_LWP_NAME, &lwp_device_type);
if (rc != 0) {
class_unregister_type(LUSTRE_OSP_NAME);
return rc;
type = class_search_type(LUSTRE_OSP_NAME);
- type->typ_procsym = lprocfs_seq_register("osc", proc_lustre_root,
- NULL, NULL);
+ type->typ_procsym = lprocfs_register("osc", proc_lustre_root,
+ NULL, NULL);
if (IS_ERR(type->typ_procsym)) {
CERROR("osp: can't create compat entry \"osc\": %d\n",
(int) PTR_ERR(type->typ_procsym));
MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
+MODULE_VERSION(LUSTRE_VERSION_STRING);
MODULE_LICENSE("GPL");
-cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);
+module_init(osp_mod_init);
+module_exit(osp_mod_exit);
git://git.whamcloud.com / fs / lustre-release.git / blobdiff