X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosp%2Flwp_dev.c;h=a5ecc88aa32565198464686ebe861ff247f63411;hp=fce82a23f6e1b6ba6e11ed5bf5c88712e04f07b9;hb=b046468f58a1f40e85cb59ed9abf75fd2fd5ea5a;hpb=911b6167a37c2dba835baab4e100dcf206c03113

diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c
index fce82a2..a5ecc88 100644
--- a/lustre/osp/lwp_dev.c
+++ b/lustre/osp/lwp_dev.c
@@ -21,13 +21,18 @@
  * GPL HEADER END
  */
 /*
- * Copyright (c) 2013, Intel Corporation.
+ * Copyright (c) 2013, 2014, Intel Corporation.
  * Use is subject to license terms.
  *
  * lustre/osp/lwp_dev.c
  *
- * Light Weight Proxy, which is just for managing the connection established
- * from OSTs/MDTs to MDT0.
+ * This file provides code related to the Light Weight Proxy (LWP) managing
+ * the connections established from OST to MDT, and MDT to MDT0.
+ *
+ * A LWP connection is used to send quota and FLD query requests. It's not
+ * recoverable, which means target server doesn't have an on-disk record in
+ * the last_rcvd file to remember the connection. Once LWP reconnect after
+ * server reboot, server will always regard it as a new connection.
  *
  * Author: <di.wang@intel.com>
  * Author: <yawei.niu@intel.com>
@@ -38,12 +43,14 @@
 #include <lustre_param.h>
 #include <lustre_log.h>
 
+#include "osp_internal.h"
+
 struct lwp_device {
 	struct lu_device	lpd_dev;
-	struct obd_device	*lpd_obd;
-	struct obd_uuid		lpd_cluuid;
-	struct obd_export	*lpd_exp;
-	int			lpd_connects;
+	struct obd_device      *lpd_obd;   /* corresponding OBD device */
+	struct obd_uuid		lpd_cluuid;/* UUID of LWP */
+	struct obd_export      *lpd_exp;   /* export of LWP */
+	int			lpd_connects; /* use count, 0 or 1 */
 };
 
 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
@@ -56,40 +63,27 @@ static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
 	return &d->lpd_dev;
 }
 
-static int lwp_name2fsname(char *lwpname, char *fsname)
-{
-	char *ptr;
-
-	LASSERT(lwpname != NULL);
-	LASSERT(fsname != NULL);
-
-	sprintf(fsname, "-%s-", LUSTRE_LWP_NAME);
-
-	ptr = strstr(lwpname, fsname);
-	if (ptr == NULL)
-		return -EINVAL;
-
-	while (*(--ptr) != '-') {
-		if (ptr == lwpname)
-			return -EINVAL;
-	}
-
-	strncpy(fsname, lwpname, ptr - lwpname);
-	fsname[ptr - lwpname] = '\0';
-
-	return 0;
-}
-
+/**
+ * Setup LWP device.
+ *
+ * \param[in] env	environment passed by caller
+ * \param[in] lwp	LWP device to be setup
+ * \param[in] nidstring	remote target NID
+ *
+ * \retval		0 on success
+ * \retval		negative number on error
+ */
 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
 		     char *nidstring)
 {
 	struct lustre_cfg_bufs	*bufs = NULL;
 	struct lustre_cfg	*lcfg = NULL;
-	char			*lwpname = lwp->lpd_obd->obd_name;
-	char			*fsname = NULL;
+	char			*lwp_name = lwp->lpd_obd->obd_name;
 	char			*server_uuid = NULL;
+	char			*ptr;
 	class_uuid_t		 uuid;
 	struct obd_import	*imp;
+	int			 len = strlen(lwp_name) + 1;
 	int			 rc;
 	ENTRY;
 
@@ -97,23 +91,22 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
 	if (bufs == NULL)
 		RETURN(-ENOMEM);
 
-	OBD_ALLOC(fsname, strlen(lwpname));
-	if (fsname == NULL)
+	OBD_ALLOC(server_uuid, len);
+	if (server_uuid == NULL)
 		GOTO(out, rc = -ENOMEM);
 
-	rc = lwp_name2fsname(lwpname, fsname);
-	if (rc) {
-		CERROR("%s: failed to get fsname from lwpname. %d\n",
-		       lwpname, rc);
-		GOTO(out, rc);
+	snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME);
+	ptr = cfs_strrstr(lwp_name, server_uuid);
+	if (ptr == NULL) {
+		CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
+		       lwp_name, -EINVAL);
+		GOTO(out, rc = -EINVAL);
 	}
 
-	OBD_ALLOC(server_uuid, strlen(fsname) + 15);
-	if (server_uuid == NULL)
-		GOTO(out, rc = -ENOMEM);
-
-	sprintf(server_uuid, "%s-MDT0000_UUID", fsname);
-	lustre_cfg_bufs_reset(bufs, lwpname);
+	strncpy(server_uuid, lwp_name, ptr - lwp_name);
+	server_uuid[ptr - lwp_name] = '\0';
+	strlcat(server_uuid, "_UUID", len);
+	lustre_cfg_bufs_reset(bufs, lwp_name);
 	lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
 	lustre_cfg_bufs_set_string(bufs, 2, nidstring);
 	lcfg = lustre_cfg_new(LCFG_SETUP, bufs);
@@ -138,9 +131,7 @@ out:
 	if (bufs != NULL)
 		OBD_FREE_PTR(bufs);
 	if (server_uuid != NULL)
-		OBD_FREE(server_uuid, strlen(fsname) + 15);
-	if (fsname != NULL)
-		OBD_FREE(fsname, strlen(lwpname));
+		OBD_FREE(server_uuid, len);
 	if (lcfg != NULL)
 		lustre_cfg_free(lcfg);
 	if (rc)
@@ -149,16 +140,27 @@ out:
 	RETURN(rc);
 }
 
-int lwp_disconnect(struct lwp_device *d)
+/**
+ * Disconnect the import from LWP.
+ *
+ * \param[in] d		LWP device to be disconnected
+ *
+ * \retval		0 on success
+ * \retval		negative number on error
+ */
+static int lwp_disconnect(struct lwp_device *d)
 {
 	struct obd_import *imp;
 	int rc = 0;
 
 	imp = d->lpd_obd->u.cli.cl_import;
 
-	/* Mark import deactivated now, so we don't try to reconnect if any
+	/*
+	 * Mark import deactivated now, so we don't try to reconnect if any
 	 * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
-	 * fully deactivate the import, or that would drop all requests. */
+	 * fully deactivate the import because that would cause all requests
+	 * to be dropped.
+	 */
 	LASSERT(imp != NULL);
 	spin_lock(&imp->imp_lock);
 	imp->imp_deactive = 1;
@@ -166,21 +168,34 @@ int lwp_disconnect(struct lwp_device *d)
 
 	ptlrpc_deactivate_import(imp);
 
-	/* Some non-replayable imports (MDS's OSCs) are pinged, so just
+	/*
+	 * Some non-replayable imports (MDS's OSCs) are pinged, so just
 	 * delete it regardless.  (It's safe to delete an import that was
-	 * never added.) */
-	(void)ptlrpc_pinger_del_import(imp);
-
+	 * never added.)
+	 */
+	ptlrpc_pinger_del_import(imp);
 	rc = ptlrpc_disconnect_import(imp, 0);
-	if (rc && rc != -ETIMEDOUT)
-		CERROR("%s: can't disconnect: rc = %d\n",
-		       d->lpd_obd->obd_name, rc);
+	if (rc != 0)
+		CWARN("%s: can't disconnect: rc = %d\n",
+		      d->lpd_obd->obd_name, rc);
 
 	ptlrpc_invalidate_import(imp);
 
 	RETURN(rc);
 }
 
+/**
+ * Implementation of lu_device_operations::ldo_process_config.
+ *
+ * Process a Lustre configuration request.
+ *
+ * \param[in] env	environment passed by caller
+ * \param[in] dev	device to be processed
+ * \param[in] lcfg	lustre_cfg, LCFG_PRE_CLEANUP or LCFG_CLEANUP
+ *
+ * \retval		0 on success
+ * \retval		negative number on error
+ */
 static int lwp_process_config(const struct lu_env *env,
 			      struct lu_device *dev, struct lustre_cfg *lcfg)
 {
@@ -206,29 +221,25 @@ static int lwp_process_config(const struct lu_env *env,
 	RETURN(rc);
 }
 
-const struct lu_device_operations lwp_lu_ops = {
+static const struct lu_device_operations lwp_lu_ops = {
 	.ldo_process_config	= lwp_process_config,
 };
 
-static struct lprocfs_vars lprocfs_lwp_module_vars[] = {
-	{ "num_refs",		lprocfs_rd_numrefs, 0, 0 },
-	{ 0 }
-};
-
-static struct lprocfs_vars lprocfs_lwp_obd_vars[] = {
-	{ 0 }
-};
-
-void lprocfs_lwp_init_vars(struct lprocfs_static_vars *lvars)
-{
-	lvars->module_vars = lprocfs_lwp_module_vars;
-	lvars->obd_vars = lprocfs_lwp_obd_vars;
-}
-
-int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
-	      struct lu_device_type *ldt, struct lustre_cfg *cfg)
+/**
+ * Initialize LWP device.
+ *
+ * \param[in] env	environment passed by caller
+ * \param[in] lwp	device to be initialized
+ * \param[in] ldt	not used
+ * \param[in] cfg	lustre_cfg contains remote target uuid
+ *
+ * \retval		0 on success
+ * \retval		-ENODEV if the device name cannot be found
+ * \retval		negative numbers on other errors
+ */
+static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
+		     struct lu_device_type *ldt, struct lustre_cfg *cfg)
 {
-	struct lprocfs_static_vars lvars = { 0 };
 	int			   rc;
 	ENTRY;
 
@@ -257,20 +268,33 @@ int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
 		RETURN(rc);
 	}
 
-	lprocfs_lwp_init_vars(&lvars);
-	if (lprocfs_obd_setup(lwp->lpd_obd, lvars.obd_vars) == 0)
+	if (lprocfs_obd_setup(lwp->lpd_obd) == 0) {
+		sptlrpc_lprocfs_cliobd_attach(lwp->lpd_obd);
 		ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
+	}
 
 	RETURN(0);
 }
 
+/**
+ * Implementation of lu_device_type_operations::ldto_device_free.
+ *
+ * Free a LWP device.
+ *
+ * \param[in] env	environment passed by caller
+ * \param[in] lu	device to be freed
+ *
+ * \retval		NULL to indicate that this is the bottom device
+ *			of the stack and there are no more devices
+ *			below this one to be cleaned up.
+ */
 static struct lu_device *lwp_device_free(const struct lu_env *env,
 					 struct lu_device *lu)
 {
 	struct lwp_device *m = lu2lwp_dev(lu);
 	ENTRY;
 
-	if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
+	if (atomic_read(&lu->ld_ref) && lu->ld_site) {
 		LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 		lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 	}
@@ -279,35 +303,57 @@ static struct lu_device *lwp_device_free(const struct lu_env *env,
 	RETURN(NULL);
 }
 
+/**
+ * Implementation of lu_device_type_operations::ldto_device_alloc.
+ *
+ * Allocate a LWP device.
+ *
+ * \param[in] env	environment passed by caller
+ * \param[in] ldt	device type whose name is LUSTRE_LWP_NAME
+ * \param[in] lcfg	lustre_cfg contains remote target UUID
+ *
+ * \retval		pointer of allocated LWP device on success
+ * \retval		ERR_PTR(errno) on error
+ */
 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
-					  struct lu_device_type *t,
+					  struct lu_device_type *ldt,
 					  struct lustre_cfg *lcfg)
 {
 	struct lwp_device *lwp;
-	struct lu_device  *l;
+	struct lu_device  *ludev;
 
 	OBD_ALLOC_PTR(lwp);
 	if (lwp == NULL) {
-		l = ERR_PTR(-ENOMEM);
+		ludev = ERR_PTR(-ENOMEM);
 	} else {
 		int rc;
 
-		l = lwp2lu_dev(lwp);
-		lu_device_init(&lwp->lpd_dev, t);
-		rc = lwp_init0(env, lwp, t, lcfg);
+		ludev = lwp2lu_dev(lwp);
+		lu_device_init(&lwp->lpd_dev, ldt);
+		rc = lwp_init0(env, lwp, ldt, lcfg);
 		if (rc != 0) {
-			lwp_device_free(env, l);
-			l = ERR_PTR(rc);
+			lwp_device_free(env, ludev);
+			ludev = ERR_PTR(rc);
 		}
 	}
-	return l;
+	return ludev;
 }
 
 
+/**
+ * Implementation of lu_device_type_operations::ltdo_device_fini.
+ *
+ * Finalize LWP device.
+ *
+ * \param[in] env	environment passed by caller
+ * \param[in] ludev	device to be finalized
+ *
+ * \retval		NULL on success
+ */
 static struct lu_device *lwp_device_fini(const struct lu_env *env,
-					 struct lu_device *d)
+					 struct lu_device *ludev)
 {
-	struct lwp_device *m = lu2lwp_dev(d);
+	struct lwp_device *m = lu2lwp_dev(ludev);
 	struct obd_import *imp;
 	int                rc;
 	ENTRY;
@@ -317,11 +363,6 @@ static struct lu_device *lwp_device_fini(const struct lu_env *env,
 
 	imp = m->lpd_obd->u.cli.cl_import;
 
-	if (imp->imp_rq_pool) {
-		ptlrpc_free_rq_pool(imp->imp_rq_pool);
-		imp->imp_rq_pool = NULL;
-	}
-
 	LASSERT(m->lpd_obd);
 	ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
 	lprocfs_obd_cleanup(m->lpd_obd);
@@ -347,13 +388,29 @@ struct lu_device_type lwp_device_type = {
 	.ldt_ctx_tags = LCT_MD_THREAD
 };
 
+/**
+ * Implementation of OBD device operations obd_ops::o_connect.
+ *
+ * Create export for LWP, and connect to target server.
+ *
+ * \param[in] env	the environment passed by caller
+ * \param[out] exp	export for the connection to be established
+ * \param[in] obd	OBD device to perform the connect on
+ * \param[in] cluuid	UUID of the OBD device
+ * \param[in] data	connect data containing compatibility flags
+ * \param[in] localdata	not used
+ *
+ * \retval		0 on success
+ * \retval		negative number on error
+ */
 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 			   struct obd_device *obd, struct obd_uuid *cluuid,
 			   struct obd_connect_data *data, void *localdata)
 {
 	struct lwp_device       *lwp = lu2lwp_dev(obd->obd_lu_dev);
+	struct client_obd	*cli = &lwp->lpd_obd->u.cli;
+	struct obd_import       *imp = cli->cl_import;
 	struct obd_connect_data *ocd;
-	struct obd_import       *imp;
 	struct lustre_handle     conn;
 	int                      rc;
 
@@ -361,19 +418,22 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 
 	CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
 
+	*exp = NULL;
+	down_write(&cli->cl_sem);
 	rc = class_connect(&conn, obd, cluuid);
-	if (rc)
-		RETURN(rc);
+	if (rc != 0)
+		GOTO(out_sem, rc);
 
 	*exp = class_conn2export(&conn);
 	lwp->lpd_exp = *exp;
 
-	/* Why should there ever be more than 1 connect? */
 	lwp->lpd_connects++;
 	LASSERT(lwp->lpd_connects == 1);
 
-	imp = lwp->lpd_obd->u.cli.cl_import;
 	imp->imp_dlm_handle = conn;
+	rc = ptlrpc_init_import(imp);
+	if (rc != 0)
+		GOTO(out_dis, rc);
 
 	LASSERT(data != NULL);
 	ocd = &imp->imp_connect_data;
@@ -385,17 +445,39 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 	imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 
 	rc = ptlrpc_connect_import(imp);
-	if (rc) {
+	if (rc != 0) {
 		CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
-		GOTO(out, rc);
+		GOTO(out_dis, rc);
 	}
 
 	ptlrpc_pinger_add_import(imp);
 
-out:
-	RETURN(rc);
+	GOTO(out_dis, rc = 0);
+
+out_dis:
+	if (rc != 0) {
+		class_disconnect(*exp);
+		*exp = NULL;
+		lwp->lpd_exp = NULL;
+	}
+
+out_sem:
+	up_write(&cli->cl_sem);
+
+	return rc;
 }
 
+/**
+ * Implementation of OBD device operations obd_ops::o_disconnect.
+ *
+ * Release export for the LWP. Only disconnect the underlying layers
+ * on the final disconnect.
+ *
+ * \param[in] exp	the export to perform disconnect on
+ *
+ * \retval		0 on success
+ * \retval		negative number on error
+ */
 static int lwp_obd_disconnect(struct obd_export *exp)
 {
 	struct obd_device *obd = exp->exp_obd;
@@ -403,7 +485,6 @@ static int lwp_obd_disconnect(struct obd_export *exp)
 	int                rc;
 	ENTRY;
 
-	/* Only disconnect the underlying layers on the final disconnect. */
 	LASSERT(lwp->lpd_connects == 1);
 	lwp->lpd_connects--;
 
@@ -415,6 +496,16 @@ static int lwp_obd_disconnect(struct obd_export *exp)
 	RETURN(rc);
 }
 
+/**
+ * Handle import events for the LWP device.
+ *
+ * \param[in] obd	OBD device associated with the import
+ * \param[in] imp	the import which event happened on
+ * \param[in] event	event type
+ *
+ * \retval		0 on success
+ * \retval		negative number on error
+ */
 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
 			    enum obd_import_event event)
 {