4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 021110-1307, USA
24 * Copyright (c) 2013, 2014, Intel Corporation.
25 * Use is subject to license terms.
27 * lustre/osp/lwp_dev.c
29 * This file provides code related to the Light Weight Proxy (LWP) managing
30 * the connections established from OST to MDT, and MDT to MDT0.
32 * A LWP connection is used to send quota and FLD query requests. It's not
33 * recoverable, which means target server doesn't have an on-disk record in
34 * the last_rcvd file to remember the connection. Once LWP reconnect after
35 * server reboot, server will always regard it as a new connection.
37 * Author: <di.wang@intel.com>
38 * Author: <yawei.niu@intel.com>
40 #define DEBUG_SUBSYSTEM S_OST
42 #include <obd_class.h>
43 #include <lustre_param.h>
44 #include <lustre_log.h>
46 #include "osp_internal.h"
49 struct lu_device lpd_dev;
50 struct obd_device *lpd_obd; /* corresponding OBD device */
51 struct obd_uuid lpd_cluuid;/* UUID of LWP */
52 struct obd_export *lpd_exp; /* export of LWP */
53 int lpd_connects; /* use count, 0 or 1 */
56 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
58 return container_of0(d, struct lwp_device, lpd_dev);
61 static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
69 * \param[in] env environment passed by caller
70 * \param[in] lwp LWP device to be setup
71 * \param[in] nidstring remote target NID
73 * \retval 0 on success
74 * \retval negative number on error
76 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
79 struct lustre_cfg_bufs *bufs = NULL;
80 struct lustre_cfg *lcfg = NULL;
81 char *lwp_name = lwp->lpd_obd->obd_name;
82 char *server_uuid = NULL;
85 struct obd_import *imp;
86 int len = strlen(lwp_name) + 1;
94 OBD_ALLOC(server_uuid, len);
95 if (server_uuid == NULL)
96 GOTO(out, rc = -ENOMEM);
98 snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME);
99 ptr = cfs_strrstr(lwp_name, server_uuid);
101 CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
103 GOTO(out, rc = -EINVAL);
106 strncpy(server_uuid, lwp_name, ptr - lwp_name);
107 server_uuid[ptr - lwp_name] = '\0';
108 strlcat(server_uuid, "_UUID", len);
109 lustre_cfg_bufs_reset(bufs, lwp_name);
110 lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
111 lustre_cfg_bufs_set_string(bufs, 2, nidstring);
112 lcfg = lustre_cfg_new(LCFG_SETUP, bufs);
114 GOTO(out, rc = -ENOMEM);
116 rc = client_obd_setup(lwp->lpd_obd, lcfg);
118 CERROR("%s: client obd setup error: rc = %d\n",
119 lwp->lpd_obd->obd_name, rc);
123 imp = lwp->lpd_obd->u.cli.cl_import;
124 rc = ptlrpc_init_import(imp);
128 ll_generate_random_uuid(uuid);
129 class_uuid_unparse(uuid, &lwp->lpd_cluuid);
133 if (server_uuid != NULL)
134 OBD_FREE(server_uuid, len);
136 lustre_cfg_free(lcfg);
138 client_obd_cleanup(lwp->lpd_obd);
144 * Disconnect the import from LWP.
146 * \param[in] d LWP device to be disconnected
148 * \retval 0 on success
149 * \retval negative number on error
151 static int lwp_disconnect(struct lwp_device *d)
153 struct obd_import *imp;
156 imp = d->lpd_obd->u.cli.cl_import;
159 * Mark import deactivated now, so we don't try to reconnect if any
160 * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't
161 * fully deactivate the import because that would cause all requests
164 LASSERT(imp != NULL);
165 spin_lock(&imp->imp_lock);
166 imp->imp_deactive = 1;
167 spin_unlock(&imp->imp_lock);
169 ptlrpc_deactivate_import(imp);
172 * Some non-replayable imports (MDS's OSCs) are pinged, so just
173 * delete it regardless. (It's safe to delete an import that was
176 ptlrpc_pinger_del_import(imp);
177 rc = ptlrpc_disconnect_import(imp, 0);
179 CWARN("%s: can't disconnect: rc = %d\n",
180 d->lpd_obd->obd_name, rc);
182 ptlrpc_invalidate_import(imp);
188 * Implementation of lu_device_operations::ldo_process_config.
190 * Process a Lustre configuration request.
192 * \param[in] env environment passed by caller
193 * \param[in] dev device to be processed
194 * \param[in] lcfg lustre_cfg, LCFG_PRE_CLEANUP or LCFG_CLEANUP
196 * \retval 0 on success
197 * \retval negative number on error
199 static int lwp_process_config(const struct lu_env *env,
200 struct lu_device *dev, struct lustre_cfg *lcfg)
202 struct lwp_device *d = lu2lwp_dev(dev);
206 switch (lcfg->lcfg_command) {
207 case LCFG_PRE_CLEANUP:
209 rc = lwp_disconnect(d);
215 CERROR("%s: unknown command %u\n",
216 (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
224 static const struct lu_device_operations lwp_lu_ops = {
225 .ldo_process_config = lwp_process_config,
229 * Initialize LWP device.
231 * \param[in] env environment passed by caller
232 * \param[in] lwp device to be initialized
233 * \param[in] ldt not used
234 * \param[in] cfg lustre_cfg contains remote target uuid
236 * \retval 0 on success
237 * \retval -ENODEV if the device name cannot be found
238 * \retval negative numbers on other errors
240 static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
241 struct lu_device_type *ldt, struct lustre_cfg *cfg)
246 lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
247 if (lwp->lpd_obd == NULL) {
248 CERROR("Cannot find obd with name %s\n",
249 lustre_cfg_string(cfg, 0));
253 lwp->lpd_dev.ld_ops = &lwp_lu_ops;
254 lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev;
256 rc = ptlrpcd_addref();
258 CERROR("%s: ptlrpcd addref error: rc =%d\n",
259 lwp->lpd_obd->obd_name, rc);
263 rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1));
265 CERROR("%s: setup lwp failed. %d\n",
266 lwp->lpd_obd->obd_name, rc);
271 if (lprocfs_obd_setup(lwp->lpd_obd) == 0)
272 ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
278 * Implementation of lu_device_type_operations::ldto_device_free.
282 * \param[in] env environment passed by caller
283 * \param[in] lu device to be freed
285 * \retval NULL to indicate that this is the bottom device
286 * of the stack and there are no more devices
287 * below this one to be cleaned up.
289 static struct lu_device *lwp_device_free(const struct lu_env *env,
290 struct lu_device *lu)
292 struct lwp_device *m = lu2lwp_dev(lu);
295 if (atomic_read(&lu->ld_ref) && lu->ld_site) {
296 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
297 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
299 lu_device_fini(&m->lpd_dev);
305 * Implementation of lu_device_type_operations::ldto_device_alloc.
307 * Allocate a LWP device.
309 * \param[in] env environment passed by caller
310 * \param[in] ldt device type whose name is LUSTRE_LWP_NAME
311 * \param[in] lcfg lustre_cfg contains remote target UUID
313 * \retval pointer of allocated LWP device on success
314 * \retval ERR_PTR(errno) on error
316 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
317 struct lu_device_type *ldt,
318 struct lustre_cfg *lcfg)
320 struct lwp_device *lwp;
321 struct lu_device *ludev;
325 ludev = ERR_PTR(-ENOMEM);
329 ludev = lwp2lu_dev(lwp);
330 lu_device_init(&lwp->lpd_dev, ldt);
331 rc = lwp_init0(env, lwp, ldt, lcfg);
333 lwp_device_free(env, ludev);
342 * Implementation of lu_device_type_operations::ltdo_device_fini.
344 * Finalize LWP device.
346 * \param[in] env environment passed by caller
347 * \param[in] ludev device to be finalized
349 * \retval NULL on success
351 static struct lu_device *lwp_device_fini(const struct lu_env *env,
352 struct lu_device *ludev)
354 struct lwp_device *m = lu2lwp_dev(ludev);
355 struct obd_import *imp;
359 if (m->lpd_exp != NULL)
360 class_disconnect(m->lpd_exp);
362 imp = m->lpd_obd->u.cli.cl_import;
364 if (imp->imp_rq_pool) {
365 ptlrpc_free_rq_pool(imp->imp_rq_pool);
366 imp->imp_rq_pool = NULL;
370 ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
371 lprocfs_obd_cleanup(m->lpd_obd);
373 rc = client_obd_cleanup(m->lpd_obd);
374 LASSERTF(rc == 0, "error %d\n", rc);
381 static struct lu_device_type_operations lwp_device_type_ops = {
382 .ldto_device_alloc = lwp_device_alloc,
383 .ldto_device_free = lwp_device_free,
384 .ldto_device_fini = lwp_device_fini
387 struct lu_device_type lwp_device_type = {
388 .ldt_tags = LU_DEVICE_DT,
389 .ldt_name = LUSTRE_LWP_NAME,
390 .ldt_ops = &lwp_device_type_ops,
391 .ldt_ctx_tags = LCT_MD_THREAD
395 * Implementation of OBD device operations obd_ops::o_connect.
397 * Create export for LWP, and connect to target server.
399 * \param[in] env the environment passed by caller
400 * \param[out] exp export for the connection to be established
401 * \param[in] obd OBD device to perform the connect on
402 * \param[in] cluuid UUID of the OBD device
403 * \param[in] data connect data containing compatibility flags
404 * \param[in] localdata not used
406 * \retval 0 on success
407 * \retval negative number on error
409 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
410 struct obd_device *obd, struct obd_uuid *cluuid,
411 struct obd_connect_data *data, void *localdata)
413 struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
414 struct client_obd *cli = &lwp->lpd_obd->u.cli;
415 struct obd_import *imp = cli->cl_import;
416 struct obd_connect_data *ocd;
417 struct lustre_handle conn;
422 CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
425 down_write(&cli->cl_sem);
426 rc = class_connect(&conn, obd, cluuid);
430 *exp = class_conn2export(&conn);
434 LASSERT(lwp->lpd_connects == 1);
436 imp->imp_dlm_handle = conn;
437 rc = ptlrpc_init_import(imp);
441 LASSERT(data != NULL);
442 ocd = &imp->imp_connect_data;
445 LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT);
447 ocd->ocd_version = LUSTRE_VERSION_CODE;
448 imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
450 rc = ptlrpc_connect_import(imp);
452 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
456 ptlrpc_pinger_add_import(imp);
458 GOTO(out_dis, rc = 0);
462 class_disconnect(*exp);
468 up_write(&cli->cl_sem);
474 * Implementation of OBD device operations obd_ops::o_disconnect.
476 * Release export for the LWP. Only disconnect the underlying layers
477 * on the final disconnect.
479 * \param[in] exp the export to perform disconnect on
481 * \retval 0 on success
482 * \retval negative number on error
484 static int lwp_obd_disconnect(struct obd_export *exp)
486 struct obd_device *obd = exp->exp_obd;
487 struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
491 LASSERT(lwp->lpd_connects == 1);
494 rc = class_disconnect(exp);
496 CERROR("%s: class disconnect error: rc = %d\n",
503 * Handle import events for the LWP device.
505 * \param[in] obd OBD device associated with the import
506 * \param[in] imp the import which event happened on
507 * \param[in] event event type
509 * \retval 0 on success
510 * \retval negative number on error
512 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
513 enum obd_import_event event)
516 case IMP_EVENT_DISCON:
517 case IMP_EVENT_INACTIVE:
518 case IMP_EVENT_ACTIVE:
520 case IMP_EVENT_INVALIDATE:
521 if (obd->obd_namespace == NULL)
523 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
528 CERROR("%s: unsupported import event: %#x\n",
529 obd->obd_name, event);
534 struct obd_ops lwp_obd_device_ops = {
535 .o_owner = THIS_MODULE,
536 .o_add_conn = client_import_add_conn,
537 .o_del_conn = client_import_del_conn,
538 .o_connect = lwp_obd_connect,
539 .o_disconnect = lwp_obd_disconnect,
540 .o_import_event = lwp_import_event,