4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 021110-1307, USA
24 * Copyright (c) 2013, 2017, Intel Corporation.
25 * Use is subject to license terms.
27 * lustre/osp/lwp_dev.c
29 * This file provides code related to the Light Weight Proxy (LWP) managing
30 * the connections established from OST to MDT, and MDT to MDT0.
32 * A LWP connection is used to send quota and FLD query requests. It's not
33 * recoverable, which means target server doesn't have an on-disk record in
34 * the last_rcvd file to remember the connection. Once LWP reconnect after
35 * server reboot, server will always regard it as a new connection.
37 * Author: <di.wang@intel.com>
38 * Author: <yawei.niu@intel.com>
40 #define DEBUG_SUBSYSTEM S_OST
42 #include <obd_class.h>
43 #include <uapi/linux/lustre/lustre_param.h>
44 #include <lustre_log.h>
45 #include <linux/kthread.h>
47 #include "osp_internal.h"
50 struct lu_device lpd_dev;
51 struct obd_device *lpd_obd; /* corresponding OBD device */
52 struct obd_export *lpd_exp; /* export of LWP */
53 struct task_struct *lpd_notify_task; /* notify thread */
54 int lpd_connects; /* use count, 0 or 1 */
57 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
59 return container_of_safe(d, struct lwp_device, lpd_dev);
62 static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
70 * \param[in] env environment passed by caller
71 * \param[in] lwp LWP device to be setup
72 * \param[in] nidstring remote target NID
74 * \retval 0 on success
75 * \retval negative number on error
77 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
80 struct lustre_cfg_bufs *bufs = NULL;
81 struct lustre_cfg *lcfg = NULL;
82 char *lwp_name = lwp->lpd_obd->obd_name;
83 char *server_uuid = NULL;
86 struct obd_import *imp;
87 int len = strlen(lwp_name) + 1;
89 const char *lwp_marker = "-" LUSTRE_LWP_NAME "-";
92 lwp->lpd_notify_task = NULL;
98 OBD_ALLOC(server_uuid, len);
99 if (server_uuid == NULL)
100 GOTO(out, rc = -ENOMEM);
103 while (ptr && (ptr = strstr(ptr+1, lwp_marker)) != NULL)
104 uuid_len = ptr - lwp_name;
107 CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
109 GOTO(out, rc = -EINVAL);
112 strncpy(server_uuid, lwp_name, uuid_len);
113 strlcat(server_uuid, "_UUID", len);
114 lustre_cfg_bufs_reset(bufs, lwp_name);
115 lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
116 lustre_cfg_bufs_set_string(bufs, 2, nidstring);
117 OBD_ALLOC(lcfg, lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen));
119 GOTO(out, rc = -ENOMEM);
120 lustre_cfg_init(lcfg, LCFG_SETUP, bufs);
122 rc = client_obd_setup(lwp->lpd_obd, lcfg);
124 CERROR("%s: client obd setup error: rc = %d\n",
125 lwp->lpd_obd->obd_name, rc);
129 imp = lwp->lpd_obd->u.cli.cl_import;
130 rc = ptlrpc_init_import(imp);
134 if (server_uuid != NULL)
135 OBD_FREE(server_uuid, len);
137 OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount,
138 lcfg->lcfg_buflens));
140 client_obd_cleanup(lwp->lpd_obd);
146 * Disconnect the import from LWP.
148 * \param[in] d LWP device to be disconnected
150 * \retval 0 on success
151 * \retval negative number on error
153 static int lwp_disconnect(struct lwp_device *d)
155 struct obd_import *imp;
158 imp = d->lpd_obd->u.cli.cl_import;
161 * Mark import deactivated now, so we don't try to reconnect if any
162 * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't
163 * fully deactivate the import because that would cause all requests
166 LASSERT(imp != NULL);
167 spin_lock(&imp->imp_lock);
168 imp->imp_deactive = 1;
169 spin_unlock(&imp->imp_lock);
171 ptlrpc_deactivate_import(imp);
174 * Some non-replayable imports (MDS's OSCs) are pinged, so just
175 * delete it regardless. (It's safe to delete an import that was
178 ptlrpc_pinger_del_import(imp);
179 rc = ptlrpc_disconnect_import(imp, 0);
181 CWARN("%s: can't disconnect: rc = %d\n",
182 d->lpd_obd->obd_name, rc);
184 ptlrpc_invalidate_import(imp);
190 * Implementation of lu_device_operations::ldo_process_config.
192 * Process a Lustre configuration request.
194 * \param[in] env environment passed by caller
195 * \param[in] dev device to be processed
196 * \param[in] lcfg lustre_cfg, LCFG_PRE_CLEANUP or LCFG_CLEANUP
198 * \retval 0 on success
199 * \retval negative number on error
201 static int lwp_process_config(const struct lu_env *env,
202 struct lu_device *dev, struct lustre_cfg *lcfg)
204 struct lwp_device *d = lu2lwp_dev(dev);
208 switch (lcfg->lcfg_command) {
209 case LCFG_PRE_CLEANUP:
211 rc = lwp_disconnect(d);
217 CERROR("%s: unknown command %u\n",
218 (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
226 static const struct lu_device_operations lwp_lu_ops = {
227 .ldo_process_config = lwp_process_config,
231 * Initialize LWP device.
233 * \param[in] env environment passed by caller
234 * \param[in] lwp device to be initialized
235 * \param[in] ldt not used
236 * \param[in] cfg lustre_cfg contains remote target uuid
238 * \retval 0 on success
239 * \retval -ENODEV if the device name cannot be found
240 * \retval negative numbers on other errors
242 static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
243 struct lu_device_type *ldt, struct lustre_cfg *cfg)
248 lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
249 if (lwp->lpd_obd == NULL) {
250 CERROR("Cannot find obd with name %s\n",
251 lustre_cfg_string(cfg, 0));
255 lwp->lpd_dev.ld_ops = &lwp_lu_ops;
256 lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev;
258 rc = ptlrpcd_addref();
260 CERROR("%s: ptlrpcd addref error: rc =%d\n",
261 lwp->lpd_obd->obd_name, rc);
265 rc = lprocfs_obd_setup(lwp->lpd_obd, true);
267 CERROR("%s: lprocfs_obd_setup failed. %d\n",
268 lwp->lpd_obd->obd_name, rc);
273 rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1));
275 CERROR("%s: setup lwp failed. %d\n",
276 lwp->lpd_obd->obd_name, rc);
277 lprocfs_obd_cleanup(lwp->lpd_obd);
282 rc = sptlrpc_lprocfs_cliobd_attach(lwp->lpd_obd);
284 CERROR("%s: sptlrpc_lprocfs_cliobd_attached failed. %d\n",
285 lwp->lpd_obd->obd_name, rc);
290 ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
296 * Implementation of lu_device_type_operations::ldto_device_free.
300 * \param[in] env environment passed by caller
301 * \param[in] lu device to be freed
303 * \retval NULL to indicate that this is the bottom device
304 * of the stack and there are no more devices
305 * below this one to be cleaned up.
307 static struct lu_device *lwp_device_free(const struct lu_env *env,
308 struct lu_device *lu)
310 struct lwp_device *m = lu2lwp_dev(lu);
313 lu_site_print(env, lu->ld_site, &lu->ld_ref, D_ERROR,
315 lu_device_fini(&m->lpd_dev);
321 * Implementation of lu_device_type_operations::ldto_device_alloc.
323 * Allocate a LWP device.
325 * \param[in] env environment passed by caller
326 * \param[in] ldt device type whose name is LUSTRE_LWP_NAME
327 * \param[in] lcfg lustre_cfg contains remote target UUID
329 * \retval pointer of allocated LWP device on success
330 * \retval ERR_PTR(errno) on error
332 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
333 struct lu_device_type *ldt,
334 struct lustre_cfg *lcfg)
336 struct lwp_device *lwp;
337 struct lu_device *ludev;
341 ludev = ERR_PTR(-ENOMEM);
345 ludev = lwp2lu_dev(lwp);
346 lu_device_init(&lwp->lpd_dev, ldt);
347 rc = lwp_init0(env, lwp, ldt, lcfg);
349 lwp_device_free(env, ludev);
358 * Implementation of lu_device_type_operations::ltdo_device_fini.
360 * Finalize LWP device.
362 * \param[in] env environment passed by caller
363 * \param[in] ludev device to be finalized
365 * \retval NULL on success
367 static struct lu_device *lwp_device_fini(const struct lu_env *env,
368 struct lu_device *ludev)
370 struct lwp_device *m = lu2lwp_dev(ludev);
371 struct task_struct *task = NULL;
375 task = xchg(&m->lpd_notify_task, NULL);
378 class_export_put(m->lpd_exp);
381 if (m->lpd_exp != NULL)
382 class_disconnect(m->lpd_exp);
385 rc = client_obd_cleanup(m->lpd_obd);
386 LASSERTF(rc == 0, "error %d\n", rc);
388 ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
395 static const struct lu_device_type_operations lwp_device_type_ops = {
396 .ldto_device_alloc = lwp_device_alloc,
397 .ldto_device_free = lwp_device_free,
398 .ldto_device_fini = lwp_device_fini
401 struct lu_device_type lwp_device_type = {
402 .ldt_tags = LU_DEVICE_DT,
403 .ldt_name = LUSTRE_LWP_NAME,
404 .ldt_ops = &lwp_device_type_ops,
405 .ldt_ctx_tags = LCT_MD_THREAD
408 static int lwp_notify_main(void *args)
410 struct obd_export *exp = (struct obd_export *)args;
411 struct lwp_device *lwp;
413 LASSERT(exp != NULL);
415 lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
417 lustre_notify_lwp_list(exp);
419 if (xchg(&lwp->lpd_notify_task, NULL) == NULL)
420 /* lwp_device_fini() is waiting for me
421 * Note that the wakeup comes direct from
422 * kthread_stop, not from wake_up_var().
423 * lwp_device_fini() will call class_export_put().
425 wait_var_event(lwp, kthread_should_stop());
427 class_export_put(exp);
433 * Some notify callbacks may cause deadlock in failover
434 * scenario, so we have to start thread to run callbacks
435 * asynchronously. See LU-6273.
437 static void lwp_notify_users(struct obd_export *exp)
439 struct lwp_device *lwp;
440 struct task_struct *task;
441 char name[MTI_NAME_MAXLEN];
443 LASSERT(exp != NULL);
444 lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
446 snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s",
447 exp->exp_obd->obd_name);
449 /* Notify happens only on LWP setup, so there shouldn't
450 * be notify thread running */
451 if (lwp->lpd_notify_task) {
452 CERROR("LWP notify thread: %s wasn't stopped\n", name);
456 task = kthread_create(lwp_notify_main, exp, name);
458 CERROR("Failed to start LWP notify thread:%s. %lu\n",
459 name, PTR_ERR(task));
461 lwp->lpd_notify_task = task;
462 class_export_get(exp);
463 wake_up_process(task);
468 * Implementation of OBD device operations obd_ops::o_connect.
470 * Create export for LWP, and connect to target server.
472 * \param[in] env the environment passed by caller
473 * \param[out] exp export for the connection to be established
474 * \param[in] obd OBD device to perform the connect on
475 * \param[in] cluuid UUID of the OBD device
476 * \param[in] data connect data containing compatibility flags
477 * \param[in] localdata not used
479 * \retval 0 on success
480 * \retval negative number on error
482 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
483 struct obd_device *obd, struct obd_uuid *cluuid,
484 struct obd_connect_data *data, void *localdata)
486 struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
487 struct client_obd *cli = &lwp->lpd_obd->u.cli;
488 struct obd_import *imp = cli->cl_import;
489 struct obd_connect_data *ocd;
490 struct lustre_handle conn;
495 CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
498 down_write(&cli->cl_sem);
499 rc = class_connect(&conn, obd, cluuid);
503 *exp = class_conn2export(&conn);
507 LASSERT(lwp->lpd_connects == 1);
509 imp->imp_dlm_handle = conn;
510 rc = ptlrpc_init_import(imp);
514 LASSERT(data != NULL);
515 ocd = &imp->imp_connect_data;
518 LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT);
520 ocd->ocd_version = LUSTRE_VERSION_CODE;
521 imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
522 imp->imp_connect_flags2_orig = ocd->ocd_connect_flags2;
524 rc = ptlrpc_connect_import(imp);
526 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
530 ptlrpc_pinger_add_import(imp);
532 GOTO(out_dis, rc = 0);
536 class_disconnect(*exp);
542 up_write(&cli->cl_sem);
545 lwp_notify_users(*exp);
551 * Implementation of OBD device operations obd_ops::o_disconnect.
553 * Release export for the LWP. Only disconnect the underlying layers
554 * on the final disconnect.
556 * \param[in] exp the export to perform disconnect on
558 * \retval 0 on success
559 * \retval negative number on error
561 static int lwp_obd_disconnect(struct obd_export *exp)
563 struct obd_device *obd = exp->exp_obd;
564 struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
568 LASSERT(lwp->lpd_connects == 1);
571 rc = class_disconnect(exp);
573 CERROR("%s: class disconnect error: rc = %d\n",
580 * Handle import events for the LWP device.
582 * \param[in] obd OBD device associated with the import
583 * \param[in] imp the import which event happened on
584 * \param[in] event event type
586 * \retval 0 on success
587 * \retval negative number on error
589 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
590 enum obd_import_event event)
593 case IMP_EVENT_DISCON:
594 case IMP_EVENT_INACTIVE:
595 case IMP_EVENT_ACTIVE:
597 case IMP_EVENT_INVALIDATE:
598 if (obd->obd_namespace == NULL)
600 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
605 CERROR("%s: unsupported import event: %#x\n",
606 obd->obd_name, event);
611 static int lwp_set_info_async(const struct lu_env *env,
612 struct obd_export *exp,
613 u32 keylen, void *key,
614 u32 vallen, void *val,
615 struct ptlrpc_request_set *set)
619 if (KEY_IS(KEY_SPTLRPC_CONF)) {
620 sptlrpc_conf_client_adapt(exp->exp_obd);
624 CERROR("Unknown key %s\n", (char *)key);
628 const struct obd_ops lwp_obd_device_ops = {
629 .o_owner = THIS_MODULE,
630 .o_add_conn = client_import_add_conn,
631 .o_del_conn = client_import_del_conn,
632 .o_connect = lwp_obd_connect,
633 .o_disconnect = lwp_obd_disconnect,
634 .o_import_event = lwp_import_event,
635 .o_set_info_async = lwp_set_info_async,