Whamcloud - gitweb
LU-12616 obclass: fix MDS start/stop race
[fs/lustre-release.git] / lustre / osp / lwp_dev.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; if not, write to the
18  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19  * Boston, MA 021110-1307, USA
20  *
21  * GPL HEADER END
22  */
23 /*
24  * Copyright (c) 2013, 2017, Intel Corporation.
25  * Use is subject to license terms.
26  *
27  * lustre/osp/lwp_dev.c
28  *
29  * This file provides code related to the Light Weight Proxy (LWP) managing
30  * the connections established from OST to MDT, and MDT to MDT0.
31  *
32  * A LWP connection is used to send quota and FLD query requests. It's not
33  * recoverable, which means target server doesn't have an on-disk record in
34  * the last_rcvd file to remember the connection. Once LWP reconnect after
35  * server reboot, server will always regard it as a new connection.
36  *
37  * Author: <di.wang@intel.com>
38  * Author: <yawei.niu@intel.com>
39  */
40 #define DEBUG_SUBSYSTEM S_OST
41
42 #include <obd_class.h>
43 #include <uapi/linux/lustre/lustre_param.h>
44 #include <lustre_log.h>
45 #include <linux/kthread.h>
46
47 #include "osp_internal.h"
48
49 struct lwp_device {
50         struct lu_device        lpd_dev;
51         struct obd_device      *lpd_obd;   /* corresponding OBD device */
52         struct obd_export      *lpd_exp;   /* export of LWP */
53         struct ptlrpc_thread    lpd_notify_thread; /* notify thread */
54         int                     lpd_connects; /* use count, 0 or 1 */
55 };
56
57 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
58 {
59         return container_of0(d, struct lwp_device, lpd_dev);
60 }
61
62 static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
63 {
64         return &d->lpd_dev;
65 }
66
67 /**
68  * Setup LWP device.
69  *
70  * \param[in] env       environment passed by caller
71  * \param[in] lwp       LWP device to be setup
72  * \param[in] nidstring remote target NID
73  *
74  * \retval              0 on success
75  * \retval              negative number on error
76  */
77 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
78                      char *nidstring)
79 {
80         struct lustre_cfg_bufs  *bufs = NULL;
81         struct lustre_cfg       *lcfg = NULL;
82         char                    *lwp_name = lwp->lpd_obd->obd_name;
83         char                    *server_uuid = NULL;
84         char                    *ptr;
85         struct obd_import       *imp;
86         int                      len = strlen(lwp_name) + 1;
87         int                      rc;
88         ENTRY;
89
90         thread_set_flags(&lwp->lpd_notify_thread, SVC_STOPPED);
91         init_waitqueue_head(&lwp->lpd_notify_thread.t_ctl_waitq);
92
93         OBD_ALLOC_PTR(bufs);
94         if (bufs == NULL)
95                 RETURN(-ENOMEM);
96
97         OBD_ALLOC(server_uuid, len);
98         if (server_uuid == NULL)
99                 GOTO(out, rc = -ENOMEM);
100
101         snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME);
102         ptr = cfs_strrstr(lwp_name, server_uuid);
103         if (ptr == NULL) {
104                 CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
105                        lwp_name, -EINVAL);
106                 GOTO(out, rc = -EINVAL);
107         }
108
109         strncpy(server_uuid, lwp_name, ptr - lwp_name);
110         server_uuid[ptr - lwp_name] = '\0';
111         strlcat(server_uuid, "_UUID", len);
112         lustre_cfg_bufs_reset(bufs, lwp_name);
113         lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
114         lustre_cfg_bufs_set_string(bufs, 2, nidstring);
115         OBD_ALLOC(lcfg, lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen));
116         if (!lcfg)
117                 GOTO(out, rc = -ENOMEM);
118         lustre_cfg_init(lcfg, LCFG_SETUP, bufs);
119
120         rc = client_obd_setup(lwp->lpd_obd, lcfg);
121         if (rc != 0) {
122                 CERROR("%s: client obd setup error: rc = %d\n",
123                        lwp->lpd_obd->obd_name, rc);
124                 GOTO(out, rc);
125         }
126
127         imp = lwp->lpd_obd->u.cli.cl_import;
128         rc = ptlrpc_init_import(imp);
129 out:
130         if (bufs != NULL)
131                 OBD_FREE_PTR(bufs);
132         if (server_uuid != NULL)
133                 OBD_FREE(server_uuid, len);
134         if (lcfg)
135                 OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount,
136                                               lcfg->lcfg_buflens));
137         if (rc)
138                 client_obd_cleanup(lwp->lpd_obd);
139
140         RETURN(rc);
141 }
142
143 /**
144  * Disconnect the import from LWP.
145  *
146  * \param[in] d         LWP device to be disconnected
147  *
148  * \retval              0 on success
149  * \retval              negative number on error
150  */
151 static int lwp_disconnect(struct lwp_device *d)
152 {
153         struct obd_import *imp;
154         int rc = 0;
155
156         imp = d->lpd_obd->u.cli.cl_import;
157
158         /*
159          * Mark import deactivated now, so we don't try to reconnect if any
160          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
161          * fully deactivate the import because that would cause all requests
162          * to be dropped.
163          */
164         LASSERT(imp != NULL);
165         spin_lock(&imp->imp_lock);
166         imp->imp_deactive = 1;
167         spin_unlock(&imp->imp_lock);
168
169         ptlrpc_deactivate_import(imp);
170
171         /*
172          * Some non-replayable imports (MDS's OSCs) are pinged, so just
173          * delete it regardless.  (It's safe to delete an import that was
174          * never added.)
175          */
176         ptlrpc_pinger_del_import(imp);
177         rc = ptlrpc_disconnect_import(imp, 0);
178         if (rc != 0)
179                 CWARN("%s: can't disconnect: rc = %d\n",
180                       d->lpd_obd->obd_name, rc);
181
182         ptlrpc_invalidate_import(imp);
183
184         RETURN(rc);
185 }
186
187 /**
188  * Implementation of lu_device_operations::ldo_process_config.
189  *
190  * Process a Lustre configuration request.
191  *
192  * \param[in] env       environment passed by caller
193  * \param[in] dev       device to be processed
194  * \param[in] lcfg      lustre_cfg, LCFG_PRE_CLEANUP or LCFG_CLEANUP
195  *
196  * \retval              0 on success
197  * \retval              negative number on error
198  */
199 static int lwp_process_config(const struct lu_env *env,
200                               struct lu_device *dev, struct lustre_cfg *lcfg)
201 {
202         struct lwp_device               *d = lu2lwp_dev(dev);
203         int                              rc;
204         ENTRY;
205
206         switch (lcfg->lcfg_command) {
207         case LCFG_PRE_CLEANUP:
208         case LCFG_CLEANUP:
209                 rc = lwp_disconnect(d);
210                 break;
211         case LCFG_PARAM:
212                 rc = -ENOSYS;
213                 break;
214         default:
215                 CERROR("%s: unknown command %u\n",
216                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
217                 rc = 0;
218                 break;
219         }
220
221         RETURN(rc);
222 }
223
224 static const struct lu_device_operations lwp_lu_ops = {
225         .ldo_process_config     = lwp_process_config,
226 };
227
228 /**
229  * Initialize LWP device.
230  *
231  * \param[in] env       environment passed by caller
232  * \param[in] lwp       device to be initialized
233  * \param[in] ldt       not used
234  * \param[in] cfg       lustre_cfg contains remote target uuid
235  *
236  * \retval              0 on success
237  * \retval              -ENODEV if the device name cannot be found
238  * \retval              negative numbers on other errors
239  */
240 static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
241                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
242 {
243         int                        rc;
244         ENTRY;
245
246         lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
247         if (lwp->lpd_obd == NULL) {
248                 CERROR("Cannot find obd with name %s\n",
249                        lustre_cfg_string(cfg, 0));
250                 RETURN(-ENODEV);
251         }
252
253         lwp->lpd_dev.ld_ops = &lwp_lu_ops;
254         lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev;
255
256         rc = ptlrpcd_addref();
257         if (rc) {
258                 CERROR("%s: ptlrpcd addref error: rc =%d\n",
259                        lwp->lpd_obd->obd_name, rc);
260                 RETURN(rc);
261         }
262
263         rc = lprocfs_obd_setup(lwp->lpd_obd, true);
264         if (rc) {
265                 CERROR("%s: lprocfs_obd_setup failed. %d\n",
266                        lwp->lpd_obd->obd_name, rc);
267                 ptlrpcd_decref();
268                 RETURN(rc);
269         }
270
271         rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1));
272         if (rc) {
273                 CERROR("%s: setup lwp failed. %d\n",
274                        lwp->lpd_obd->obd_name, rc);
275                 lprocfs_obd_cleanup(lwp->lpd_obd);
276                 ptlrpcd_decref();
277                 RETURN(rc);
278         }
279
280         rc = sptlrpc_lprocfs_cliobd_attach(lwp->lpd_obd);
281         if (rc) {
282                 CERROR("%s: sptlrpc_lprocfs_cliobd_attached failed. %d\n",
283                        lwp->lpd_obd->obd_name, rc);
284                 ptlrpcd_decref();
285                 RETURN(rc);
286         }
287
288         ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
289
290         RETURN(0);
291 }
292
293 /**
294  * Implementation of lu_device_type_operations::ldto_device_free.
295  *
296  * Free a LWP device.
297  *
298  * \param[in] env       environment passed by caller
299  * \param[in] lu        device to be freed
300  *
301  * \retval              NULL to indicate that this is the bottom device
302  *                      of the stack and there are no more devices
303  *                      below this one to be cleaned up.
304  */
305 static struct lu_device *lwp_device_free(const struct lu_env *env,
306                                          struct lu_device *lu)
307 {
308         struct lwp_device *m = lu2lwp_dev(lu);
309         ENTRY;
310
311         if (atomic_read(&lu->ld_ref) && lu->ld_site) {
312                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
313                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
314         }
315         lu_device_fini(&m->lpd_dev);
316         OBD_FREE_PTR(m);
317         RETURN(NULL);
318 }
319
320 /**
321  * Implementation of lu_device_type_operations::ldto_device_alloc.
322  *
323  * Allocate a LWP device.
324  *
325  * \param[in] env       environment passed by caller
326  * \param[in] ldt       device type whose name is LUSTRE_LWP_NAME
327  * \param[in] lcfg      lustre_cfg contains remote target UUID
328  *
329  * \retval              pointer of allocated LWP device on success
330  * \retval              ERR_PTR(errno) on error
331  */
332 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
333                                           struct lu_device_type *ldt,
334                                           struct lustre_cfg *lcfg)
335 {
336         struct lwp_device *lwp;
337         struct lu_device  *ludev;
338
339         OBD_ALLOC_PTR(lwp);
340         if (lwp == NULL) {
341                 ludev = ERR_PTR(-ENOMEM);
342         } else {
343                 int rc;
344
345                 ludev = lwp2lu_dev(lwp);
346                 lu_device_init(&lwp->lpd_dev, ldt);
347                 rc = lwp_init0(env, lwp, ldt, lcfg);
348                 if (rc != 0) {
349                         lwp_device_free(env, ludev);
350                         ludev = ERR_PTR(rc);
351                 }
352         }
353         return ludev;
354 }
355
356
357 /**
358  * Implementation of lu_device_type_operations::ltdo_device_fini.
359  *
360  * Finalize LWP device.
361  *
362  * \param[in] env       environment passed by caller
363  * \param[in] ludev     device to be finalized
364  *
365  * \retval              NULL on success
366  */
367 static struct lu_device *lwp_device_fini(const struct lu_env *env,
368                                          struct lu_device *ludev)
369 {
370         struct lwp_device       *m = lu2lwp_dev(ludev);
371         struct ptlrpc_thread    *thread = &m->lpd_notify_thread;
372         struct l_wait_info       lwi = { 0 };
373         int                      rc;
374         ENTRY;
375
376         if (!thread_is_stopped(thread))
377                 l_wait_event(thread->t_ctl_waitq, thread_is_stopped(thread),
378                              &lwi);
379
380         if (m->lpd_exp != NULL)
381                 class_disconnect(m->lpd_exp);
382
383         LASSERT(m->lpd_obd);
384         rc = client_obd_cleanup(m->lpd_obd);
385         LASSERTF(rc == 0, "error %d\n", rc);
386
387         ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
388
389         ptlrpcd_decref();
390
391         RETURN(NULL);
392 }
393
394 static struct lu_device_type_operations lwp_device_type_ops = {
395         .ldto_device_alloc   = lwp_device_alloc,
396         .ldto_device_free    = lwp_device_free,
397         .ldto_device_fini    = lwp_device_fini
398 };
399
400 struct lu_device_type lwp_device_type = {
401         .ldt_tags     = LU_DEVICE_DT,
402         .ldt_name     = LUSTRE_LWP_NAME,
403         .ldt_ops      = &lwp_device_type_ops,
404         .ldt_ctx_tags = LCT_MD_THREAD
405 };
406
407 static int lwp_notify_main(void *args)
408 {
409         struct obd_export       *exp = (struct obd_export *)args;
410         struct lwp_device       *lwp;
411         struct ptlrpc_thread    *thread;
412
413         LASSERT(exp != NULL);
414         class_export_get(exp);
415
416         lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
417         thread = &lwp->lpd_notify_thread;
418
419         thread_set_flags(thread, SVC_RUNNING);
420         wake_up(&thread->t_ctl_waitq);
421
422         lustre_notify_lwp_list(exp);
423
424         class_export_put(exp);
425         thread_set_flags(thread, SVC_STOPPED);
426         wake_up(&thread->t_ctl_waitq);
427         return 0;
428 }
429
430 /*
431  * Some notify callbacks may cause deadlock in failover
432  * scenario, so we have to start thread to run callbacks
433  * asynchronously. See LU-6273.
434  */
435 static void lwp_notify_users(struct obd_export *exp)
436 {
437         struct lwp_device       *lwp;
438         struct ptlrpc_thread    *thread;
439         struct task_struct      *task;
440         struct l_wait_info       lwi = { 0 };
441         char                     name[MTI_NAME_MAXLEN];
442
443         LASSERT(exp != NULL);
444         lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
445         thread = &lwp->lpd_notify_thread;
446
447         snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s",
448                  exp->exp_obd->obd_name);
449
450         /* Notify happens only on LWP setup, so there shouldn't
451          * be notify thread running */
452         if (!thread_is_stopped(thread)) {
453                 CERROR("LWP notify thread: %s wasn't stopped\n", name);
454                 return;
455         }
456
457         task = kthread_run(lwp_notify_main, exp, name);
458         if (IS_ERR(task)) {
459                 thread_set_flags(thread, SVC_STOPPED);
460                 CERROR("Failed to start LWP notify thread:%s. %lu\n",
461                        name, PTR_ERR(task));
462         }
463
464         l_wait_event(thread->t_ctl_waitq,
465                      thread_is_running(thread) || thread_is_stopped(thread),
466                      &lwi);
467 }
468
469 /**
470  * Implementation of OBD device operations obd_ops::o_connect.
471  *
472  * Create export for LWP, and connect to target server.
473  *
474  * \param[in] env       the environment passed by caller
475  * \param[out] exp      export for the connection to be established
476  * \param[in] obd       OBD device to perform the connect on
477  * \param[in] cluuid    UUID of the OBD device
478  * \param[in] data      connect data containing compatibility flags
479  * \param[in] localdata not used
480  *
481  * \retval              0 on success
482  * \retval              negative number on error
483  */
484 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
485                            struct obd_device *obd, struct obd_uuid *cluuid,
486                            struct obd_connect_data *data, void *localdata)
487 {
488         struct lwp_device       *lwp = lu2lwp_dev(obd->obd_lu_dev);
489         struct client_obd       *cli = &lwp->lpd_obd->u.cli;
490         struct obd_import       *imp = cli->cl_import;
491         struct obd_connect_data *ocd;
492         struct lustre_handle     conn;
493         int                      rc;
494
495         ENTRY;
496
497         CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
498
499         *exp = NULL;
500         down_write(&cli->cl_sem);
501         rc = class_connect(&conn, obd, cluuid);
502         if (rc != 0)
503                 GOTO(out_sem, rc);
504
505         *exp = class_conn2export(&conn);
506         lwp->lpd_exp = *exp;
507
508         lwp->lpd_connects++;
509         LASSERT(lwp->lpd_connects == 1);
510
511         imp->imp_dlm_handle = conn;
512         rc = ptlrpc_init_import(imp);
513         if (rc != 0)
514                 GOTO(out_dis, rc);
515
516         LASSERT(data != NULL);
517         ocd = &imp->imp_connect_data;
518         *ocd = *data;
519
520         LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT);
521
522         ocd->ocd_version = LUSTRE_VERSION_CODE;
523         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
524         imp->imp_connect_flags2_orig = ocd->ocd_connect_flags2;
525
526         rc = ptlrpc_connect_import(imp);
527         if (rc != 0) {
528                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
529                 GOTO(out_dis, rc);
530         }
531
532         ptlrpc_pinger_add_import(imp);
533
534         GOTO(out_dis, rc = 0);
535
536 out_dis:
537         if (rc != 0) {
538                 class_disconnect(*exp);
539                 *exp = NULL;
540                 lwp->lpd_exp = NULL;
541         }
542
543 out_sem:
544         up_write(&cli->cl_sem);
545
546         if (rc == 0)
547                 lwp_notify_users(*exp);
548
549         return rc;
550 }
551
552 /**
553  * Implementation of OBD device operations obd_ops::o_disconnect.
554  *
555  * Release export for the LWP. Only disconnect the underlying layers
556  * on the final disconnect.
557  *
558  * \param[in] exp       the export to perform disconnect on
559  *
560  * \retval              0 on success
561  * \retval              negative number on error
562  */
563 static int lwp_obd_disconnect(struct obd_export *exp)
564 {
565         struct obd_device *obd = exp->exp_obd;
566         struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
567         int                rc;
568         ENTRY;
569
570         LASSERT(lwp->lpd_connects == 1);
571         lwp->lpd_connects--;
572
573         rc = class_disconnect(exp);
574         if (rc)
575                 CERROR("%s: class disconnect error: rc = %d\n",
576                        obd->obd_name, rc);
577
578         RETURN(rc);
579 }
580
581 /**
582  * Handle import events for the LWP device.
583  *
584  * \param[in] obd       OBD device associated with the import
585  * \param[in] imp       the import which event happened on
586  * \param[in] event     event type
587  *
588  * \retval              0 on success
589  * \retval              negative number on error
590  */
591 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
592                             enum obd_import_event event)
593 {
594         switch (event) {
595         case IMP_EVENT_DISCON:
596         case IMP_EVENT_INACTIVE:
597         case IMP_EVENT_ACTIVE:
598                 break;
599         case IMP_EVENT_INVALIDATE:
600                 if (obd->obd_namespace == NULL)
601                         break;
602                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
603                 break;
604         case IMP_EVENT_OCD:
605                 break;
606         default:
607                 CERROR("%s: unsupported import event: %#x\n",
608                        obd->obd_name, event);
609         }
610         return 0;
611 }
612
613 static int lwp_set_info_async(const struct lu_env *env,
614                               struct obd_export *exp,
615                               u32 keylen, void *key,
616                               u32 vallen, void *val,
617                               struct ptlrpc_request_set *set)
618 {
619         ENTRY;
620
621         if (KEY_IS(KEY_SPTLRPC_CONF)) {
622                 sptlrpc_conf_client_adapt(exp->exp_obd);
623                 RETURN(0);
624         }
625
626         CERROR("Unknown key %s\n", (char *)key);
627         RETURN(-EINVAL);
628 }
629
630 struct obd_ops lwp_obd_device_ops = {
631         .o_owner        = THIS_MODULE,
632         .o_add_conn     = client_import_add_conn,
633         .o_del_conn     = client_import_del_conn,
634         .o_connect      = lwp_obd_connect,
635         .o_disconnect   = lwp_obd_disconnect,
636         .o_import_event = lwp_import_event,
637         .o_set_info_async   = lwp_set_info_async,
638 };