lustre/osp/lwp_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; if not, write to the
  18  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19  * Boston, MA 021110-1307, USA
  20  *
  21  * GPL HEADER END
  22  */
  23 /*
  24  * Copyright (c) 2013, 2017, Intel Corporation.
  25  * Use is subject to license terms.
  26  *
  27  * lustre/osp/lwp_dev.c
  28  *
  29  * This file provides code related to the Light Weight Proxy (LWP) managing
  30  * the connections established from OST to MDT, and MDT to MDT0.
  31  *
  32  * A LWP connection is used to send quota and FLD query requests. It's not
  33  * recoverable, which means target server doesn't have an on-disk record in
  34  * the last_rcvd file to remember the connection. Once LWP reconnect after
  35  * server reboot, server will always regard it as a new connection.
  36  *
  37  * Author: <di.wang@intel.com>
  38  * Author: <yawei.niu@intel.com>
  39  */
  40 #define DEBUG_SUBSYSTEM S_OST
  41
  42 #include <obd_class.h>
  43 #include <uapi/linux/lustre/lustre_param.h>
  44 #include <lustre_log.h>
  45 #include <linux/kthread.h>
  46
  47 #include "osp_internal.h"
  48
  49 struct lwp_device {
  50         struct lu_device        lpd_dev;
  51         struct obd_device      *lpd_obd;   /* corresponding OBD device */
  52         struct obd_export      *lpd_exp;   /* export of LWP */
  53         struct ptlrpc_thread    lpd_notify_thread; /* notify thread */
  54         int                     lpd_connects; /* use count, 0 or 1 */
  55 };
  56
  57 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
  58 {
  59         return container_of0(d, struct lwp_device, lpd_dev);
  60 }
  61
  62 static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
  63 {
  64         return &d->lpd_dev;
  65 }
  66
  67 /**
  68  * Setup LWP device.
  69  *
  70  * \param[in] env       environment passed by caller
  71  * \param[in] lwp       LWP device to be setup
  72  * \param[in] nidstring remote target NID
  73  *
  74  * \retval              0 on success
  75  * \retval              negative number on error
  76  */
  77 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
  78                      char *nidstring)
  79 {
  80         struct lustre_cfg_bufs  *bufs = NULL;
  81         struct lustre_cfg       *lcfg = NULL;
  82         char                    *lwp_name = lwp->lpd_obd->obd_name;
  83         char                    *server_uuid = NULL;
  84         char                    *ptr;
  85         struct obd_import       *imp;
  86         int                      len = strlen(lwp_name) + 1;
  87         int                      rc;
  88         ENTRY;
  89
  90         thread_set_flags(&lwp->lpd_notify_thread, SVC_STOPPED);
  91         init_waitqueue_head(&lwp->lpd_notify_thread.t_ctl_waitq);
  92
  93         OBD_ALLOC_PTR(bufs);
  94         if (bufs == NULL)
  95                 RETURN(-ENOMEM);
  96
  97         OBD_ALLOC(server_uuid, len);
  98         if (server_uuid == NULL)
  99                 GOTO(out, rc = -ENOMEM);
 100
 101         snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME);
 102         ptr = cfs_strrstr(lwp_name, server_uuid);
 103         if (ptr == NULL) {
 104                 CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
 105                        lwp_name, -EINVAL);
 106                 GOTO(out, rc = -EINVAL);
 107         }
 108
 109         strncpy(server_uuid, lwp_name, ptr - lwp_name);
 110         server_uuid[ptr - lwp_name] = '\0';
 111         strlcat(server_uuid, "_UUID", len);
 112         lustre_cfg_bufs_reset(bufs, lwp_name);
 113         lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
 114         lustre_cfg_bufs_set_string(bufs, 2, nidstring);
 115         OBD_ALLOC(lcfg, lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen));
 116         if (!lcfg)
 117                 GOTO(out, rc = -ENOMEM);
 118         lustre_cfg_init(lcfg, LCFG_SETUP, bufs);
 119
 120         rc = client_obd_setup(lwp->lpd_obd, lcfg);
 121         if (rc != 0) {
 122                 CERROR("%s: client obd setup error: rc = %d\n",
 123                        lwp->lpd_obd->obd_name, rc);
 124                 GOTO(out, rc);
 125         }
 126
 127         imp = lwp->lpd_obd->u.cli.cl_import;
 128         rc = ptlrpc_init_import(imp);
 129 out:
 130         if (bufs != NULL)
 131                 OBD_FREE_PTR(bufs);
 132         if (server_uuid != NULL)
 133                 OBD_FREE(server_uuid, len);
 134         if (lcfg)
 135                 OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount,
 136                                               lcfg->lcfg_buflens));
 137         if (rc)
 138                 client_obd_cleanup(lwp->lpd_obd);
 139
 140         RETURN(rc);
 141 }
 142
 143 /**
 144  * Disconnect the import from LWP.
 145  *
 146  * \param[in] d         LWP device to be disconnected
 147  *
 148  * \retval              0 on success
 149  * \retval              negative number on error
 150  */
 151 static int lwp_disconnect(struct lwp_device *d)
 152 {
 153         struct obd_import *imp;
 154         int rc = 0;
 155
 156         imp = d->lpd_obd->u.cli.cl_import;
 157
 158         /*
 159          * Mark import deactivated now, so we don't try to reconnect if any
 160          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
 161          * fully deactivate the import because that would cause all requests
 162          * to be dropped.
 163          */
 164         LASSERT(imp != NULL);
 165         spin_lock(&imp->imp_lock);
 166         imp->imp_deactive = 1;
 167         spin_unlock(&imp->imp_lock);
 168
 169         ptlrpc_deactivate_import(imp);
 170
 171         /*
 172          * Some non-replayable imports (MDS's OSCs) are pinged, so just
 173          * delete it regardless.  (It's safe to delete an import that was
 174          * never added.)
 175          */
 176         ptlrpc_pinger_del_import(imp);
 177         rc = ptlrpc_disconnect_import(imp, 0);
 178         if (rc != 0)
 179                 CWARN("%s: can't disconnect: rc = %d\n",
 180                       d->lpd_obd->obd_name, rc);
 181
 182         ptlrpc_invalidate_import(imp);
 183
 184         RETURN(rc);
 185 }
 186
 187 /**
 188  * Implementation of lu_device_operations::ldo_process_config.
 189  *
 190  * Process a Lustre configuration request.
 191  *
 192  * \param[in] env       environment passed by caller
 193  * \param[in] dev       device to be processed
 194  * \param[in] lcfg      lustre_cfg, LCFG_PRE_CLEANUP or LCFG_CLEANUP
 195  *
 196  * \retval              0 on success
 197  * \retval              negative number on error
 198  */
 199 static int lwp_process_config(const struct lu_env *env,
 200                               struct lu_device *dev, struct lustre_cfg *lcfg)
 201 {
 202         struct lwp_device               *d = lu2lwp_dev(dev);
 203         int                              rc;
 204         ENTRY;
 205
 206         switch (lcfg->lcfg_command) {
 207         case LCFG_PRE_CLEANUP:
 208         case LCFG_CLEANUP:
 209                 rc = lwp_disconnect(d);
 210                 break;
 211         case LCFG_PARAM:
 212                 rc = -ENOSYS;
 213                 break;
 214         default:
 215                 CERROR("%s: unknown command %u\n",
 216                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
 217                 rc = 0;
 218                 break;
 219         }
 220
 221         RETURN(rc);
 222 }
 223
 224 static const struct lu_device_operations lwp_lu_ops = {
 225         .ldo_process_config     = lwp_process_config,
 226 };
 227
 228 /**
 229  * Initialize LWP device.
 230  *
 231  * \param[in] env       environment passed by caller
 232  * \param[in] lwp       device to be initialized
 233  * \param[in] ldt       not used
 234  * \param[in] cfg       lustre_cfg contains remote target uuid
 235  *
 236  * \retval              0 on success
 237  * \retval              -ENODEV if the device name cannot be found
 238  * \retval              negative numbers on other errors
 239  */
 240 static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
 241                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
 242 {
 243         int                        rc;
 244         ENTRY;
 245
 246         lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
 247         if (lwp->lpd_obd == NULL) {
 248                 CERROR("Cannot find obd with name %s\n",
 249                        lustre_cfg_string(cfg, 0));
 250                 RETURN(-ENODEV);
 251         }
 252
 253         lwp->lpd_dev.ld_ops = &lwp_lu_ops;
 254         lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev;
 255
 256         rc = ptlrpcd_addref();
 257         if (rc) {
 258                 CERROR("%s: ptlrpcd addref error: rc =%d\n",
 259                        lwp->lpd_obd->obd_name, rc);
 260                 RETURN(rc);
 261         }
 262
 263         rc = lprocfs_obd_setup(lwp->lpd_obd, true);
 264         if (rc) {
 265                 CERROR("%s: lprocfs_obd_setup failed. %d\n",
 266                        lwp->lpd_obd->obd_name, rc);
 267                 ptlrpcd_decref();
 268                 RETURN(rc);
 269         }
 270
 271         rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1));
 272         if (rc) {
 273                 CERROR("%s: setup lwp failed. %d\n",
 274                        lwp->lpd_obd->obd_name, rc);
 275                 lprocfs_obd_cleanup(lwp->lpd_obd);
 276                 ptlrpcd_decref();
 277                 RETURN(rc);
 278         }
 279
 280         rc = sptlrpc_lprocfs_cliobd_attach(lwp->lpd_obd);
 281         if (rc) {
 282                 CERROR("%s: sptlrpc_lprocfs_cliobd_attached failed. %d\n",
 283                        lwp->lpd_obd->obd_name, rc);
 284                 ptlrpcd_decref();
 285                 RETURN(rc);
 286         }
 287
 288         ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
 289
 290         RETURN(0);
 291 }
 292
 293 /**
 294  * Implementation of lu_device_type_operations::ldto_device_free.
 295  *
 296  * Free a LWP device.
 297  *
 298  * \param[in] env       environment passed by caller
 299  * \param[in] lu        device to be freed
 300  *
 301  * \retval              NULL to indicate that this is the bottom device
 302  *                      of the stack and there are no more devices
 303  *                      below this one to be cleaned up.
 304  */
 305 static struct lu_device *lwp_device_free(const struct lu_env *env,
 306                                          struct lu_device *lu)
 307 {
 308         struct lwp_device *m = lu2lwp_dev(lu);
 309         ENTRY;
 310
 311         if (atomic_read(&lu->ld_ref) && lu->ld_site) {
 312                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 313                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 314         }
 315         lu_device_fini(&m->lpd_dev);
 316         OBD_FREE_PTR(m);
 317         RETURN(NULL);
 318 }
 319
 320 /**
 321  * Implementation of lu_device_type_operations::ldto_device_alloc.
 322  *
 323  * Allocate a LWP device.
 324  *
 325  * \param[in] env       environment passed by caller
 326  * \param[in] ldt       device type whose name is LUSTRE_LWP_NAME
 327  * \param[in] lcfg      lustre_cfg contains remote target UUID
 328  *
 329  * \retval              pointer of allocated LWP device on success
 330  * \retval              ERR_PTR(errno) on error
 331  */
 332 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
 333                                           struct lu_device_type *ldt,
 334                                           struct lustre_cfg *lcfg)
 335 {
 336         struct lwp_device *lwp;
 337         struct lu_device  *ludev;
 338
 339         OBD_ALLOC_PTR(lwp);
 340         if (lwp == NULL) {
 341                 ludev = ERR_PTR(-ENOMEM);
 342         } else {
 343                 int rc;
 344
 345                 ludev = lwp2lu_dev(lwp);
 346                 lu_device_init(&lwp->lpd_dev, ldt);
 347                 rc = lwp_init0(env, lwp, ldt, lcfg);
 348                 if (rc != 0) {
 349                         lwp_device_free(env, ludev);
 350                         ludev = ERR_PTR(rc);
 351                 }
 352         }
 353         return ludev;
 354 }
 355
 356
 357 /**
 358  * Implementation of lu_device_type_operations::ltdo_device_fini.
 359  *
 360  * Finalize LWP device.
 361  *
 362  * \param[in] env       environment passed by caller
 363  * \param[in] ludev     device to be finalized
 364  *
 365  * \retval              NULL on success
 366  */
 367 static struct lu_device *lwp_device_fini(const struct lu_env *env,
 368                                          struct lu_device *ludev)
 369 {
 370         struct lwp_device       *m = lu2lwp_dev(ludev);
 371         struct ptlrpc_thread    *thread = &m->lpd_notify_thread;
 372         struct l_wait_info       lwi = { 0 };
 373         int                      rc;
 374         ENTRY;
 375
 376         if (!thread_is_stopped(thread))
 377                 l_wait_event(thread->t_ctl_waitq, thread_is_stopped(thread),
 378                              &lwi);
 379
 380         if (m->lpd_exp != NULL)
 381                 class_disconnect(m->lpd_exp);
 382
 383         LASSERT(m->lpd_obd);
 384         rc = client_obd_cleanup(m->lpd_obd);
 385         LASSERTF(rc == 0, "error %d\n", rc);
 386
 387         ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
 388
 389         ptlrpcd_decref();
 390
 391         RETURN(NULL);
 392 }
 393
 394 static struct lu_device_type_operations lwp_device_type_ops = {
 395         .ldto_device_alloc   = lwp_device_alloc,
 396         .ldto_device_free    = lwp_device_free,
 397         .ldto_device_fini    = lwp_device_fini
 398 };
 399
 400 struct lu_device_type lwp_device_type = {
 401         .ldt_tags     = LU_DEVICE_DT,
 402         .ldt_name     = LUSTRE_LWP_NAME,
 403         .ldt_ops      = &lwp_device_type_ops,
 404         .ldt_ctx_tags = LCT_MD_THREAD
 405 };
 406
 407 static int lwp_notify_main(void *args)
 408 {
 409         struct obd_export       *exp = (struct obd_export *)args;
 410         struct lwp_device       *lwp;
 411         struct ptlrpc_thread    *thread;
 412
 413         LASSERT(exp != NULL);
 414         class_export_get(exp);
 415
 416         lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
 417         thread = &lwp->lpd_notify_thread;
 418
 419         thread_set_flags(thread, SVC_RUNNING);
 420         wake_up(&thread->t_ctl_waitq);
 421
 422         lustre_notify_lwp_list(exp);
 423
 424         class_export_put(exp);
 425         thread_set_flags(thread, SVC_STOPPED);
 426         wake_up(&thread->t_ctl_waitq);
 427         return 0;
 428 }
 429
 430 /*
 431  * Some notify callbacks may cause deadlock in failover
 432  * scenario, so we have to start thread to run callbacks
 433  * asynchronously. See LU-6273.
 434  */
 435 static void lwp_notify_users(struct obd_export *exp)
 436 {
 437         struct lwp_device       *lwp;
 438         struct ptlrpc_thread    *thread;
 439         struct task_struct      *task;
 440         struct l_wait_info       lwi = { 0 };
 441         char                     name[MTI_NAME_MAXLEN];
 442
 443         LASSERT(exp != NULL);
 444         lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
 445         thread = &lwp->lpd_notify_thread;
 446
 447         snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s",
 448                  exp->exp_obd->obd_name);
 449
 450         /* Notify happens only on LWP setup, so there shouldn't
 451          * be notify thread running */
 452         if (!thread_is_stopped(thread)) {
 453                 CERROR("LWP notify thread: %s wasn't stopped\n", name);
 454                 return;
 455         }
 456
 457         task = kthread_run(lwp_notify_main, exp, name);
 458         if (IS_ERR(task)) {
 459                 thread_set_flags(thread, SVC_STOPPED);
 460                 CERROR("Failed to start LWP notify thread:%s. %lu\n",
 461                        name, PTR_ERR(task));
 462         }
 463
 464         l_wait_event(thread->t_ctl_waitq,
 465                      thread_is_running(thread) || thread_is_stopped(thread),
 466                      &lwi);
 467 }
 468
 469 /**
 470  * Implementation of OBD device operations obd_ops::o_connect.
 471  *
 472  * Create export for LWP, and connect to target server.
 473  *
 474  * \param[in] env       the environment passed by caller
 475  * \param[out] exp      export for the connection to be established
 476  * \param[in] obd       OBD device to perform the connect on
 477  * \param[in] cluuid    UUID of the OBD device
 478  * \param[in] data      connect data containing compatibility flags
 479  * \param[in] localdata not used
 480  *
 481  * \retval              0 on success
 482  * \retval              negative number on error
 483  */
 484 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 485                            struct obd_device *obd, struct obd_uuid *cluuid,
 486                            struct obd_connect_data *data, void *localdata)
 487 {
 488         struct lwp_device       *lwp = lu2lwp_dev(obd->obd_lu_dev);
 489         struct client_obd       *cli = &lwp->lpd_obd->u.cli;
 490         struct obd_import       *imp = cli->cl_import;
 491         struct obd_connect_data *ocd;
 492         struct lustre_handle     conn;
 493         int                      rc;
 494
 495         ENTRY;
 496
 497         CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
 498
 499         *exp = NULL;
 500         down_write(&cli->cl_sem);
 501         rc = class_connect(&conn, obd, cluuid);
 502         if (rc != 0)
 503                 GOTO(out_sem, rc);
 504
 505         *exp = class_conn2export(&conn);
 506         lwp->lpd_exp = *exp;
 507
 508         lwp->lpd_connects++;
 509         LASSERT(lwp->lpd_connects == 1);
 510
 511         imp->imp_dlm_handle = conn;
 512         rc = ptlrpc_init_import(imp);
 513         if (rc != 0)
 514                 GOTO(out_dis, rc);
 515
 516         LASSERT(data != NULL);
 517         ocd = &imp->imp_connect_data;
 518         *ocd = *data;
 519
 520         LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT);
 521
 522         ocd->ocd_version = LUSTRE_VERSION_CODE;
 523         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 524         imp->imp_connect_flags2_orig = ocd->ocd_connect_flags2;
 525
 526         rc = ptlrpc_connect_import(imp);
 527         if (rc != 0) {
 528                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
 529                 GOTO(out_dis, rc);
 530         }
 531
 532         ptlrpc_pinger_add_import(imp);
 533
 534         GOTO(out_dis, rc = 0);
 535
 536 out_dis:
 537         if (rc != 0) {
 538                 class_disconnect(*exp);
 539                 *exp = NULL;
 540                 lwp->lpd_exp = NULL;
 541         }
 542
 543 out_sem:
 544         up_write(&cli->cl_sem);
 545
 546         if (rc == 0)
 547                 lwp_notify_users(*exp);
 548
 549         return rc;
 550 }
 551
 552 /**
 553  * Implementation of OBD device operations obd_ops::o_disconnect.
 554  *
 555  * Release export for the LWP. Only disconnect the underlying layers
 556  * on the final disconnect.
 557  *
 558  * \param[in] exp       the export to perform disconnect on
 559  *
 560  * \retval              0 on success
 561  * \retval              negative number on error
 562  */
 563 static int lwp_obd_disconnect(struct obd_export *exp)
 564 {
 565         struct obd_device *obd = exp->exp_obd;
 566         struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
 567         int                rc;
 568         ENTRY;
 569
 570         LASSERT(lwp->lpd_connects == 1);
 571         lwp->lpd_connects--;
 572
 573         rc = class_disconnect(exp);
 574         if (rc)
 575                 CERROR("%s: class disconnect error: rc = %d\n",
 576                        obd->obd_name, rc);
 577
 578         RETURN(rc);
 579 }
 580
 581 /**
 582  * Handle import events for the LWP device.
 583  *
 584  * \param[in] obd       OBD device associated with the import
 585  * \param[in] imp       the import which event happened on
 586  * \param[in] event     event type
 587  *
 588  * \retval              0 on success
 589  * \retval              negative number on error
 590  */
 591 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
 592                             enum obd_import_event event)
 593 {
 594         switch (event) {
 595         case IMP_EVENT_DISCON:
 596         case IMP_EVENT_INACTIVE:
 597         case IMP_EVENT_ACTIVE:
 598                 break;
 599         case IMP_EVENT_INVALIDATE:
 600                 if (obd->obd_namespace == NULL)
 601                         break;
 602                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
 603                 break;
 604         case IMP_EVENT_OCD:
 605                 break;
 606         default:
 607                 CERROR("%s: unsupported import event: %#x\n",
 608                        obd->obd_name, event);
 609         }
 610         return 0;
 611 }
 612
 613 static int lwp_set_info_async(const struct lu_env *env,
 614                               struct obd_export *exp,
 615                               u32 keylen, void *key,
 616                               u32 vallen, void *val,
 617                               struct ptlrpc_request_set *set)
 618 {
 619         ENTRY;
 620
 621         if (KEY_IS(KEY_SPTLRPC_CONF)) {
 622                 sptlrpc_conf_client_adapt(exp->exp_obd);
 623                 RETURN(0);
 624         }
 625
 626         CERROR("Unknown key %s\n", (char *)key);
 627         RETURN(-EINVAL);
 628 }
 629
 630 struct obd_ops lwp_obd_device_ops = {
 631         .o_owner        = THIS_MODULE,
 632         .o_add_conn     = client_import_add_conn,
 633         .o_del_conn     = client_import_del_conn,
 634         .o_connect      = lwp_obd_connect,
 635         .o_disconnect   = lwp_obd_disconnect,
 636         .o_import_event = lwp_import_event,
 637         .o_set_info_async   = lwp_set_info_async,
 638 };