lustre/osp/lwp_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; if not, write to the
  18  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19  * Boston, MA 021110-1307, USA
  20  *
  21  * GPL HEADER END
  22  */
  23 /*
  24  * Copyright (c) 2013, 2016, Intel Corporation.
  25  * Use is subject to license terms.
  26  *
  27  * lustre/osp/lwp_dev.c
  28  *
  29  * This file provides code related to the Light Weight Proxy (LWP) managing
  30  * the connections established from OST to MDT, and MDT to MDT0.
  31  *
  32  * A LWP connection is used to send quota and FLD query requests. It's not
  33  * recoverable, which means target server doesn't have an on-disk record in
  34  * the last_rcvd file to remember the connection. Once LWP reconnect after
  35  * server reboot, server will always regard it as a new connection.
  36  *
  37  * Author: <di.wang@intel.com>
  38  * Author: <yawei.niu@intel.com>
  39  */
  40 #define DEBUG_SUBSYSTEM S_OST
  41
  42 #include <obd_class.h>
  43 #include <uapi/linux/lustre/lustre_param.h>
  44 #include <lustre_log.h>
  45 #include <linux/kthread.h>
  46
  47 #include "osp_internal.h"
  48
  49 struct lwp_device {
  50         struct lu_device        lpd_dev;
  51         struct obd_device      *lpd_obd;   /* corresponding OBD device */
  52         struct obd_uuid         lpd_cluuid;/* UUID of LWP */
  53         struct obd_export      *lpd_exp;   /* export of LWP */
  54         struct ptlrpc_thread    lpd_notify_thread; /* notify thread */
  55         int                     lpd_connects; /* use count, 0 or 1 */
  56 };
  57
  58 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
  59 {
  60         return container_of0(d, struct lwp_device, lpd_dev);
  61 }
  62
  63 static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
  64 {
  65         return &d->lpd_dev;
  66 }
  67
  68 /**
  69  * Setup LWP device.
  70  *
  71  * \param[in] env       environment passed by caller
  72  * \param[in] lwp       LWP device to be setup
  73  * \param[in] nidstring remote target NID
  74  *
  75  * \retval              0 on success
  76  * \retval              negative number on error
  77  */
  78 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
  79                      char *nidstring)
  80 {
  81         struct lustre_cfg_bufs  *bufs = NULL;
  82         struct lustre_cfg       *lcfg = NULL;
  83         char                    *lwp_name = lwp->lpd_obd->obd_name;
  84         char                    *server_uuid = NULL;
  85         char                    *ptr;
  86         class_uuid_t             uuid;
  87         struct obd_import       *imp;
  88         int                      len = strlen(lwp_name) + 1;
  89         int                      rc;
  90         ENTRY;
  91
  92         thread_set_flags(&lwp->lpd_notify_thread, SVC_STOPPED);
  93         init_waitqueue_head(&lwp->lpd_notify_thread.t_ctl_waitq);
  94
  95         OBD_ALLOC_PTR(bufs);
  96         if (bufs == NULL)
  97                 RETURN(-ENOMEM);
  98
  99         OBD_ALLOC(server_uuid, len);
 100         if (server_uuid == NULL)
 101                 GOTO(out, rc = -ENOMEM);
 102
 103         snprintf(server_uuid, len, "-%s-", LUSTRE_LWP_NAME);
 104         ptr = cfs_strrstr(lwp_name, server_uuid);
 105         if (ptr == NULL) {
 106                 CERROR("%s: failed to get server_uuid from lwp_name: rc = %d\n",
 107                        lwp_name, -EINVAL);
 108                 GOTO(out, rc = -EINVAL);
 109         }
 110
 111         strncpy(server_uuid, lwp_name, ptr - lwp_name);
 112         server_uuid[ptr - lwp_name] = '\0';
 113         strlcat(server_uuid, "_UUID", len);
 114         lustre_cfg_bufs_reset(bufs, lwp_name);
 115         lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
 116         lustre_cfg_bufs_set_string(bufs, 2, nidstring);
 117         OBD_ALLOC(lcfg, lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen));
 118         if (!lcfg)
 119                 GOTO(out, rc = -ENOMEM);
 120         lustre_cfg_init(lcfg, LCFG_SETUP, bufs);
 121
 122         rc = client_obd_setup(lwp->lpd_obd, lcfg);
 123         if (rc != 0) {
 124                 CERROR("%s: client obd setup error: rc = %d\n",
 125                        lwp->lpd_obd->obd_name, rc);
 126                 GOTO(out, rc);
 127         }
 128
 129         imp = lwp->lpd_obd->u.cli.cl_import;
 130         rc = ptlrpc_init_import(imp);
 131         if (rc)
 132                 GOTO(out, rc);
 133
 134         ll_generate_random_uuid(uuid);
 135         class_uuid_unparse(uuid, &lwp->lpd_cluuid);
 136 out:
 137         if (bufs != NULL)
 138                 OBD_FREE_PTR(bufs);
 139         if (server_uuid != NULL)
 140                 OBD_FREE(server_uuid, len);
 141         if (lcfg)
 142                 OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount,
 143                                               lcfg->lcfg_buflens));
 144         if (rc)
 145                 client_obd_cleanup(lwp->lpd_obd);
 146
 147         RETURN(rc);
 148 }
 149
 150 /**
 151  * Disconnect the import from LWP.
 152  *
 153  * \param[in] d         LWP device to be disconnected
 154  *
 155  * \retval              0 on success
 156  * \retval              negative number on error
 157  */
 158 static int lwp_disconnect(struct lwp_device *d)
 159 {
 160         struct obd_import *imp;
 161         int rc = 0;
 162
 163         imp = d->lpd_obd->u.cli.cl_import;
 164
 165         /*
 166          * Mark import deactivated now, so we don't try to reconnect if any
 167          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
 168          * fully deactivate the import because that would cause all requests
 169          * to be dropped.
 170          */
 171         LASSERT(imp != NULL);
 172         spin_lock(&imp->imp_lock);
 173         imp->imp_deactive = 1;
 174         spin_unlock(&imp->imp_lock);
 175
 176         ptlrpc_deactivate_import(imp);
 177
 178         /*
 179          * Some non-replayable imports (MDS's OSCs) are pinged, so just
 180          * delete it regardless.  (It's safe to delete an import that was
 181          * never added.)
 182          */
 183         ptlrpc_pinger_del_import(imp);
 184         rc = ptlrpc_disconnect_import(imp, 0);
 185         if (rc != 0)
 186                 CWARN("%s: can't disconnect: rc = %d\n",
 187                       d->lpd_obd->obd_name, rc);
 188
 189         ptlrpc_invalidate_import(imp);
 190
 191         RETURN(rc);
 192 }
 193
 194 /**
 195  * Implementation of lu_device_operations::ldo_process_config.
 196  *
 197  * Process a Lustre configuration request.
 198  *
 199  * \param[in] env       environment passed by caller
 200  * \param[in] dev       device to be processed
 201  * \param[in] lcfg      lustre_cfg, LCFG_PRE_CLEANUP or LCFG_CLEANUP
 202  *
 203  * \retval              0 on success
 204  * \retval              negative number on error
 205  */
 206 static int lwp_process_config(const struct lu_env *env,
 207                               struct lu_device *dev, struct lustre_cfg *lcfg)
 208 {
 209         struct lwp_device               *d = lu2lwp_dev(dev);
 210         int                              rc;
 211         ENTRY;
 212
 213         switch (lcfg->lcfg_command) {
 214         case LCFG_PRE_CLEANUP:
 215         case LCFG_CLEANUP:
 216                 rc = lwp_disconnect(d);
 217                 break;
 218         case LCFG_PARAM:
 219                 rc = -ENOSYS;
 220                 break;
 221         default:
 222                 CERROR("%s: unknown command %u\n",
 223                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
 224                 rc = 0;
 225                 break;
 226         }
 227
 228         RETURN(rc);
 229 }
 230
 231 static const struct lu_device_operations lwp_lu_ops = {
 232         .ldo_process_config     = lwp_process_config,
 233 };
 234
 235 /**
 236  * Initialize LWP device.
 237  *
 238  * \param[in] env       environment passed by caller
 239  * \param[in] lwp       device to be initialized
 240  * \param[in] ldt       not used
 241  * \param[in] cfg       lustre_cfg contains remote target uuid
 242  *
 243  * \retval              0 on success
 244  * \retval              -ENODEV if the device name cannot be found
 245  * \retval              negative numbers on other errors
 246  */
 247 static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
 248                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
 249 {
 250         int                        rc;
 251         ENTRY;
 252
 253         lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
 254         if (lwp->lpd_obd == NULL) {
 255                 CERROR("Cannot find obd with name %s\n",
 256                        lustre_cfg_string(cfg, 0));
 257                 RETURN(-ENODEV);
 258         }
 259
 260         lwp->lpd_dev.ld_ops = &lwp_lu_ops;
 261         lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev;
 262
 263         rc = ptlrpcd_addref();
 264         if (rc) {
 265                 CERROR("%s: ptlrpcd addref error: rc =%d\n",
 266                        lwp->lpd_obd->obd_name, rc);
 267                 RETURN(rc);
 268         }
 269
 270         rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1));
 271         if (rc) {
 272                 CERROR("%s: setup lwp failed. %d\n",
 273                        lwp->lpd_obd->obd_name, rc);
 274                 ptlrpcd_decref();
 275                 RETURN(rc);
 276         }
 277
 278         rc = lprocfs_obd_setup(lwp->lpd_obd, true);
 279         if (rc) {
 280                 CERROR("%s: lprocfs_obd_setup failed. %d\n",
 281                        lwp->lpd_obd->obd_name, rc);
 282                 ptlrpcd_decref();
 283                 RETURN(rc);
 284         }
 285
 286         rc = sptlrpc_lprocfs_cliobd_attach(lwp->lpd_obd);
 287         if (rc) {
 288                 CERROR("%s: sptlrpc_lprocfs_cliobd_attached failed. %d\n",
 289                        lwp->lpd_obd->obd_name, rc);
 290                 ptlrpcd_decref();
 291                 RETURN(rc);
 292         }
 293
 294         ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
 295
 296         RETURN(0);
 297 }
 298
 299 /**
 300  * Implementation of lu_device_type_operations::ldto_device_free.
 301  *
 302  * Free a LWP device.
 303  *
 304  * \param[in] env       environment passed by caller
 305  * \param[in] lu        device to be freed
 306  *
 307  * \retval              NULL to indicate that this is the bottom device
 308  *                      of the stack and there are no more devices
 309  *                      below this one to be cleaned up.
 310  */
 311 static struct lu_device *lwp_device_free(const struct lu_env *env,
 312                                          struct lu_device *lu)
 313 {
 314         struct lwp_device *m = lu2lwp_dev(lu);
 315         ENTRY;
 316
 317         if (atomic_read(&lu->ld_ref) && lu->ld_site) {
 318                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 319                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 320         }
 321         lu_device_fini(&m->lpd_dev);
 322         OBD_FREE_PTR(m);
 323         RETURN(NULL);
 324 }
 325
 326 /**
 327  * Implementation of lu_device_type_operations::ldto_device_alloc.
 328  *
 329  * Allocate a LWP device.
 330  *
 331  * \param[in] env       environment passed by caller
 332  * \param[in] ldt       device type whose name is LUSTRE_LWP_NAME
 333  * \param[in] lcfg      lustre_cfg contains remote target UUID
 334  *
 335  * \retval              pointer of allocated LWP device on success
 336  * \retval              ERR_PTR(errno) on error
 337  */
 338 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
 339                                           struct lu_device_type *ldt,
 340                                           struct lustre_cfg *lcfg)
 341 {
 342         struct lwp_device *lwp;
 343         struct lu_device  *ludev;
 344
 345         OBD_ALLOC_PTR(lwp);
 346         if (lwp == NULL) {
 347                 ludev = ERR_PTR(-ENOMEM);
 348         } else {
 349                 int rc;
 350
 351                 ludev = lwp2lu_dev(lwp);
 352                 lu_device_init(&lwp->lpd_dev, ldt);
 353                 rc = lwp_init0(env, lwp, ldt, lcfg);
 354                 if (rc != 0) {
 355                         lwp_device_free(env, ludev);
 356                         ludev = ERR_PTR(rc);
 357                 }
 358         }
 359         return ludev;
 360 }
 361
 362
 363 /**
 364  * Implementation of lu_device_type_operations::ltdo_device_fini.
 365  *
 366  * Finalize LWP device.
 367  *
 368  * \param[in] env       environment passed by caller
 369  * \param[in] ludev     device to be finalized
 370  *
 371  * \retval              NULL on success
 372  */
 373 static struct lu_device *lwp_device_fini(const struct lu_env *env,
 374                                          struct lu_device *ludev)
 375 {
 376         struct lwp_device       *m = lu2lwp_dev(ludev);
 377         struct ptlrpc_thread    *thread = &m->lpd_notify_thread;
 378         struct l_wait_info       lwi = { 0 };
 379         int                      rc;
 380         ENTRY;
 381
 382         if (!thread_is_stopped(thread))
 383                 l_wait_event(thread->t_ctl_waitq, thread_is_stopped(thread),
 384                              &lwi);
 385
 386         if (m->lpd_exp != NULL)
 387                 class_disconnect(m->lpd_exp);
 388
 389         LASSERT(m->lpd_obd);
 390         ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
 391         lprocfs_obd_cleanup(m->lpd_obd);
 392
 393         rc = client_obd_cleanup(m->lpd_obd);
 394         LASSERTF(rc == 0, "error %d\n", rc);
 395
 396         ptlrpcd_decref();
 397
 398         RETURN(NULL);
 399 }
 400
 401 static struct lu_device_type_operations lwp_device_type_ops = {
 402         .ldto_device_alloc   = lwp_device_alloc,
 403         .ldto_device_free    = lwp_device_free,
 404         .ldto_device_fini    = lwp_device_fini
 405 };
 406
 407 struct lu_device_type lwp_device_type = {
 408         .ldt_tags     = LU_DEVICE_DT,
 409         .ldt_name     = LUSTRE_LWP_NAME,
 410         .ldt_ops      = &lwp_device_type_ops,
 411         .ldt_ctx_tags = LCT_MD_THREAD
 412 };
 413
 414 static int lwp_notify_main(void *args)
 415 {
 416         struct obd_export       *exp = (struct obd_export *)args;
 417         struct lwp_device       *lwp;
 418         struct ptlrpc_thread    *thread;
 419
 420         LASSERT(exp != NULL);
 421         class_export_get(exp);
 422
 423         lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
 424         thread = &lwp->lpd_notify_thread;
 425
 426         thread_set_flags(thread, SVC_RUNNING);
 427         wake_up(&thread->t_ctl_waitq);
 428
 429         lustre_notify_lwp_list(exp);
 430
 431         class_export_put(exp);
 432         thread_set_flags(thread, SVC_STOPPED);
 433         wake_up(&thread->t_ctl_waitq);
 434         return 0;
 435 }
 436
 437 /*
 438  * Some notify callbacks may cause deadlock in failover
 439  * scenario, so we have to start thread to run callbacks
 440  * asynchronously. See LU-6273.
 441  */
 442 static void lwp_notify_users(struct obd_export *exp)
 443 {
 444         struct lwp_device       *lwp;
 445         struct ptlrpc_thread    *thread;
 446         struct task_struct      *task;
 447         struct l_wait_info       lwi = { 0 };
 448         char                     name[MTI_NAME_MAXLEN];
 449
 450         LASSERT(exp != NULL);
 451         lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
 452         thread = &lwp->lpd_notify_thread;
 453
 454         snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s",
 455                  exp->exp_obd->obd_name);
 456
 457         /* Notify happens only on LWP setup, so there shouldn't
 458          * be notify thread running */
 459         if (!thread_is_stopped(thread)) {
 460                 CERROR("LWP notify thread: %s wasn't stopped\n", name);
 461                 return;
 462         }
 463
 464         task = kthread_run(lwp_notify_main, exp, name);
 465         if (IS_ERR(task)) {
 466                 thread_set_flags(thread, SVC_STOPPED);
 467                 CERROR("Failed to start LWP notify thread:%s. %lu\n",
 468                        name, PTR_ERR(task));
 469         }
 470
 471         l_wait_event(thread->t_ctl_waitq,
 472                      thread_is_running(thread) || thread_is_stopped(thread),
 473                      &lwi);
 474 }
 475
 476 /**
 477  * Implementation of OBD device operations obd_ops::o_connect.
 478  *
 479  * Create export for LWP, and connect to target server.
 480  *
 481  * \param[in] env       the environment passed by caller
 482  * \param[out] exp      export for the connection to be established
 483  * \param[in] obd       OBD device to perform the connect on
 484  * \param[in] cluuid    UUID of the OBD device
 485  * \param[in] data      connect data containing compatibility flags
 486  * \param[in] localdata not used
 487  *
 488  * \retval              0 on success
 489  * \retval              negative number on error
 490  */
 491 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 492                            struct obd_device *obd, struct obd_uuid *cluuid,
 493                            struct obd_connect_data *data, void *localdata)
 494 {
 495         struct lwp_device       *lwp = lu2lwp_dev(obd->obd_lu_dev);
 496         struct client_obd       *cli = &lwp->lpd_obd->u.cli;
 497         struct obd_import       *imp = cli->cl_import;
 498         struct obd_connect_data *ocd;
 499         struct lustre_handle     conn;
 500         int                      rc;
 501
 502         ENTRY;
 503
 504         CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
 505
 506         *exp = NULL;
 507         down_write(&cli->cl_sem);
 508         rc = class_connect(&conn, obd, cluuid);
 509         if (rc != 0)
 510                 GOTO(out_sem, rc);
 511
 512         *exp = class_conn2export(&conn);
 513         lwp->lpd_exp = *exp;
 514
 515         lwp->lpd_connects++;
 516         LASSERT(lwp->lpd_connects == 1);
 517
 518         imp->imp_dlm_handle = conn;
 519         rc = ptlrpc_init_import(imp);
 520         if (rc != 0)
 521                 GOTO(out_dis, rc);
 522
 523         LASSERT(data != NULL);
 524         ocd = &imp->imp_connect_data;
 525         *ocd = *data;
 526
 527         LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT);
 528
 529         ocd->ocd_version = LUSTRE_VERSION_CODE;
 530         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 531         imp->imp_connect_flags2_orig = ocd->ocd_connect_flags2;
 532
 533         rc = ptlrpc_connect_import(imp);
 534         if (rc != 0) {
 535                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
 536                 GOTO(out_dis, rc);
 537         }
 538
 539         ptlrpc_pinger_add_import(imp);
 540
 541         GOTO(out_dis, rc = 0);
 542
 543 out_dis:
 544         if (rc != 0) {
 545                 class_disconnect(*exp);
 546                 *exp = NULL;
 547                 lwp->lpd_exp = NULL;
 548         }
 549
 550 out_sem:
 551         up_write(&cli->cl_sem);
 552
 553         if (rc == 0)
 554                 lwp_notify_users(*exp);
 555
 556         return rc;
 557 }
 558
 559 /**
 560  * Implementation of OBD device operations obd_ops::o_disconnect.
 561  *
 562  * Release export for the LWP. Only disconnect the underlying layers
 563  * on the final disconnect.
 564  *
 565  * \param[in] exp       the export to perform disconnect on
 566  *
 567  * \retval              0 on success
 568  * \retval              negative number on error
 569  */
 570 static int lwp_obd_disconnect(struct obd_export *exp)
 571 {
 572         struct obd_device *obd = exp->exp_obd;
 573         struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
 574         int                rc;
 575         ENTRY;
 576
 577         LASSERT(lwp->lpd_connects == 1);
 578         lwp->lpd_connects--;
 579
 580         rc = class_disconnect(exp);
 581         if (rc)
 582                 CERROR("%s: class disconnect error: rc = %d\n",
 583                        obd->obd_name, rc);
 584
 585         RETURN(rc);
 586 }
 587
 588 /**
 589  * Handle import events for the LWP device.
 590  *
 591  * \param[in] obd       OBD device associated with the import
 592  * \param[in] imp       the import which event happened on
 593  * \param[in] event     event type
 594  *
 595  * \retval              0 on success
 596  * \retval              negative number on error
 597  */
 598 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
 599                             enum obd_import_event event)
 600 {
 601         switch (event) {
 602         case IMP_EVENT_DISCON:
 603         case IMP_EVENT_INACTIVE:
 604         case IMP_EVENT_ACTIVE:
 605                 break;
 606         case IMP_EVENT_INVALIDATE:
 607                 if (obd->obd_namespace == NULL)
 608                         break;
 609                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
 610                 break;
 611         case IMP_EVENT_OCD:
 612                 break;
 613         default:
 614                 CERROR("%s: unsupported import event: %#x\n",
 615                        obd->obd_name, event);
 616         }
 617         return 0;
 618 }
 619
 620 static int lwp_set_info_async(const struct lu_env *env,
 621                               struct obd_export *exp,
 622                               u32 keylen, void *key,
 623                               u32 vallen, void *val,
 624                               struct ptlrpc_request_set *set)
 625 {
 626         ENTRY;
 627
 628         if (KEY_IS(KEY_SPTLRPC_CONF)) {
 629                 sptlrpc_conf_client_adapt(exp->exp_obd);
 630                 RETURN(0);
 631         }
 632
 633         CERROR("Unknown key %s\n", (char *)key);
 634         RETURN(-EINVAL);
 635 }
 636
 637 struct obd_ops lwp_obd_device_ops = {
 638         .o_owner        = THIS_MODULE,
 639         .o_add_conn     = client_import_add_conn,
 640         .o_del_conn     = client_import_del_conn,
 641         .o_connect      = lwp_obd_connect,
 642         .o_disconnect   = lwp_obd_disconnect,
 643         .o_import_event = lwp_import_event,
 644         .o_set_info_async   = lwp_set_info_async,
 645 };