lustre/mdt/mdt_mds.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License version 2 for more details.  A copy is
  14  * included in the COPYING file that accompanied this code.
  15
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2012 Intel Corporation
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  *
  31  * lustre/mdt/mdt_mds.c
  32  *
  33  * Lustre Metadata Service Layer
  34  *
  35  * Author: Di Wang <di.wang@whamcloud.com>
  36  **/
  37
  38 #define DEBUG_SUBSYSTEM S_MDS
  39
  40 #include <linux/module.h>
  41
  42 #include <obd_support.h>
  43 /* struct ptlrpc_request */
  44 #include <lustre_net.h>
  45 /* struct obd_export */
  46 #include <lustre_export.h>
  47 /* struct obd_device */
  48 #include <obd.h>
  49 /* lu2dt_dev() */
  50 #include <dt_object.h>
  51 #include <lustre_mds.h>
  52 #include <lustre_mdt.h>
  53 #include "mdt_internal.h"
  54 #ifdef HAVE_QUOTA_SUPPORT
  55 # include <lustre_quota.h>
  56 #endif
  57 #include <lustre_acl.h>
  58 #include <lustre_param.h>
  59 #include <lustre_fsfilt.h>
  60
  61 struct mds_device {
  62         /* super-class */
  63         struct md_device           mds_md_dev;
  64         struct ptlrpc_service     *mds_regular_service;
  65         struct ptlrpc_service     *mds_readpage_service;
  66         struct ptlrpc_service     *mds_setattr_service;
  67         struct ptlrpc_service     *mds_mdsc_service;
  68         struct ptlrpc_service     *mds_mdss_service;
  69         struct ptlrpc_service     *mds_fld_service;
  70 };
  71
  72 /*
  73  *  * Initialized in mdt_mod_init().
  74  *   */
  75 static unsigned long mdt_num_threads;
  76 CFS_MODULE_PARM(mdt_num_threads, "ul", ulong, 0444,
  77                 "number of MDS service threads to start "
  78                 "(deprecated in favor of mds_num_threads)");
  79
  80 static unsigned long mds_num_threads;
  81 CFS_MODULE_PARM(mds_num_threads, "ul", ulong, 0444,
  82                 "number of MDS service threads to start");
  83
  84 static char *mds_num_cpts;
  85 CFS_MODULE_PARM(mds_num_cpts, "c", charp, 0444,
  86                 "CPU partitions MDS threads should run on");
  87
  88 static unsigned long mds_rdpg_num_threads;
  89 CFS_MODULE_PARM(mds_rdpg_num_threads, "ul", ulong, 0444,
  90                 "number of MDS readpage service threads to start");
  91
  92 static char *mds_rdpg_num_cpts;
  93 CFS_MODULE_PARM(mds_rdpg_num_cpts, "c", charp, 0444,
  94                 "CPU partitions MDS readpage threads should run on");
  95
  96 /* NB: these two should be removed along with setattr service in the future */
  97 static unsigned long mds_attr_num_threads;
  98 CFS_MODULE_PARM(mds_attr_num_threads, "ul", ulong, 0444,
  99                 "number of MDS setattr service threads to start");
 100
 101 static char *mds_attr_num_cpts;
 102 CFS_MODULE_PARM(mds_attr_num_cpts, "c", charp, 0444,
 103                 "CPU partitions MDS setattr threads should run on");
 104
 105 #define DEFINE_RPC_HANDLER(base, flags, opc, fn, fmt)                   \
 106 [opc - base] = {                                                        \
 107         .mh_name        = #opc,                                         \
 108         .mh_fail_id     = OBD_FAIL_ ## opc ## _NET,                     \
 109         .mh_opc         = opc,                                          \
 110         .mh_flags       = flags,                                        \
 111         .mh_act         = fn,                                           \
 112         .mh_fmt         = fmt                                           \
 113 }
 114
 115 /* Request with a format known in advance */
 116 #define DEF_MDT_HDL(flags, name, fn)                                    \
 117         DEFINE_RPC_HANDLER(MDS_GETATTR, flags, name, fn, &RQF_ ## name)
 118
 119 /* Request with a format we do not yet know */
 120 #define DEF_MDT_HDL_VAR(flags, name, fn)                                \
 121         DEFINE_RPC_HANDLER(MDS_GETATTR, flags, name, fn, NULL)
 122
 123 /* Map one non-standard request format handler.  This should probably get
 124  * a common OBD_SET_INFO RPC opcode instead of this mismatch. */
 125 #define RQF_MDS_SET_INFO RQF_OBD_SET_INFO
 126
 127 static struct mdt_handler mdt_mds_ops[] = {
 128 DEF_MDT_HDL(0,                          MDS_CONNECT,      mdt_connect),
 129 DEF_MDT_HDL(0,                          MDS_DISCONNECT,   mdt_disconnect),
 130 DEF_MDT_HDL(0,                          MDS_SET_INFO,     mdt_set_info),
 131 DEF_MDT_HDL(0,                          MDS_GET_INFO,     mdt_get_info),
 132 DEF_MDT_HDL(0           | HABEO_REFERO, MDS_GETSTATUS,    mdt_getstatus),
 133 DEF_MDT_HDL(HABEO_CORPUS,               MDS_GETATTR,      mdt_getattr),
 134 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_GETATTR_NAME, mdt_getattr_name),
 135 DEF_MDT_HDL(HABEO_CORPUS,               MDS_GETXATTR,     mdt_getxattr),
 136 DEF_MDT_HDL(0           | HABEO_REFERO, MDS_STATFS,       mdt_statfs),
 137 DEF_MDT_HDL(0           | MUTABOR,      MDS_REINT,        mdt_reint),
 138 DEF_MDT_HDL(HABEO_CORPUS,               MDS_CLOSE,        mdt_close),
 139 DEF_MDT_HDL(HABEO_CORPUS,               MDS_DONE_WRITING, mdt_done_writing),
 140 DEF_MDT_HDL(0           | HABEO_REFERO, MDS_PIN,          mdt_pin),
 141 DEF_MDT_HDL_VAR(0,                      MDS_SYNC,         mdt_sync),
 142 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_IS_SUBDIR,    mdt_is_subdir),
 143 DEF_MDT_HDL(0,                          MDS_QUOTACHECK,   mdt_quotacheck),
 144 DEF_MDT_HDL(0,                          MDS_QUOTACTL,     mdt_quotactl)
 145 };
 146
 147 #define DEF_OBD_HDL(flags, name, fn)                                    \
 148         DEFINE_RPC_HANDLER(OBD_PING, flags, name, fn, NULL)
 149
 150 static struct mdt_handler mdt_obd_ops[] = {
 151 DEF_OBD_HDL(0,                          OBD_PING,         mdt_obd_ping),
 152 DEF_OBD_HDL(0,                          OBD_LOG_CANCEL,   mdt_obd_log_cancel),
 153 DEF_OBD_HDL(0,                          OBD_QC_CALLBACK,  mdt_obd_qc_callback),
 154 DEF_OBD_HDL(0,                          OBD_IDX_READ,     mdt_obd_idx_read)
 155 };
 156
 157 #define DEF_DLM_HDL_VAR(flags, name, fn)                                \
 158         DEFINE_RPC_HANDLER(LDLM_ENQUEUE, flags, name, fn, NULL)
 159 #define DEF_DLM_HDL(flags, name, fn)                                    \
 160         DEFINE_RPC_HANDLER(LDLM_ENQUEUE, flags, name, fn, &RQF_ ## name)
 161
 162 static struct mdt_handler mdt_dlm_ops[] = {
 163 DEF_DLM_HDL    (HABEO_CLAVIS,           LDLM_ENQUEUE,     mdt_enqueue),
 164 DEF_DLM_HDL_VAR(HABEO_CLAVIS,           LDLM_CONVERT,     mdt_convert),
 165 DEF_DLM_HDL_VAR(0,                      LDLM_BL_CALLBACK, mdt_bl_callback),
 166 DEF_DLM_HDL_VAR(0,                      LDLM_CP_CALLBACK, mdt_cp_callback)
 167 };
 168
 169 #define DEF_LLOG_HDL(flags, name, fn)                                   \
 170         DEFINE_RPC_HANDLER(LLOG_ORIGIN_HANDLE_CREATE, flags, name, fn, NULL)
 171
 172 static struct mdt_handler mdt_llog_ops[] = {
 173 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_CREATE,        mdt_llog_create),
 174 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_NEXT_BLOCK,    mdt_llog_next_block),
 175 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_READ_HEADER,   mdt_llog_read_header),
 176 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_WRITE_REC,     NULL),
 177 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_CLOSE,         NULL),
 178 DEF_LLOG_HDL(0,         LLOG_ORIGIN_CONNECT,              NULL),
 179 DEF_LLOG_HDL(0,         LLOG_CATINFO,                     NULL),
 180 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_PREV_BLOCK,    mdt_llog_prev_block),
 181 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_DESTROY,       mdt_llog_destroy),
 182 };
 183
 184 #define DEF_SEC_HDL(flags, name, fn)                                    \
 185         DEFINE_RPC_HANDLER(SEC_CTX_INIT, flags, name, fn, NULL)
 186
 187 static struct mdt_handler mdt_sec_ctx_ops[] = {
 188 DEF_SEC_HDL(0,                          SEC_CTX_INIT,     mdt_sec_ctx_handle),
 189 DEF_SEC_HDL(0,                          SEC_CTX_INIT_CONT,mdt_sec_ctx_handle),
 190 DEF_SEC_HDL(0,                          SEC_CTX_FINI,     mdt_sec_ctx_handle)
 191 };
 192
 193 #define DEF_QUOTA_HDL(flags, name, fn)                          \
 194         DEFINE_RPC_HANDLER(QUOTA_DQACQ, flags, name, fn, &RQF_ ## name)
 195
 196 static struct mdt_handler mdt_quota_ops[] = {
 197 DEF_QUOTA_HDL(HABEO_REFERO,             QUOTA_DQACQ,      mdt_quota_dqacq),
 198 };
 199
 200 struct mdt_opc_slice mdt_regular_handlers[] = {
 201         {
 202                 .mos_opc_start  = MDS_GETATTR,
 203                 .mos_opc_end    = MDS_LAST_OPC,
 204                 .mos_hs         = mdt_mds_ops
 205         },
 206         {
 207                 .mos_opc_start  = OBD_PING,
 208                 .mos_opc_end    = OBD_LAST_OPC,
 209                 .mos_hs         = mdt_obd_ops
 210         },
 211         {
 212                 .mos_opc_start  = LDLM_ENQUEUE,
 213                 .mos_opc_end    = LDLM_LAST_OPC,
 214                 .mos_hs         = mdt_dlm_ops
 215         },
 216         {
 217                 .mos_opc_start  = LLOG_ORIGIN_HANDLE_CREATE,
 218                 .mos_opc_end    = LLOG_LAST_OPC,
 219                 .mos_hs         = mdt_llog_ops
 220         },
 221         {
 222                 .mos_opc_start  = SEC_CTX_INIT,
 223                 .mos_opc_end    = SEC_LAST_OPC,
 224                 .mos_hs         = mdt_sec_ctx_ops
 225         },
 226         {
 227                 .mos_opc_start  = QUOTA_DQACQ,
 228                 .mos_opc_end    = QUOTA_LAST_OPC,
 229                 .mos_hs         = mdt_quota_ops
 230         },
 231         {
 232                 .mos_hs         = NULL
 233         }
 234 };
 235
 236 /* Readpage/readdir handlers */
 237 static struct mdt_handler mdt_readpage_ops[] = {
 238 DEF_MDT_HDL(0,                  MDS_CONNECT,  mdt_connect),
 239 DEF_MDT_HDL(HABEO_CORPUS | HABEO_REFERO, MDS_READPAGE, mdt_readpage),
 240 /* XXX: this is ugly and should be fixed one day, see mdc_close() for
 241  * detailed comments. --umka */
 242 DEF_MDT_HDL(HABEO_CORPUS,               MDS_CLOSE,        mdt_close),
 243 DEF_MDT_HDL(HABEO_CORPUS,               MDS_DONE_WRITING, mdt_done_writing),
 244 };
 245
 246 static struct mdt_opc_slice mdt_readpage_handlers[] = {
 247         {
 248                 .mos_opc_start = MDS_GETATTR,
 249                 .mos_opc_end   = MDS_LAST_OPC,
 250                 .mos_hs = mdt_readpage_ops
 251         },
 252         {
 253                 .mos_opc_start = OBD_FIRST_OPC,
 254                 .mos_opc_end   = OBD_LAST_OPC,
 255                 .mos_hs = mdt_obd_ops
 256         },
 257         {
 258                 .mos_hs = NULL
 259         }
 260 };
 261
 262 /* Sequence service handlers */
 263 #define DEF_SEQ_HDL(flags, name, fn)                                    \
 264         DEFINE_RPC_HANDLER(SEQ_QUERY, flags, name, fn, &RQF_ ## name)
 265
 266 static struct mdt_handler mdt_seq_ops[] = {
 267 DEF_SEQ_HDL(0,                          SEQ_QUERY,        (void *)seq_query),
 268 };
 269
 270 struct mdt_opc_slice mdt_seq_handlers[] = {
 271         {
 272                 .mos_opc_start = SEQ_QUERY,
 273                 .mos_opc_end   = SEQ_LAST_OPC,
 274                 .mos_hs = mdt_seq_ops
 275         },
 276         {
 277                 .mos_hs = NULL
 278         }
 279 };
 280
 281 /* FID Location Database handlers */
 282 #define DEF_FLD_HDL(flags, name, fn)                                    \
 283         DEFINE_RPC_HANDLER(FLD_QUERY, flags, name, fn, &RQF_ ## name)
 284
 285 static struct mdt_handler mdt_fld_ops[] = {
 286 DEF_FLD_HDL(0,                          FLD_QUERY,        (void *)fld_query),
 287 };
 288
 289 struct mdt_opc_slice mdt_fld_handlers[] = {
 290         {
 291                 .mos_opc_start = FLD_QUERY,
 292                 .mos_opc_end   = FLD_LAST_OPC,
 293                 .mos_hs = mdt_fld_ops
 294         },
 295         {
 296                 .mos_hs = NULL
 297         }
 298 };
 299
 300 static int mds_regular_handle(struct ptlrpc_request *req)
 301 {
 302         return mdt_handle_common(req, mdt_regular_handlers);
 303 }
 304
 305 static int mds_readpage_handle(struct ptlrpc_request *req)
 306 {
 307         return mdt_handle_common(req, mdt_readpage_handlers);
 308 }
 309
 310 static int mds_mdsc_handle(struct ptlrpc_request *req)
 311 {
 312         return mdt_handle_common(req, mdt_seq_handlers);
 313 }
 314
 315 static int mds_mdss_handle(struct ptlrpc_request *req)
 316 {
 317         return mdt_handle_common(req, mdt_seq_handlers);
 318 }
 319
 320 static int mds_fld_handle(struct ptlrpc_request *req)
 321 {
 322         return mdt_handle_common(req, mdt_fld_handlers);
 323 }
 324
 325 /* device init/fini methods */
 326 static void mds_stop_ptlrpc_service(struct mds_device *m)
 327 {
 328         ENTRY;
 329         if (m->mds_regular_service != NULL) {
 330                 ptlrpc_unregister_service(m->mds_regular_service);
 331                 m->mds_regular_service = NULL;
 332         }
 333         if (m->mds_readpage_service != NULL) {
 334                 ptlrpc_unregister_service(m->mds_readpage_service);
 335                 m->mds_readpage_service = NULL;
 336         }
 337         if (m->mds_setattr_service != NULL) {
 338                 ptlrpc_unregister_service(m->mds_setattr_service);
 339                 m->mds_setattr_service = NULL;
 340         }
 341         if (m->mds_mdsc_service != NULL) {
 342                 ptlrpc_unregister_service(m->mds_mdsc_service);
 343                 m->mds_mdsc_service = NULL;
 344         }
 345         if (m->mds_mdss_service != NULL) {
 346                 ptlrpc_unregister_service(m->mds_mdss_service);
 347                 m->mds_mdss_service = NULL;
 348         }
 349         if (m->mds_fld_service != NULL) {
 350                 ptlrpc_unregister_service(m->mds_fld_service);
 351                 m->mds_fld_service = NULL;
 352         }
 353         EXIT;
 354 }
 355
 356 static int mds_start_ptlrpc_service(struct mds_device *m)
 357 {
 358         static struct ptlrpc_service_conf conf;
 359         struct obd_device *obd = m->mds_md_dev.md_lu_dev.ld_obd;
 360         cfs_proc_dir_entry_t *procfs_entry;
 361         int rc = 0;
 362         ENTRY;
 363
 364         procfs_entry = obd->obd_proc_entry;
 365         LASSERT(procfs_entry != NULL);
 366
 367         conf = (typeof(conf)) {
 368                 .psc_name               = LUSTRE_MDT_NAME,
 369                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 370                 .psc_buf                = {
 371                         .bc_nbufs               = MDS_NBUFS,
 372                         .bc_buf_size            = MDS_BUFSIZE,
 373                         .bc_req_max_size        = MDS_MAXREQSIZE,
 374                         .bc_rep_max_size        = MDS_MAXREPSIZE,
 375                         .bc_req_portal          = MDS_REQUEST_PORTAL,
 376                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 377                 },
 378                 /*
 379                  * We'd like to have a mechanism to set this on a per-device
 380                  * basis, but alas...
 381                  */
 382                 .psc_thr                = {
 383                         .tc_thr_name            = LUSTRE_MDT_NAME,
 384                         .tc_thr_factor          = MDS_THR_FACTOR,
 385                         .tc_nthrs_init          = MDS_NTHRS_INIT,
 386                         .tc_nthrs_base          = MDS_NTHRS_BASE,
 387                         .tc_nthrs_max           = MDS_NTHRS_MAX,
 388                         .tc_nthrs_user          = mds_num_threads,
 389                         .tc_cpu_affinity        = 1,
 390                         .tc_ctx_tags            = LCT_MD_THREAD,
 391                 },
 392                 .psc_cpt                = {
 393                         .cc_pattern             = mds_num_cpts,
 394                 },
 395                 .psc_ops                = {
 396                         .so_req_handler         = mds_regular_handle,
 397                         .so_req_printer         = target_print_req,
 398                         .so_hpreq_handler       = ptlrpc_hpreq_handler,
 399                 },
 400         };
 401         m->mds_regular_service = ptlrpc_register_service(&conf, procfs_entry);
 402         if (IS_ERR(m->mds_regular_service)) {
 403                 rc = PTR_ERR(m->mds_regular_service);
 404                 CERROR("failed to start regular mdt service: %d\n", rc);
 405                 m->mds_regular_service = NULL;
 406
 407                 RETURN(rc);
 408         }
 409
 410         /*
 411          * readpage service configuration. Parameters have to be adjusted,
 412          * ideally.
 413          */
 414         memset(&conf, 0, sizeof(conf));
 415         conf = (typeof(conf)) {
 416                 .psc_name               = LUSTRE_MDT_NAME "_readpage",
 417                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 418                 .psc_buf                = {
 419                         .bc_nbufs               = MDS_NBUFS,
 420                         .bc_buf_size            = MDS_BUFSIZE,
 421                         .bc_req_max_size        = MDS_MAXREQSIZE,
 422                         .bc_rep_max_size        = MDS_MAXREPSIZE,
 423                         .bc_req_portal          = MDS_READPAGE_PORTAL,
 424                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 425                 },
 426                 .psc_thr                = {
 427                         .tc_thr_name            = LUSTRE_MDT_NAME "_rdpg",
 428                         .tc_thr_factor          = MDS_RDPG_THR_FACTOR,
 429                         .tc_nthrs_init          = MDS_RDPG_NTHRS_INIT,
 430                         .tc_nthrs_base          = MDS_RDPG_NTHRS_BASE,
 431                         .tc_nthrs_max           = MDS_RDPG_NTHRS_MAX,
 432                         .tc_nthrs_user          = mds_rdpg_num_threads,
 433                         .tc_cpu_affinity        = 1,
 434                         .tc_ctx_tags            = LCT_MD_THREAD,
 435                 },
 436                 .psc_cpt                = {
 437                         .cc_pattern             = mds_rdpg_num_cpts,
 438                 },
 439                 .psc_ops                = {
 440                         .so_req_handler         = mds_readpage_handle,
 441                         .so_req_printer         = target_print_req,
 442                 },
 443         };
 444         m->mds_readpage_service = ptlrpc_register_service(&conf, procfs_entry);
 445         if (IS_ERR(m->mds_readpage_service)) {
 446                 rc = PTR_ERR(m->mds_readpage_service);
 447                 CERROR("failed to start readpage service: %d\n", rc);
 448                 m->mds_readpage_service = NULL;
 449
 450                 GOTO(err_mds_svc, rc);
 451         }
 452
 453         /*
 454          * setattr service configuration.
 455          *
 456          * XXX To keep the compatibility with old client(< 2.2), we need to
 457          * preserve this portal for a certain time, it should be removed
 458          * eventually. LU-617.
 459          */
 460         memset(&conf, 0, sizeof(conf));
 461         conf = (typeof(conf)) {
 462                 .psc_name               = LUSTRE_MDT_NAME "_setattr",
 463                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 464                 .psc_buf                = {
 465                         .bc_nbufs               = MDS_NBUFS,
 466                         .bc_buf_size            = MDS_BUFSIZE,
 467                         .bc_req_max_size        = MDS_MAXREQSIZE,
 468                         .bc_rep_max_size        = MDS_MAXREPSIZE,
 469                         .bc_req_portal          = MDS_SETATTR_PORTAL,
 470                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 471                 },
 472                 .psc_thr                = {
 473                         .tc_thr_name            = LUSTRE_MDT_NAME "_attr",
 474                         .tc_thr_factor          = MDS_SETA_THR_FACTOR,
 475                         .tc_nthrs_init          = MDS_SETA_NTHRS_INIT,
 476                         .tc_nthrs_base          = MDS_SETA_NTHRS_BASE,
 477                         .tc_nthrs_max           = MDS_SETA_NTHRS_MAX,
 478                         .tc_nthrs_user          = mds_attr_num_threads,
 479                         .tc_cpu_affinity        = 1,
 480                         .tc_ctx_tags            = LCT_MD_THREAD,
 481                 },
 482                 .psc_cpt                = {
 483                         .cc_pattern             = mds_attr_num_cpts,
 484                 },
 485                 .psc_ops                = {
 486                         .so_req_handler         = mds_regular_handle,
 487                         .so_req_printer         = target_print_req,
 488                         .so_hpreq_handler       = NULL,
 489                 },
 490         };
 491         m->mds_setattr_service = ptlrpc_register_service(&conf, procfs_entry);
 492         if (IS_ERR(m->mds_setattr_service)) {
 493                 rc = PTR_ERR(m->mds_setattr_service);
 494                 CERROR("failed to start setattr service: %d\n", rc);
 495                 m->mds_setattr_service = NULL;
 496
 497                 GOTO(err_mds_svc, rc);
 498         }
 499
 500         /*
 501          * sequence controller service configuration
 502          */
 503         memset(&conf, 0, sizeof(conf));
 504         conf = (typeof(conf)) {
 505                 .psc_name               = LUSTRE_MDT_NAME "_seqs",
 506                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 507                 .psc_buf                = {
 508                         .bc_nbufs               = MDS_NBUFS,
 509                         .bc_buf_size            = MDS_BUFSIZE,
 510                         .bc_req_max_size        = SEQ_MAXREQSIZE,
 511                         .bc_rep_max_size        = SEQ_MAXREPSIZE,
 512                         .bc_req_portal          = SEQ_CONTROLLER_PORTAL,
 513                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 514                 },
 515                 .psc_thr                = {
 516                         .tc_thr_name            = LUSTRE_MDT_NAME "_seqs",
 517                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
 518                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
 519                         .tc_ctx_tags            = LCT_MD_THREAD,
 520                 },
 521                 .psc_ops                = {
 522                         .so_req_handler         = mds_mdsc_handle,
 523                         .so_req_printer         = target_print_req,
 524                         .so_hpreq_handler       = NULL,
 525                 },
 526         };
 527         m->mds_mdsc_service = ptlrpc_register_service(&conf, procfs_entry);
 528         if (IS_ERR(m->mds_mdsc_service)) {
 529                 rc = PTR_ERR(m->mds_mdsc_service);
 530                 CERROR("failed to start seq controller service: %d\n", rc);
 531                 m->mds_mdsc_service = NULL;
 532
 533                 GOTO(err_mds_svc, rc);
 534         }
 535
 536         /*
 537          * metadata sequence server service configuration
 538          */
 539         memset(&conf, 0, sizeof(conf));
 540         conf = (typeof(conf)) {
 541                 .psc_name               = LUSTRE_MDT_NAME "_seqm",
 542                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 543                 .psc_buf                = {
 544                         .bc_nbufs               = MDS_NBUFS,
 545                         .bc_buf_size            = MDS_BUFSIZE,
 546                         .bc_req_max_size        = SEQ_MAXREQSIZE,
 547                         .bc_rep_max_size        = SEQ_MAXREPSIZE,
 548                         .bc_req_portal          = SEQ_METADATA_PORTAL,
 549                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 550                 },
 551                 .psc_thr                = {
 552                         .tc_thr_name            = LUSTRE_MDT_NAME "_seqm",
 553                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
 554                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
 555                         .tc_ctx_tags            = LCT_MD_THREAD | LCT_DT_THREAD
 556                 },
 557                 .psc_ops                = {
 558                         .so_req_handler         = mds_mdss_handle,
 559                         .so_req_printer         = target_print_req,
 560                         .so_hpreq_handler       = NULL,
 561                 },
 562         };
 563         m->mds_mdss_service = ptlrpc_register_service(&conf, procfs_entry);
 564         if (IS_ERR(m->mds_mdss_service)) {
 565                 rc = PTR_ERR(m->mds_mdss_service);
 566                 CERROR("failed to start metadata seq server service: %d\n", rc);
 567                 m->mds_mdss_service = NULL;
 568
 569                 GOTO(err_mds_svc, rc);
 570         }
 571
 572         /* FLD service start */
 573         memset(&conf, 0, sizeof(conf));
 574         conf = (typeof(conf)) {
 575                 .psc_name            = LUSTRE_MDT_NAME "_fld",
 576                 .psc_watchdog_factor = MDT_SERVICE_WATCHDOG_FACTOR,
 577                 .psc_buf                = {
 578                         .bc_nbufs               = MDS_NBUFS,
 579                         .bc_buf_size            = MDS_BUFSIZE,
 580                         .bc_req_max_size        = FLD_MAXREQSIZE,
 581                         .bc_rep_max_size        = FLD_MAXREPSIZE,
 582                         .bc_req_portal          = FLD_REQUEST_PORTAL,
 583                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 584                 },
 585                 .psc_thr                = {
 586                         .tc_thr_name            = LUSTRE_MDT_NAME "_fld",
 587                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
 588                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
 589                         .tc_ctx_tags            = LCT_DT_THREAD | LCT_MD_THREAD
 590                 },
 591                 .psc_ops                = {
 592                         .so_req_handler         = mds_fld_handle,
 593                         .so_req_printer         = target_print_req,
 594                         .so_hpreq_handler       = NULL,
 595                 },
 596         };
 597         m->mds_fld_service = ptlrpc_register_service(&conf, procfs_entry);
 598         if (IS_ERR(m->mds_fld_service)) {
 599                 rc = PTR_ERR(m->mds_fld_service);
 600                 CERROR("failed to start fld service: %d\n", rc);
 601                 m->mds_fld_service = NULL;
 602
 603                 GOTO(err_mds_svc, rc);
 604         }
 605
 606         EXIT;
 607 err_mds_svc:
 608         if (rc)
 609                 mds_stop_ptlrpc_service(m);
 610
 611         return rc;
 612 }
 613
 614 static inline struct mds_device *mds_dev(struct lu_device *d)
 615 {
 616         return container_of0(d, struct mds_device, mds_md_dev.md_lu_dev);
 617 }
 618
 619 static struct lu_device *mds_device_fini(const struct lu_env *env,
 620                                          struct lu_device *d)
 621 {
 622         struct mds_device *m = mds_dev(d);
 623         struct obd_device *obd = d->ld_obd;
 624         ENTRY;
 625
 626         mds_stop_ptlrpc_service(m);
 627         lprocfs_obd_cleanup(obd);
 628         RETURN(NULL);
 629 }
 630
 631 static struct lu_device *mds_device_free(const struct lu_env *env,
 632                                          struct lu_device *d)
 633 {
 634         struct mds_device *m = mds_dev(d);
 635         ENTRY;
 636
 637         md_device_fini(&m->mds_md_dev);
 638         OBD_FREE_PTR(m);
 639         RETURN(NULL);
 640 }
 641
 642 static struct lu_device *mds_device_alloc(const struct lu_env *env,
 643                                           struct lu_device_type *t,
 644                                           struct lustre_cfg *cfg)
 645 {
 646         struct mds_device        *m;
 647         struct obd_device        *obd;
 648         struct lu_device          *l;
 649         int rc;
 650
 651         OBD_ALLOC_PTR(m);
 652         if (m == NULL)
 653                 return ERR_PTR(-ENOMEM);
 654
 655         md_device_init(&m->mds_md_dev, t);
 656         l = &m->mds_md_dev.md_lu_dev;
 657
 658         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 659         LASSERT(obd != NULL);
 660
 661         l->ld_obd = obd;
 662         /* set this lu_device to obd, because error handling need it */
 663         obd->obd_lu_dev = l;
 664
 665         rc = lprocfs_obd_setup(obd, lprocfs_mds_obd_vars);
 666         if (rc != 0) {
 667                 mds_device_free(env, l);
 668                 l = ERR_PTR(rc);
 669                 return l;
 670         }
 671
 672         rc = mds_start_ptlrpc_service(m);
 673
 674         if (rc != 0) {
 675                 mds_device_free(env, l);
 676                 l = ERR_PTR(rc);
 677                 return l;
 678         }
 679
 680         return l;
 681 }
 682
 683 /* type constructor/destructor: mdt_type_init, mdt_type_fini */
 684 LU_TYPE_INIT_FINI(mds, &mdt_thread_key);
 685
 686 static struct lu_device_type_operations mds_device_type_ops = {
 687         .ldto_init = mds_type_init,
 688         .ldto_fini = mds_type_fini,
 689
 690         .ldto_start = mds_type_start,
 691         .ldto_stop  = mds_type_stop,
 692
 693         .ldto_device_alloc = mds_device_alloc,
 694         .ldto_device_free  = mds_device_free,
 695         .ldto_device_fini  = mds_device_fini
 696 };
 697
 698 static struct lu_device_type mds_device_type = {
 699         .ldt_tags     = LU_DEVICE_MD,
 700         .ldt_name     = LUSTRE_MDS_NAME,
 701         .ldt_ops      = &mds_device_type_ops,
 702         .ldt_ctx_tags = LCT_MD_THREAD
 703 };
 704
 705 static struct obd_ops mds_obd_device_ops = {
 706         .o_owner           = THIS_MODULE,
 707 };
 708
 709 int mds_mod_init(void)
 710 {
 711         int rc;
 712
 713         if (mdt_num_threads != 0 && mds_num_threads == 0) {
 714                 LCONSOLE_INFO("mdt_num_threads module parameter is deprecated, "
 715                               "use mds_num_threads instead or unset both for "
 716                               "dynamic thread startup\n");
 717                 mds_num_threads = mdt_num_threads;
 718         }
 719
 720         rc = class_register_type(&mds_obd_device_ops, NULL,
 721                                  lprocfs_mds_module_vars, LUSTRE_MDS_NAME,
 722                                  &mds_device_type);
 723         return rc;
 724 }
 725
 726 void mds_mod_exit(void)
 727 {
 728         class_unregister_type(LUSTRE_MDS_NAME);
 729 }