lustre/mdt/mdt_mds.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License version 2 for more details.  A copy is
  14  * included in the COPYING file that accompanied this code.
  15
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2013, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  *
  31  * lustre/mdt/mdt_mds.c
  32  *
  33  * Lustre Metadata Service Layer
  34  *
  35  * Author: Di Wang <di.wang@whamcloud.com>
  36  **/
  37
  38 #define DEBUG_SUBSYSTEM S_MDS
  39
  40 #include <linux/module.h>
  41
  42 #include <obd_support.h>
  43 /* struct ptlrpc_request */
  44 #include <lustre_net.h>
  45 /* struct obd_export */
  46 #include <lustre_export.h>
  47 /* struct obd_device */
  48 #include <obd.h>
  49 /* lu2dt_dev() */
  50 #include <dt_object.h>
  51 #include <lustre_mds.h>
  52 #include <lustre_mdt.h>
  53 #include "mdt_internal.h"
  54 #include <lustre_quota.h>
  55 #include <lustre_acl.h>
  56 #include <lustre_param.h>
  57
  58 struct mds_device {
  59         /* super-class */
  60         struct md_device           mds_md_dev;
  61         struct ptlrpc_service     *mds_regular_service;
  62         struct ptlrpc_service     *mds_readpage_service;
  63         struct ptlrpc_service     *mds_out_service;
  64         struct ptlrpc_service     *mds_setattr_service;
  65         struct ptlrpc_service     *mds_mdsc_service;
  66         struct ptlrpc_service     *mds_mdss_service;
  67         struct ptlrpc_service     *mds_fld_service;
  68 };
  69
  70 /*
  71  *  * Initialized in mdt_mod_init().
  72  *   */
  73 static unsigned long mdt_num_threads;
  74 CFS_MODULE_PARM(mdt_num_threads, "ul", ulong, 0444,
  75                 "number of MDS service threads to start "
  76                 "(deprecated in favor of mds_num_threads)");
  77
  78 static unsigned long mds_num_threads;
  79 CFS_MODULE_PARM(mds_num_threads, "ul", ulong, 0444,
  80                 "number of MDS service threads to start");
  81
  82 static char *mds_num_cpts;
  83 CFS_MODULE_PARM(mds_num_cpts, "c", charp, 0444,
  84                 "CPU partitions MDS threads should run on");
  85
  86 static unsigned long mds_rdpg_num_threads;
  87 CFS_MODULE_PARM(mds_rdpg_num_threads, "ul", ulong, 0444,
  88                 "number of MDS readpage service threads to start");
  89
  90 static char *mds_rdpg_num_cpts;
  91 CFS_MODULE_PARM(mds_rdpg_num_cpts, "c", charp, 0444,
  92                 "CPU partitions MDS readpage threads should run on");
  93
  94 /* NB: these two should be removed along with setattr service in the future */
  95 static unsigned long mds_attr_num_threads;
  96 CFS_MODULE_PARM(mds_attr_num_threads, "ul", ulong, 0444,
  97                 "number of MDS setattr service threads to start");
  98
  99 static char *mds_attr_num_cpts;
 100 CFS_MODULE_PARM(mds_attr_num_cpts, "c", charp, 0444,
 101                 "CPU partitions MDS setattr threads should run on");
 102
 103 #define DEFINE_RPC_HANDLER(base, flags, opc, fn, fmt)                   \
 104 [opc - base] = {                                                        \
 105         .mh_name        = #opc,                                         \
 106         .mh_fail_id     = OBD_FAIL_ ## opc ## _NET,                     \
 107         .mh_opc         = opc,                                          \
 108         .mh_flags       = flags,                                        \
 109         .mh_act         = fn,                                           \
 110         .mh_fmt         = fmt                                           \
 111 }
 112
 113 /* Request with a format known in advance */
 114 #define DEF_MDT_HDL(flags, name, fn)                                    \
 115         DEFINE_RPC_HANDLER(MDS_GETATTR, flags, name, fn, &RQF_ ## name)
 116
 117 /* Request with a format we do not yet know */
 118 #define DEF_MDT_HDL_VAR(flags, name, fn)                                \
 119         DEFINE_RPC_HANDLER(MDS_GETATTR, flags, name, fn, NULL)
 120
 121 /* Map one non-standard request format handler.  This should probably get
 122  * a common OBD_SET_INFO RPC opcode instead of this mismatch. */
 123 #define RQF_MDS_SET_INFO RQF_OBD_SET_INFO
 124
 125 static struct mdt_handler mdt_mds_ops[] = {
 126 DEF_MDT_HDL(0,                          MDS_CONNECT,      mdt_connect),
 127 DEF_MDT_HDL(0,                          MDS_DISCONNECT,   mdt_disconnect),
 128 DEF_MDT_HDL(0,                          MDS_SET_INFO,     mdt_set_info),
 129 DEF_MDT_HDL(0,                          MDS_GET_INFO,     mdt_get_info),
 130 DEF_MDT_HDL(0           | HABEO_REFERO, MDS_GETSTATUS,    mdt_getstatus),
 131 DEF_MDT_HDL(HABEO_CORPUS,               MDS_GETATTR,      mdt_getattr),
 132 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_GETATTR_NAME, mdt_getattr_name),
 133 DEF_MDT_HDL(HABEO_CORPUS,               MDS_GETXATTR,     mdt_getxattr),
 134 DEF_MDT_HDL(0           | HABEO_REFERO, MDS_STATFS,       mdt_statfs),
 135 DEF_MDT_HDL(0           | MUTABOR,      MDS_REINT,        mdt_reint),
 136 DEF_MDT_HDL(HABEO_CORPUS,               MDS_CLOSE,        mdt_close),
 137 DEF_MDT_HDL(HABEO_CORPUS,               MDS_DONE_WRITING, mdt_done_writing),
 138 DEF_MDT_HDL(0           | HABEO_REFERO, MDS_PIN,          mdt_pin),
 139 DEF_MDT_HDL_VAR(0,                      MDS_SYNC,         mdt_sync),
 140 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_IS_SUBDIR,    mdt_is_subdir),
 141 DEF_MDT_HDL(0,                          MDS_QUOTACHECK,   mdt_quotacheck),
 142 DEF_MDT_HDL(0,                          MDS_QUOTACTL,     mdt_quotactl),
 143 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_PROGRESS, mdt_hsm_progress),
 144 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_CT_REGISTER,
 145                                                 mdt_hsm_ct_register),
 146 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_CT_UNREGISTER,
 147                                                 mdt_hsm_ct_unregister),
 148 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_STATE_GET,
 149                                                 mdt_hsm_state_get),
 150 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_STATE_SET,
 151                                                 mdt_hsm_state_set),
 152 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_ACTION, mdt_hsm_action),
 153 DEF_MDT_HDL(HABEO_CORPUS| HABEO_REFERO, MDS_HSM_REQUEST, mdt_hsm_request),
 154 DEF_MDT_HDL(HABEO_CORPUS|HABEO_REFERO,  MDS_SWAP_LAYOUTS, mdt_swap_layouts)
 155 };
 156
 157 #define DEF_OBD_HDL(flags, name, fn)                                    \
 158         DEFINE_RPC_HANDLER(OBD_PING, flags, name, fn, NULL)
 159
 160 static struct mdt_handler mdt_obd_ops[] = {
 161 DEF_OBD_HDL(0,                          OBD_PING,         mdt_obd_ping),
 162 DEF_OBD_HDL(0,                          OBD_LOG_CANCEL,   mdt_obd_log_cancel),
 163 DEF_OBD_HDL(0,                          OBD_QC_CALLBACK,  mdt_obd_qc_callback),
 164 DEF_OBD_HDL(0,                          OBD_IDX_READ,     mdt_obd_idx_read)
 165 };
 166
 167 #define DEF_DLM_HDL_VAR(flags, name, fn)                                \
 168         DEFINE_RPC_HANDLER(LDLM_ENQUEUE, flags, name, fn, NULL)
 169 #define DEF_DLM_HDL(flags, name, fn)                                    \
 170         DEFINE_RPC_HANDLER(LDLM_ENQUEUE, flags, name, fn, &RQF_ ## name)
 171
 172 static struct mdt_handler mdt_dlm_ops[] = {
 173 DEF_DLM_HDL    (HABEO_CLAVIS,           LDLM_ENQUEUE,     mdt_enqueue),
 174 DEF_DLM_HDL_VAR(HABEO_CLAVIS,           LDLM_CONVERT,     mdt_convert),
 175 DEF_DLM_HDL_VAR(0,                      LDLM_BL_CALLBACK, mdt_bl_callback),
 176 DEF_DLM_HDL_VAR(0,                      LDLM_CP_CALLBACK, mdt_cp_callback)
 177 };
 178
 179 #define DEF_LLOG_HDL(flags, name, fn)                                   \
 180         DEFINE_RPC_HANDLER(LLOG_ORIGIN_HANDLE_CREATE, flags, name, fn, NULL)
 181
 182 static struct mdt_handler mdt_llog_ops[] = {
 183 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_CREATE,        mdt_llog_create),
 184 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_NEXT_BLOCK,    mdt_llog_next_block),
 185 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_READ_HEADER,   mdt_llog_read_header),
 186 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_WRITE_REC,     NULL),
 187 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_CLOSE,         NULL),
 188 DEF_LLOG_HDL(0,         LLOG_ORIGIN_CONNECT,              NULL),
 189 DEF_LLOG_HDL(0,         LLOG_CATINFO,                     NULL),
 190 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_PREV_BLOCK,    mdt_llog_prev_block),
 191 DEF_LLOG_HDL(0,         LLOG_ORIGIN_HANDLE_DESTROY,       mdt_llog_destroy),
 192 };
 193
 194 #define DEF_SEC_HDL(flags, name, fn)                                    \
 195         DEFINE_RPC_HANDLER(SEC_CTX_INIT, flags, name, fn, NULL)
 196
 197 static struct mdt_handler mdt_sec_ctx_ops[] = {
 198 DEF_SEC_HDL(0,                          SEC_CTX_INIT,     mdt_sec_ctx_handle),
 199 DEF_SEC_HDL(0,                          SEC_CTX_INIT_CONT,mdt_sec_ctx_handle),
 200 DEF_SEC_HDL(0,                          SEC_CTX_FINI,     mdt_sec_ctx_handle)
 201 };
 202
 203 #define DEF_QUOTA_HDL(flags, name, fn)                          \
 204         DEFINE_RPC_HANDLER(QUOTA_DQACQ, flags, name, fn, &RQF_ ## name)
 205
 206 static struct mdt_handler mdt_quota_ops[] = {
 207 DEF_QUOTA_HDL(HABEO_REFERO,             QUOTA_DQACQ,      mdt_quota_dqacq),
 208 };
 209
 210 struct mdt_opc_slice mdt_regular_handlers[] = {
 211         {
 212                 .mos_opc_start  = MDS_GETATTR,
 213                 .mos_opc_end    = MDS_LAST_OPC,
 214                 .mos_hs         = mdt_mds_ops
 215         },
 216         {
 217                 .mos_opc_start  = OBD_PING,
 218                 .mos_opc_end    = OBD_LAST_OPC,
 219                 .mos_hs         = mdt_obd_ops
 220         },
 221         {
 222                 .mos_opc_start  = LDLM_ENQUEUE,
 223                 .mos_opc_end    = LDLM_LAST_OPC,
 224                 .mos_hs         = mdt_dlm_ops
 225         },
 226         {
 227                 .mos_opc_start  = LLOG_ORIGIN_HANDLE_CREATE,
 228                 .mos_opc_end    = LLOG_LAST_OPC,
 229                 .mos_hs         = mdt_llog_ops
 230         },
 231         {
 232                 .mos_opc_start  = SEC_CTX_INIT,
 233                 .mos_opc_end    = SEC_LAST_OPC,
 234                 .mos_hs         = mdt_sec_ctx_ops
 235         },
 236         {
 237                 .mos_opc_start  = QUOTA_DQACQ,
 238                 .mos_opc_end    = QUOTA_LAST_OPC,
 239                 .mos_hs         = mdt_quota_ops
 240         },
 241         {
 242                 .mos_hs         = NULL
 243         }
 244 };
 245
 246 /* Readpage/readdir handlers */
 247 static struct mdt_handler mdt_readpage_ops[] = {
 248 DEF_MDT_HDL(0,                  MDS_CONNECT,  mdt_connect),
 249 DEF_MDT_HDL(HABEO_CORPUS | HABEO_REFERO, MDS_READPAGE, mdt_readpage),
 250 /* XXX: this is ugly and should be fixed one day, see mdc_close() for
 251  * detailed comments. --umka */
 252 DEF_MDT_HDL(HABEO_CORPUS,               MDS_CLOSE,        mdt_close),
 253 DEF_MDT_HDL(HABEO_CORPUS,               MDS_DONE_WRITING, mdt_done_writing),
 254 };
 255
 256 static struct mdt_opc_slice mdt_readpage_handlers[] = {
 257         {
 258                 .mos_opc_start = MDS_GETATTR,
 259                 .mos_opc_end   = MDS_LAST_OPC,
 260                 .mos_hs = mdt_readpage_ops
 261         },
 262         {
 263                 .mos_opc_start = OBD_FIRST_OPC,
 264                 .mos_opc_end   = OBD_LAST_OPC,
 265                 .mos_hs = mdt_obd_ops
 266         },
 267         {
 268                 .mos_hs = NULL
 269         }
 270 };
 271
 272 /* Sequence service handlers */
 273 #define DEF_SEQ_HDL(flags, name, fn)                                    \
 274         DEFINE_RPC_HANDLER(SEQ_QUERY, flags, name, fn, &RQF_ ## name)
 275
 276 static struct mdt_handler mdt_seq_ops[] = {
 277 DEF_SEQ_HDL(0,                          SEQ_QUERY,        (void *)seq_query),
 278 };
 279
 280 struct mdt_opc_slice mdt_seq_handlers[] = {
 281         {
 282                 .mos_opc_start = SEQ_QUERY,
 283                 .mos_opc_end   = SEQ_LAST_OPC,
 284                 .mos_hs = mdt_seq_ops
 285         },
 286         {
 287                 .mos_hs = NULL
 288         }
 289 };
 290
 291 static int mds_regular_handle(struct ptlrpc_request *req)
 292 {
 293         return mdt_handle_common(req, mdt_regular_handlers);
 294 }
 295
 296 static int mds_readpage_handle(struct ptlrpc_request *req)
 297 {
 298         return mdt_handle_common(req, mdt_readpage_handlers);
 299 }
 300
 301 static int mds_mdsc_handle(struct ptlrpc_request *req)
 302 {
 303         return mdt_handle_common(req, mdt_seq_handlers);
 304 }
 305
 306 static int mds_mdss_handle(struct ptlrpc_request *req)
 307 {
 308         return mdt_handle_common(req, mdt_seq_handlers);
 309 }
 310
 311 /* device init/fini methods */
 312 static void mds_stop_ptlrpc_service(struct mds_device *m)
 313 {
 314         ENTRY;
 315         if (m->mds_regular_service != NULL) {
 316                 ptlrpc_unregister_service(m->mds_regular_service);
 317                 m->mds_regular_service = NULL;
 318         }
 319         if (m->mds_readpage_service != NULL) {
 320                 ptlrpc_unregister_service(m->mds_readpage_service);
 321                 m->mds_readpage_service = NULL;
 322         }
 323         if (m->mds_out_service != NULL) {
 324                 ptlrpc_unregister_service(m->mds_out_service);
 325                 m->mds_out_service = NULL;
 326         }
 327         if (m->mds_setattr_service != NULL) {
 328                 ptlrpc_unregister_service(m->mds_setattr_service);
 329                 m->mds_setattr_service = NULL;
 330         }
 331         if (m->mds_mdsc_service != NULL) {
 332                 ptlrpc_unregister_service(m->mds_mdsc_service);
 333                 m->mds_mdsc_service = NULL;
 334         }
 335         if (m->mds_mdss_service != NULL) {
 336                 ptlrpc_unregister_service(m->mds_mdss_service);
 337                 m->mds_mdss_service = NULL;
 338         }
 339         if (m->mds_fld_service != NULL) {
 340                 ptlrpc_unregister_service(m->mds_fld_service);
 341                 m->mds_fld_service = NULL;
 342         }
 343         EXIT;
 344 }
 345
 346 static int mds_start_ptlrpc_service(struct mds_device *m)
 347 {
 348         static struct ptlrpc_service_conf conf;
 349         struct obd_device *obd = m->mds_md_dev.md_lu_dev.ld_obd;
 350         cfs_proc_dir_entry_t *procfs_entry;
 351         int rc = 0;
 352         ENTRY;
 353
 354         procfs_entry = obd->obd_proc_entry;
 355         LASSERT(procfs_entry != NULL);
 356
 357         conf = (typeof(conf)) {
 358                 .psc_name               = LUSTRE_MDT_NAME,
 359                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 360                 .psc_buf                = {
 361                         .bc_nbufs               = MDS_NBUFS,
 362                         .bc_buf_size            = MDS_REG_BUFSIZE,
 363                         .bc_req_max_size        = MDS_REG_MAXREQSIZE,
 364                         .bc_rep_max_size        = MDS_REG_MAXREPSIZE,
 365                         .bc_req_portal          = MDS_REQUEST_PORTAL,
 366                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 367                 },
 368                 /*
 369                  * We'd like to have a mechanism to set this on a per-device
 370                  * basis, but alas...
 371                  */
 372                 .psc_thr                = {
 373                         .tc_thr_name            = LUSTRE_MDT_NAME,
 374                         .tc_thr_factor          = MDS_THR_FACTOR,
 375                         .tc_nthrs_init          = MDS_NTHRS_INIT,
 376                         .tc_nthrs_base          = MDS_NTHRS_BASE,
 377                         .tc_nthrs_max           = MDS_NTHRS_MAX,
 378                         .tc_nthrs_user          = mds_num_threads,
 379                         .tc_cpu_affinity        = 1,
 380                         .tc_ctx_tags            = LCT_MD_THREAD,
 381                 },
 382                 .psc_cpt                = {
 383                         .cc_pattern             = mds_num_cpts,
 384                 },
 385                 .psc_ops                = {
 386                         .so_req_handler         = mds_regular_handle,
 387                         .so_req_printer         = target_print_req,
 388                         .so_hpreq_handler       = ptlrpc_hpreq_handler,
 389                 },
 390         };
 391         m->mds_regular_service = ptlrpc_register_service(&conf, procfs_entry);
 392         if (IS_ERR(m->mds_regular_service)) {
 393                 rc = PTR_ERR(m->mds_regular_service);
 394                 CERROR("failed to start regular mdt service: %d\n", rc);
 395                 m->mds_regular_service = NULL;
 396
 397                 RETURN(rc);
 398         }
 399
 400         /*
 401          * readpage service configuration. Parameters have to be adjusted,
 402          * ideally.
 403          */
 404         memset(&conf, 0, sizeof(conf));
 405         conf = (typeof(conf)) {
 406                 .psc_name               = LUSTRE_MDT_NAME "_readpage",
 407                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 408                 .psc_buf                = {
 409                         .bc_nbufs               = MDS_NBUFS,
 410                         .bc_buf_size            = MDS_BUFSIZE,
 411                         .bc_req_max_size        = MDS_MAXREQSIZE,
 412                         .bc_rep_max_size        = MDS_MAXREPSIZE,
 413                         .bc_req_portal          = MDS_READPAGE_PORTAL,
 414                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 415                 },
 416                 .psc_thr                = {
 417                         .tc_thr_name            = LUSTRE_MDT_NAME "_rdpg",
 418                         .tc_thr_factor          = MDS_RDPG_THR_FACTOR,
 419                         .tc_nthrs_init          = MDS_RDPG_NTHRS_INIT,
 420                         .tc_nthrs_base          = MDS_RDPG_NTHRS_BASE,
 421                         .tc_nthrs_max           = MDS_RDPG_NTHRS_MAX,
 422                         .tc_nthrs_user          = mds_rdpg_num_threads,
 423                         .tc_cpu_affinity        = 1,
 424                         .tc_ctx_tags            = LCT_MD_THREAD,
 425                 },
 426                 .psc_cpt                = {
 427                         .cc_pattern             = mds_rdpg_num_cpts,
 428                 },
 429                 .psc_ops                = {
 430                         .so_req_handler         = mds_readpage_handle,
 431                         .so_req_printer         = target_print_req,
 432                 },
 433         };
 434         m->mds_readpage_service = ptlrpc_register_service(&conf, procfs_entry);
 435         if (IS_ERR(m->mds_readpage_service)) {
 436                 rc = PTR_ERR(m->mds_readpage_service);
 437                 CERROR("failed to start readpage service: %d\n", rc);
 438                 m->mds_readpage_service = NULL;
 439
 440                 GOTO(err_mds_svc, rc);
 441         }
 442
 443         /*
 444          * setattr service configuration.
 445          *
 446          * XXX To keep the compatibility with old client(< 2.2), we need to
 447          * preserve this portal for a certain time, it should be removed
 448          * eventually. LU-617.
 449          */
 450         memset(&conf, 0, sizeof(conf));
 451         conf = (typeof(conf)) {
 452                 .psc_name               = LUSTRE_MDT_NAME "_setattr",
 453                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 454                 .psc_buf                = {
 455                         .bc_nbufs               = MDS_NBUFS,
 456                         .bc_buf_size            = MDS_BUFSIZE,
 457                         .bc_req_max_size        = MDS_MAXREQSIZE,
 458                         .bc_rep_max_size        = MDS_LOV_MAXREPSIZE,
 459                         .bc_req_portal          = MDS_SETATTR_PORTAL,
 460                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 461                 },
 462                 .psc_thr                = {
 463                         .tc_thr_name            = LUSTRE_MDT_NAME "_attr",
 464                         .tc_thr_factor          = MDS_SETA_THR_FACTOR,
 465                         .tc_nthrs_init          = MDS_SETA_NTHRS_INIT,
 466                         .tc_nthrs_base          = MDS_SETA_NTHRS_BASE,
 467                         .tc_nthrs_max           = MDS_SETA_NTHRS_MAX,
 468                         .tc_nthrs_user          = mds_attr_num_threads,
 469                         .tc_cpu_affinity        = 1,
 470                         .tc_ctx_tags            = LCT_MD_THREAD,
 471                 },
 472                 .psc_cpt                = {
 473                         .cc_pattern             = mds_attr_num_cpts,
 474                 },
 475                 .psc_ops                = {
 476                         .so_req_handler         = mds_regular_handle,
 477                         .so_req_printer         = target_print_req,
 478                         .so_hpreq_handler       = NULL,
 479                 },
 480         };
 481         m->mds_setattr_service = ptlrpc_register_service(&conf, procfs_entry);
 482         if (IS_ERR(m->mds_setattr_service)) {
 483                 rc = PTR_ERR(m->mds_setattr_service);
 484                 CERROR("failed to start setattr service: %d\n", rc);
 485                 m->mds_setattr_service = NULL;
 486
 487                 GOTO(err_mds_svc, rc);
 488         }
 489
 490         /* Object update service */
 491         conf = (typeof(conf)) {
 492                 .psc_name               = LUSTRE_MDT_NAME "_out",
 493                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 494                 .psc_buf                = {
 495                         .bc_nbufs               = MDS_NBUFS,
 496                         .bc_buf_size            = OUT_BUFSIZE,
 497                         .bc_req_max_size        = OUT_MAXREQSIZE,
 498                         .bc_rep_max_size        = OUT_MAXREPSIZE,
 499                         .bc_req_portal          = OUT_PORTAL,
 500                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 501                 },
 502                 /*
 503                  * We'd like to have a mechanism to set this on a per-device
 504                  * basis, but alas...
 505                  */
 506                 .psc_thr                = {
 507                         .tc_thr_name            = LUSTRE_MDT_NAME "_out",
 508                         .tc_thr_factor          = MDS_THR_FACTOR,
 509                         .tc_nthrs_init          = MDS_NTHRS_INIT,
 510                         .tc_nthrs_base          = MDS_NTHRS_BASE,
 511                         .tc_nthrs_max           = MDS_NTHRS_MAX,
 512                         .tc_nthrs_user          = mds_num_threads,
 513                         .tc_cpu_affinity        = 1,
 514                         .tc_ctx_tags            = LCT_MD_THREAD,
 515                 },
 516                 .psc_cpt                = {
 517                         .cc_pattern             = mds_num_cpts,
 518                 },
 519                 .psc_ops                = {
 520                         .so_req_handler         = tgt_request_handle,
 521                         .so_req_printer         = target_print_req,
 522                         .so_hpreq_handler       = NULL,
 523                 },
 524         };
 525         m->mds_out_service = ptlrpc_register_service(&conf, procfs_entry);
 526         if (IS_ERR(m->mds_out_service)) {
 527                 rc = PTR_ERR(m->mds_out_service);
 528                 CERROR("failed to start out service: %d\n", rc);
 529                 m->mds_out_service = NULL;
 530                 GOTO(err_mds_svc, rc);
 531         }
 532
 533         /*
 534          * sequence controller service configuration
 535          */
 536         memset(&conf, 0, sizeof(conf));
 537         conf = (typeof(conf)) {
 538                 .psc_name               = LUSTRE_MDT_NAME "_seqs",
 539                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 540                 .psc_buf                = {
 541                         .bc_nbufs               = MDS_NBUFS,
 542                         .bc_buf_size            = SEQ_BUFSIZE,
 543                         .bc_req_max_size        = SEQ_MAXREQSIZE,
 544                         .bc_rep_max_size        = SEQ_MAXREPSIZE,
 545                         .bc_req_portal          = SEQ_CONTROLLER_PORTAL,
 546                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 547                 },
 548                 .psc_thr                = {
 549                         .tc_thr_name            = LUSTRE_MDT_NAME "_seqs",
 550                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
 551                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
 552                         .tc_ctx_tags            = LCT_MD_THREAD,
 553                 },
 554                 .psc_ops                = {
 555                         .so_req_handler         = mds_mdsc_handle,
 556                         .so_req_printer         = target_print_req,
 557                         .so_hpreq_handler       = NULL,
 558                 },
 559         };
 560         m->mds_mdsc_service = ptlrpc_register_service(&conf, procfs_entry);
 561         if (IS_ERR(m->mds_mdsc_service)) {
 562                 rc = PTR_ERR(m->mds_mdsc_service);
 563                 CERROR("failed to start seq controller service: %d\n", rc);
 564                 m->mds_mdsc_service = NULL;
 565
 566                 GOTO(err_mds_svc, rc);
 567         }
 568
 569         /*
 570          * metadata sequence server service configuration
 571          */
 572         memset(&conf, 0, sizeof(conf));
 573         conf = (typeof(conf)) {
 574                 .psc_name               = LUSTRE_MDT_NAME "_seqm",
 575                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
 576                 .psc_buf                = {
 577                         .bc_nbufs               = MDS_NBUFS,
 578                         .bc_buf_size            = SEQ_BUFSIZE,
 579                         .bc_req_max_size        = SEQ_MAXREQSIZE,
 580                         .bc_rep_max_size        = SEQ_MAXREPSIZE,
 581                         .bc_req_portal          = SEQ_METADATA_PORTAL,
 582                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 583                 },
 584                 .psc_thr                = {
 585                         .tc_thr_name            = LUSTRE_MDT_NAME "_seqm",
 586                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
 587                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
 588                         .tc_ctx_tags            = LCT_MD_THREAD | LCT_DT_THREAD
 589                 },
 590                 .psc_ops                = {
 591                         .so_req_handler         = mds_mdss_handle,
 592                         .so_req_printer         = target_print_req,
 593                         .so_hpreq_handler       = NULL,
 594                 },
 595         };
 596         m->mds_mdss_service = ptlrpc_register_service(&conf, procfs_entry);
 597         if (IS_ERR(m->mds_mdss_service)) {
 598                 rc = PTR_ERR(m->mds_mdss_service);
 599                 CERROR("failed to start metadata seq server service: %d\n", rc);
 600                 m->mds_mdss_service = NULL;
 601
 602                 GOTO(err_mds_svc, rc);
 603         }
 604
 605         /* FLD service start */
 606         memset(&conf, 0, sizeof(conf));
 607         conf = (typeof(conf)) {
 608                 .psc_name            = LUSTRE_MDT_NAME "_fld",
 609                 .psc_watchdog_factor = MDT_SERVICE_WATCHDOG_FACTOR,
 610                 .psc_buf                = {
 611                         .bc_nbufs               = MDS_NBUFS,
 612                         .bc_buf_size            = FLD_BUFSIZE,
 613                         .bc_req_max_size        = FLD_MAXREQSIZE,
 614                         .bc_rep_max_size        = FLD_MAXREPSIZE,
 615                         .bc_req_portal          = FLD_REQUEST_PORTAL,
 616                         .bc_rep_portal          = MDC_REPLY_PORTAL,
 617                 },
 618                 .psc_thr                = {
 619                         .tc_thr_name            = LUSTRE_MDT_NAME "_fld",
 620                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
 621                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
 622                         .tc_ctx_tags            = LCT_DT_THREAD | LCT_MD_THREAD,
 623                 },
 624                 .psc_ops                = {
 625                         .so_req_handler         = tgt_request_handle,
 626                         .so_req_printer         = target_print_req,
 627                         .so_hpreq_handler       = NULL,
 628                 },
 629         };
 630         m->mds_fld_service = ptlrpc_register_service(&conf, procfs_entry);
 631         if (IS_ERR(m->mds_fld_service)) {
 632                 rc = PTR_ERR(m->mds_fld_service);
 633                 CERROR("failed to start fld service: %d\n", rc);
 634                 m->mds_fld_service = NULL;
 635
 636                 GOTO(err_mds_svc, rc);
 637         }
 638
 639         EXIT;
 640 err_mds_svc:
 641         if (rc)
 642                 mds_stop_ptlrpc_service(m);
 643
 644         return rc;
 645 }
 646
 647 static inline struct mds_device *mds_dev(struct lu_device *d)
 648 {
 649         return container_of0(d, struct mds_device, mds_md_dev.md_lu_dev);
 650 }
 651
 652 static struct lu_device *mds_device_fini(const struct lu_env *env,
 653                                          struct lu_device *d)
 654 {
 655         struct mds_device *m = mds_dev(d);
 656         struct obd_device *obd = d->ld_obd;
 657         ENTRY;
 658
 659         mds_stop_ptlrpc_service(m);
 660         lprocfs_obd_cleanup(obd);
 661         RETURN(NULL);
 662 }
 663
 664 static struct lu_device *mds_device_free(const struct lu_env *env,
 665                                          struct lu_device *d)
 666 {
 667         struct mds_device *m = mds_dev(d);
 668         ENTRY;
 669
 670         md_device_fini(&m->mds_md_dev);
 671         OBD_FREE_PTR(m);
 672         RETURN(NULL);
 673 }
 674
 675 static struct lu_device *mds_device_alloc(const struct lu_env *env,
 676                                           struct lu_device_type *t,
 677                                           struct lustre_cfg *cfg)
 678 {
 679         struct mds_device        *m;
 680         struct obd_device        *obd;
 681         struct lu_device          *l;
 682         int rc;
 683
 684         OBD_ALLOC_PTR(m);
 685         if (m == NULL)
 686                 return ERR_PTR(-ENOMEM);
 687
 688         md_device_init(&m->mds_md_dev, t);
 689         l = &m->mds_md_dev.md_lu_dev;
 690
 691         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 692         LASSERT(obd != NULL);
 693
 694         l->ld_obd = obd;
 695         /* set this lu_device to obd, because error handling need it */
 696         obd->obd_lu_dev = l;
 697
 698         rc = lprocfs_obd_setup(obd, lprocfs_mds_obd_vars);
 699         if (rc != 0) {
 700                 mds_device_free(env, l);
 701                 l = ERR_PTR(rc);
 702                 return l;
 703         }
 704
 705         rc = mds_start_ptlrpc_service(m);
 706
 707         if (rc != 0) {
 708                 mds_device_free(env, l);
 709                 l = ERR_PTR(rc);
 710                 return l;
 711         }
 712         return l;
 713 }
 714
 715 /* type constructor/destructor: mdt_type_init, mdt_type_fini */
 716 LU_TYPE_INIT_FINI(mds, &mdt_thread_key);
 717
 718 static struct lu_device_type_operations mds_device_type_ops = {
 719         .ldto_init = mds_type_init,
 720         .ldto_fini = mds_type_fini,
 721
 722         .ldto_start = mds_type_start,
 723         .ldto_stop  = mds_type_stop,
 724
 725         .ldto_device_alloc = mds_device_alloc,
 726         .ldto_device_free  = mds_device_free,
 727         .ldto_device_fini  = mds_device_fini
 728 };
 729
 730 static struct lu_device_type mds_device_type = {
 731         .ldt_tags     = LU_DEVICE_MD,
 732         .ldt_name     = LUSTRE_MDS_NAME,
 733         .ldt_ops      = &mds_device_type_ops,
 734         .ldt_ctx_tags = LCT_MD_THREAD
 735 };
 736
 737 static struct obd_ops mds_obd_device_ops = {
 738         .o_owner           = THIS_MODULE,
 739 };
 740
 741 int mds_mod_init(void)
 742 {
 743         int rc;
 744
 745         if (mdt_num_threads != 0 && mds_num_threads == 0) {
 746                 LCONSOLE_INFO("mdt_num_threads module parameter is deprecated, "
 747                               "use mds_num_threads instead or unset both for "
 748                               "dynamic thread startup\n");
 749                 mds_num_threads = mdt_num_threads;
 750         }
 751
 752         rc = class_register_type(&mds_obd_device_ops, NULL,
 753                                  lprocfs_mds_module_vars, LUSTRE_MDS_NAME,
 754                                  &mds_device_type);
 755         return rc;
 756 }
 757
 758 void mds_mod_exit(void)
 759 {
 760         class_unregister_type(LUSTRE_MDS_NAME);
 761 }