Whamcloud - gitweb
LU-8066 obd: migrate to ksets
[fs/lustre-release.git] / lustre / mdt / mdt_mds.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License version 2 for more details.  A copy is
14  * included in the COPYING file that accompanied this code.
15
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  *
20  * GPL HEADER END
21  */
22 /*
23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Use is subject to license terms.
25  *
26  * Copyright (c) 2013, 2015, Intel Corporation.
27  */
28 /*
29  * This file is part of Lustre, http://www.lustre.org/
30  *
31  * lustre/mdt/mdt_mds.c
32  *
33  * Lustre Metadata Service Layer
34  *
35  * Author: Di Wang <di.wang@whamcloud.com>
36  **/
37
38 #define DEBUG_SUBSYSTEM S_MDS
39
40 #include <linux/module.h>
41
42 #include <obd_support.h>
43 /* struct ptlrpc_request */
44 #include <lustre_net.h>
45 /* struct obd_export */
46 #include <lustre_export.h>
47 /* struct obd_device */
48 #include <obd.h>
49 /* lu2dt_dev() */
50 #include <dt_object.h>
51 #include <lustre_mds.h>
52 #include "mdt_internal.h"
53 #include <lustre_quota.h>
54 #include <lustre_acl.h>
55 #include <uapi/linux/lustre/lustre_param.h>
56
57 struct mds_device {
58         /* super-class */
59         struct md_device         mds_md_dev;
60         struct ptlrpc_service   *mds_regular_service;
61         struct ptlrpc_service   *mds_readpage_service;
62         struct ptlrpc_service   *mds_out_service;
63         struct ptlrpc_service   *mds_setattr_service;
64         struct ptlrpc_service   *mds_mdsc_service;
65         struct ptlrpc_service   *mds_mdss_service;
66         struct ptlrpc_service   *mds_fld_service;
67         struct mutex             mds_health_mutex;
68 };
69
70 /*
71  *  * Initialized in mds_mod_init().
72  *   */
73 static unsigned long mds_num_threads;
74 module_param(mds_num_threads, ulong, 0444);
75 MODULE_PARM_DESC(mds_num_threads, "number of MDS service threads to start");
76
77 static char *mds_num_cpts;
78 module_param(mds_num_cpts, charp, 0444);
79 MODULE_PARM_DESC(mds_num_cpts, "CPU partitions MDS threads should run on");
80
81 static unsigned long mds_rdpg_num_threads;
82 module_param(mds_rdpg_num_threads, ulong, 0444);
83 MODULE_PARM_DESC(mds_rdpg_num_threads,
84                  "number of MDS readpage service threads to start");
85
86 static char *mds_rdpg_num_cpts;
87 module_param(mds_rdpg_num_cpts, charp, 0444);
88 MODULE_PARM_DESC(mds_rdpg_num_cpts,
89                  "CPU partitions MDS readpage threads should run on");
90
91 /* NB: these two should be removed along with setattr service in the future */
92 static unsigned long mds_attr_num_threads;
93 module_param(mds_attr_num_threads, ulong, 0444);
94 MODULE_PARM_DESC(mds_attr_num_threads,
95                  "number of MDS setattr service threads to start");
96
97 static char *mds_attr_num_cpts;
98 module_param(mds_attr_num_cpts, charp, 0444);
99 MODULE_PARM_DESC(mds_attr_num_cpts,
100                  "CPU partitions MDS setattr threads should run on");
101
102 /* device init/fini methods */
103 static void mds_stop_ptlrpc_service(struct mds_device *m)
104 {
105         ENTRY;
106
107         mutex_lock(&m->mds_health_mutex);
108         if (m->mds_regular_service != NULL) {
109                 ptlrpc_unregister_service(m->mds_regular_service);
110                 m->mds_regular_service = NULL;
111         }
112         if (m->mds_readpage_service != NULL) {
113                 ptlrpc_unregister_service(m->mds_readpage_service);
114                 m->mds_readpage_service = NULL;
115         }
116         if (m->mds_out_service != NULL) {
117                 ptlrpc_unregister_service(m->mds_out_service);
118                 m->mds_out_service = NULL;
119         }
120         if (m->mds_setattr_service != NULL) {
121                 ptlrpc_unregister_service(m->mds_setattr_service);
122                 m->mds_setattr_service = NULL;
123         }
124         if (m->mds_mdsc_service != NULL) {
125                 ptlrpc_unregister_service(m->mds_mdsc_service);
126                 m->mds_mdsc_service = NULL;
127         }
128         if (m->mds_mdss_service != NULL) {
129                 ptlrpc_unregister_service(m->mds_mdss_service);
130                 m->mds_mdss_service = NULL;
131         }
132         if (m->mds_fld_service != NULL) {
133                 ptlrpc_unregister_service(m->mds_fld_service);
134                 m->mds_fld_service = NULL;
135         }
136         mutex_unlock(&m->mds_health_mutex);
137
138         EXIT;
139 }
140
141 static int mds_start_ptlrpc_service(struct mds_device *m)
142 {
143         static struct ptlrpc_service_conf conf;
144         struct obd_device *obd = m->mds_md_dev.md_lu_dev.ld_obd;
145         struct proc_dir_entry *procfs_entry;
146         int rc = 0;
147         ENTRY;
148
149         procfs_entry = obd->obd_proc_entry;
150         LASSERT(procfs_entry != NULL);
151
152         conf = (typeof(conf)) {
153                 .psc_name               = LUSTRE_MDT_NAME,
154                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
155                 .psc_buf                = {
156                         .bc_nbufs               = MDS_NBUFS,
157                         .bc_buf_size            = MDS_REG_BUFSIZE,
158                         .bc_req_max_size        = MDS_REG_MAXREQSIZE,
159                         .bc_rep_max_size        = MDS_REG_MAXREPSIZE,
160                         .bc_req_portal          = MDS_REQUEST_PORTAL,
161                         .bc_rep_portal          = MDC_REPLY_PORTAL,
162                 },
163                 /*
164                  * We'd like to have a mechanism to set this on a per-device
165                  * basis, but alas...
166                  */
167                 .psc_thr                = {
168                         .tc_thr_name            = LUSTRE_MDT_NAME,
169                         .tc_thr_factor          = MDS_THR_FACTOR,
170                         .tc_nthrs_init          = MDS_NTHRS_INIT,
171                         .tc_nthrs_base          = MDS_NTHRS_BASE,
172                         .tc_nthrs_max           = MDS_NTHRS_MAX,
173                         .tc_nthrs_user          = mds_num_threads,
174                         .tc_cpu_affinity        = 1,
175                         .tc_ctx_tags            = LCT_MD_THREAD,
176                 },
177                 .psc_cpt                = {
178                         .cc_pattern             = mds_num_cpts,
179                 },
180                 .psc_ops                = {
181                         .so_req_handler         = tgt_request_handle,
182                         .so_req_printer         = target_print_req,
183                         .so_hpreq_handler       = ptlrpc_hpreq_handler,
184                 },
185         };
186         m->mds_regular_service = ptlrpc_register_service(&conf, &obd->obd_kset,
187                                                          procfs_entry);
188         if (IS_ERR(m->mds_regular_service)) {
189                 rc = PTR_ERR(m->mds_regular_service);
190                 CERROR("failed to start regular mdt service: %d\n", rc);
191                 m->mds_regular_service = NULL;
192
193                 RETURN(rc);
194         }
195
196         /*
197          * readpage service configuration. Parameters have to be adjusted,
198          * ideally.
199          */
200         memset(&conf, 0, sizeof(conf));
201         conf = (typeof(conf)) {
202                 .psc_name               = LUSTRE_MDT_NAME "_readpage",
203                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
204                 .psc_buf                = {
205                         .bc_nbufs               = MDS_NBUFS,
206                         .bc_buf_size            = MDS_BUFSIZE,
207                         .bc_req_max_size        = MDS_MAXREQSIZE,
208                         .bc_rep_max_size        = MDS_MAXREPSIZE,
209                         .bc_req_portal          = MDS_READPAGE_PORTAL,
210                         .bc_rep_portal          = MDC_REPLY_PORTAL,
211                 },
212                 .psc_thr                = {
213                         .tc_thr_name            = LUSTRE_MDT_NAME "_rdpg",
214                         .tc_thr_factor          = MDS_RDPG_THR_FACTOR,
215                         .tc_nthrs_init          = MDS_RDPG_NTHRS_INIT,
216                         .tc_nthrs_base          = MDS_RDPG_NTHRS_BASE,
217                         .tc_nthrs_max           = MDS_RDPG_NTHRS_MAX,
218                         .tc_nthrs_user          = mds_rdpg_num_threads,
219                         .tc_cpu_affinity        = 1,
220                         .tc_ctx_tags            = LCT_MD_THREAD,
221                 },
222                 .psc_cpt                = {
223                         .cc_pattern             = mds_rdpg_num_cpts,
224                 },
225                 .psc_ops                = {
226                         .so_req_handler         = tgt_request_handle,
227                         .so_req_printer         = target_print_req,
228                 },
229         };
230         m->mds_readpage_service = ptlrpc_register_service(&conf, &obd->obd_kset,
231                                                           procfs_entry);
232         if (IS_ERR(m->mds_readpage_service)) {
233                 rc = PTR_ERR(m->mds_readpage_service);
234                 CERROR("failed to start readpage service: %d\n", rc);
235                 m->mds_readpage_service = NULL;
236
237                 GOTO(err_mds_svc, rc);
238         }
239
240         /*
241          * setattr service configuration.
242          *
243          * XXX To keep the compatibility with old client(< 2.2), we need to
244          * preserve this portal for a certain time, it should be removed
245          * eventually. LU-617.
246          */
247         memset(&conf, 0, sizeof(conf));
248         conf = (typeof(conf)) {
249                 .psc_name               = LUSTRE_MDT_NAME "_setattr",
250                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
251                 .psc_buf                = {
252                         .bc_nbufs               = MDS_NBUFS,
253                         .bc_buf_size            = MDS_BUFSIZE,
254                         .bc_req_max_size        = MDS_MAXREQSIZE,
255                         .bc_rep_max_size        = MDS_LOV_MAXREPSIZE,
256                         .bc_req_portal          = MDS_SETATTR_PORTAL,
257                         .bc_rep_portal          = MDC_REPLY_PORTAL,
258                 },
259                 .psc_thr                = {
260                         .tc_thr_name            = LUSTRE_MDT_NAME "_attr",
261                         .tc_thr_factor          = MDS_SETA_THR_FACTOR,
262                         .tc_nthrs_init          = MDS_SETA_NTHRS_INIT,
263                         .tc_nthrs_base          = MDS_SETA_NTHRS_BASE,
264                         .tc_nthrs_max           = MDS_SETA_NTHRS_MAX,
265                         .tc_nthrs_user          = mds_attr_num_threads,
266                         .tc_cpu_affinity        = 1,
267                         .tc_ctx_tags            = LCT_MD_THREAD,
268                 },
269                 .psc_cpt                = {
270                         .cc_pattern             = mds_attr_num_cpts,
271                 },
272                 .psc_ops                = {
273                         .so_req_handler         = tgt_request_handle,
274                         .so_req_printer         = target_print_req,
275                         .so_hpreq_handler       = NULL,
276                 },
277         };
278         m->mds_setattr_service = ptlrpc_register_service(&conf, &obd->obd_kset,
279                                                          procfs_entry);
280         if (IS_ERR(m->mds_setattr_service)) {
281                 rc = PTR_ERR(m->mds_setattr_service);
282                 CERROR("failed to start setattr service: %d\n", rc);
283                 m->mds_setattr_service = NULL;
284
285                 GOTO(err_mds_svc, rc);
286         }
287
288         /* Object update service */
289         conf = (typeof(conf)) {
290                 .psc_name               = LUSTRE_MDT_NAME "_out",
291                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
292                 .psc_buf                = {
293                         .bc_nbufs               = MDS_NBUFS,
294                         .bc_buf_size            = OUT_BUFSIZE,
295                         .bc_req_max_size        = OUT_MAXREQSIZE,
296                         .bc_rep_max_size        = OUT_MAXREPSIZE,
297                         .bc_req_portal          = OUT_PORTAL,
298                         .bc_rep_portal          = OSC_REPLY_PORTAL,
299                 },
300                 /*
301                  * We'd like to have a mechanism to set this on a per-device
302                  * basis, but alas...
303                  */
304                 .psc_thr                = {
305                         .tc_thr_name            = LUSTRE_MDT_NAME "_out",
306                         .tc_thr_factor          = MDS_THR_FACTOR,
307                         .tc_nthrs_init          = MDS_NTHRS_INIT,
308                         .tc_nthrs_base          = MDS_NTHRS_BASE,
309                         .tc_nthrs_max           = MDS_NTHRS_MAX,
310                         .tc_nthrs_user          = mds_num_threads,
311                         .tc_cpu_affinity        = 1,
312                         .tc_ctx_tags            = LCT_MD_THREAD |
313                                                   LCT_DT_THREAD,
314                 },
315                 .psc_cpt                = {
316                         .cc_pattern             = mds_num_cpts,
317                 },
318                 .psc_ops                = {
319                         .so_req_handler         = tgt_request_handle,
320                         .so_req_printer         = target_print_req,
321                         .so_hpreq_handler       = NULL,
322                 },
323         };
324         m->mds_out_service = ptlrpc_register_service(&conf, &obd->obd_kset,
325                                                      procfs_entry);
326         if (IS_ERR(m->mds_out_service)) {
327                 rc = PTR_ERR(m->mds_out_service);
328                 CERROR("failed to start out service: %d\n", rc);
329                 m->mds_out_service = NULL;
330                 GOTO(err_mds_svc, rc);
331         }
332
333         /*
334          * sequence controller service configuration
335          */
336         memset(&conf, 0, sizeof(conf));
337         conf = (typeof(conf)) {
338                 .psc_name               = LUSTRE_MDT_NAME "_seqs",
339                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
340                 .psc_buf                = {
341                         .bc_nbufs               = MDS_NBUFS,
342                         .bc_buf_size            = SEQ_BUFSIZE,
343                         .bc_req_max_size        = SEQ_MAXREQSIZE,
344                         .bc_rep_max_size        = SEQ_MAXREPSIZE,
345                         .bc_req_portal          = SEQ_CONTROLLER_PORTAL,
346                         .bc_rep_portal          = MDC_REPLY_PORTAL,
347                 },
348                 .psc_thr                = {
349                         .tc_thr_name            = LUSTRE_MDT_NAME "_seqs",
350                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
351                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
352                         .tc_ctx_tags            = LCT_MD_THREAD,
353                 },
354                 .psc_ops                = {
355                         .so_req_handler         = tgt_request_handle,
356                         .so_req_printer         = target_print_req,
357                         .so_hpreq_handler       = NULL,
358                 },
359         };
360         m->mds_mdsc_service = ptlrpc_register_service(&conf, &obd->obd_kset,
361                                                       procfs_entry);
362         if (IS_ERR(m->mds_mdsc_service)) {
363                 rc = PTR_ERR(m->mds_mdsc_service);
364                 CERROR("failed to start seq controller service: %d\n", rc);
365                 m->mds_mdsc_service = NULL;
366
367                 GOTO(err_mds_svc, rc);
368         }
369
370         /*
371          * metadata sequence server service configuration
372          */
373         memset(&conf, 0, sizeof(conf));
374         conf = (typeof(conf)) {
375                 .psc_name               = LUSTRE_MDT_NAME "_seqm",
376                 .psc_watchdog_factor    = MDT_SERVICE_WATCHDOG_FACTOR,
377                 .psc_buf                = {
378                         .bc_nbufs               = MDS_NBUFS,
379                         .bc_buf_size            = SEQ_BUFSIZE,
380                         .bc_req_max_size        = SEQ_MAXREQSIZE,
381                         .bc_rep_max_size        = SEQ_MAXREPSIZE,
382                         .bc_req_portal          = SEQ_METADATA_PORTAL,
383                         .bc_rep_portal          = MDC_REPLY_PORTAL,
384                 },
385                 .psc_thr                = {
386                         .tc_thr_name            = LUSTRE_MDT_NAME "_seqm",
387                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
388                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
389                         .tc_ctx_tags            = LCT_MD_THREAD | LCT_DT_THREAD
390                 },
391                 .psc_ops                = {
392                         .so_req_handler         = tgt_request_handle,
393                         .so_req_printer         = target_print_req,
394                         .so_hpreq_handler       = NULL,
395                 },
396         };
397         m->mds_mdss_service = ptlrpc_register_service(&conf, &obd->obd_kset,
398                                                       procfs_entry);
399         if (IS_ERR(m->mds_mdss_service)) {
400                 rc = PTR_ERR(m->mds_mdss_service);
401                 CERROR("failed to start metadata seq server service: %d\n", rc);
402                 m->mds_mdss_service = NULL;
403
404                 GOTO(err_mds_svc, rc);
405         }
406
407         /* FLD service start */
408         memset(&conf, 0, sizeof(conf));
409         conf = (typeof(conf)) {
410                 .psc_name            = LUSTRE_MDT_NAME "_fld",
411                 .psc_watchdog_factor = MDT_SERVICE_WATCHDOG_FACTOR,
412                 .psc_buf                = {
413                         .bc_nbufs               = MDS_NBUFS,
414                         .bc_buf_size            = FLD_BUFSIZE,
415                         .bc_req_max_size        = FLD_MAXREQSIZE,
416                         .bc_rep_max_size        = FLD_MAXREPSIZE,
417                         .bc_req_portal          = FLD_REQUEST_PORTAL,
418                         .bc_rep_portal          = MDC_REPLY_PORTAL,
419                 },
420                 .psc_thr                = {
421                         .tc_thr_name            = LUSTRE_MDT_NAME "_fld",
422                         .tc_nthrs_init          = MDS_OTHR_NTHRS_INIT,
423                         .tc_nthrs_max           = MDS_OTHR_NTHRS_MAX,
424                         .tc_ctx_tags            = LCT_DT_THREAD | LCT_MD_THREAD,
425                 },
426                 .psc_ops                = {
427                         .so_req_handler         = tgt_request_handle,
428                         .so_req_printer         = target_print_req,
429                         .so_hpreq_handler       = NULL,
430                 },
431         };
432         m->mds_fld_service = ptlrpc_register_service(&conf, &obd->obd_kset,
433                                                      procfs_entry);
434         if (IS_ERR(m->mds_fld_service)) {
435                 rc = PTR_ERR(m->mds_fld_service);
436                 CERROR("failed to start fld service: %d\n", rc);
437                 m->mds_fld_service = NULL;
438
439                 GOTO(err_mds_svc, rc);
440         }
441
442         EXIT;
443 err_mds_svc:
444         if (rc)
445                 mds_stop_ptlrpc_service(m);
446
447         return rc;
448 }
449
450 static inline struct mds_device *mds_dev(struct lu_device *d)
451 {
452         return container_of0(d, struct mds_device, mds_md_dev.md_lu_dev);
453 }
454
455 static struct lu_device *mds_device_fini(const struct lu_env *env,
456                                          struct lu_device *d)
457 {
458         struct mds_device *m = mds_dev(d);
459         struct obd_device *obd = d->ld_obd;
460         ENTRY;
461
462         mds_stop_ptlrpc_service(m);
463         lprocfs_obd_cleanup(obd);
464         RETURN(NULL);
465 }
466
467 static struct lu_device *mds_device_free(const struct lu_env *env,
468                                          struct lu_device *d)
469 {
470         struct mds_device *m = mds_dev(d);
471         ENTRY;
472
473         md_device_fini(&m->mds_md_dev);
474         OBD_FREE_PTR(m);
475         RETURN(NULL);
476 }
477
478 static struct lu_device *mds_device_alloc(const struct lu_env *env,
479                                           struct lu_device_type *t,
480                                           struct lustre_cfg *cfg)
481 {
482         struct mds_device        *m;
483         struct obd_device        *obd;
484         struct lu_device          *l;
485         int rc;
486
487         OBD_ALLOC_PTR(m);
488         if (m == NULL)
489                 return ERR_PTR(-ENOMEM);
490
491         md_device_init(&m->mds_md_dev, t);
492         l = &m->mds_md_dev.md_lu_dev;
493
494         obd = class_name2obd(lustre_cfg_string(cfg, 0));
495         LASSERT(obd != NULL);
496
497         l->ld_obd = obd;
498         /* set this lu_device to obd, because error handling need it */
499         obd->obd_lu_dev = l;
500
501         rc = lprocfs_obd_setup(obd, true);
502         if (rc != 0) {
503                 mds_device_free(env, l);
504                 l = ERR_PTR(rc);
505                 return l;
506         }
507
508         mutex_init(&m->mds_health_mutex);
509
510         rc = mds_start_ptlrpc_service(m);
511         if (rc != 0) {
512                 lprocfs_obd_cleanup(obd);
513                 mds_device_free(env, l);
514                 l = ERR_PTR(rc);
515                 return l;
516         }
517         return l;
518 }
519
520 /* type constructor/destructor: mdt_type_init, mdt_type_fini */
521 LU_TYPE_INIT_FINI(mds, &mdt_thread_key);
522
523 static struct lu_device_type_operations mds_device_type_ops = {
524         .ldto_init = mds_type_init,
525         .ldto_fini = mds_type_fini,
526
527         .ldto_start = mds_type_start,
528         .ldto_stop  = mds_type_stop,
529
530         .ldto_device_alloc = mds_device_alloc,
531         .ldto_device_free  = mds_device_free,
532         .ldto_device_fini  = mds_device_fini
533 };
534
535 static struct lu_device_type mds_device_type = {
536         .ldt_tags     = LU_DEVICE_MD,
537         .ldt_name     = LUSTRE_MDS_NAME,
538         .ldt_ops      = &mds_device_type_ops,
539         .ldt_ctx_tags = LCT_MD_THREAD
540 };
541
542 static int mds_health_check(const struct lu_env *env, struct obd_device *obd)
543 {
544         struct mds_device *mds = mds_dev(obd->obd_lu_dev);
545         int rc = 0;
546
547
548         mutex_lock(&mds->mds_health_mutex);
549         rc |= ptlrpc_service_health_check(mds->mds_regular_service);
550         rc |= ptlrpc_service_health_check(mds->mds_readpage_service);
551         rc |= ptlrpc_service_health_check(mds->mds_out_service);
552         rc |= ptlrpc_service_health_check(mds->mds_setattr_service);
553         rc |= ptlrpc_service_health_check(mds->mds_mdsc_service);
554         rc |= ptlrpc_service_health_check(mds->mds_mdss_service);
555         rc |= ptlrpc_service_health_check(mds->mds_fld_service);
556         mutex_unlock(&mds->mds_health_mutex);
557
558         return rc != 0 ? 1 : 0;
559 }
560
561 static struct obd_ops mds_obd_device_ops = {
562         .o_owner           = THIS_MODULE,
563         .o_health_check    = mds_health_check,
564 };
565
566 int mds_mod_init(void)
567 {
568         return class_register_type(&mds_obd_device_ops, NULL, true, NULL,
569                                    LUSTRE_MDS_NAME, &mds_device_type);
570 }
571
572 void mds_mod_exit(void)
573 {
574         class_unregister_type(LUSTRE_MDS_NAME);
575 }