Whamcloud - gitweb
LU-3088 build: fix 'resource leak' errors
[fs/lustre-release.git] / lustre / osp / osp_dev.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2012, 2013, Intel Corporation.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lustre/osp/osp_dev.c
37  *
38  * Lustre OST Proxy Device
39  *
40  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
41  * Author: Mikhail Pershin <mike.pershin@intel.com>
42  * Author: Di Wang <di.wang@intel.com>
43  */
44
45 #ifndef EXPORT_SYMTAB
46 # define EXPORT_SYMTAB
47 #endif
48 #define DEBUG_SUBSYSTEM S_MDS
49
50 #include <obd_class.h>
51 #include <lustre_param.h>
52 #include <lustre_log.h>
53 #include <lustre_mdc.h>
54
55 #include "osp_internal.h"
56
57 /* Slab for OSP object allocation */
58 cfs_mem_cache_t *osp_object_kmem;
59
60 static struct lu_kmem_descr osp_caches[] = {
61         {
62                 .ckd_cache = &osp_object_kmem,
63                 .ckd_name  = "osp_obj",
64                 .ckd_size  = sizeof(struct osp_object)
65         },
66         {
67                 .ckd_cache = NULL
68         }
69 };
70
71 struct lu_object *osp_object_alloc(const struct lu_env *env,
72                                    const struct lu_object_header *hdr,
73                                    struct lu_device *d)
74 {
75         struct lu_object_header *h = NULL;
76         struct osp_object       *o;
77         struct lu_object        *l;
78
79         OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
80         if (o != NULL) {
81                 l = &o->opo_obj.do_lu;
82
83                 /* For data object, OSP obj would always be the top
84                  * object, i.e. hdr is always NULL, see lu_object_alloc.
85                  * But for metadata object, we always build the object
86                  * stack from MDT. i.e. mdt_object will be the top object
87                  * i.e.  hdr != NULL */
88                 if (hdr == NULL) {
89                         /* object for OST */
90                         h = &o->opo_header;
91                         lu_object_header_init(h);
92                         dt_object_init(&o->opo_obj, h, d);
93                         lu_object_add_top(h, l);
94                 } else {
95                         dt_object_init(&o->opo_obj, h, d);
96                 }
97
98                 l->lo_ops = &osp_lu_obj_ops;
99
100                 return l;
101         } else {
102                 return NULL;
103         }
104 }
105
106 static struct dt_object
107 *osp_find_or_create(const struct lu_env *env, struct osp_device *osp,
108                     struct lu_attr *attr, __u32 reg_id)
109 {
110         struct osp_thread_info *osi = osp_env_info(env);
111         struct dt_object_format dof = { 0 };
112         struct dt_object       *dto;
113         int                  rc;
114         ENTRY;
115
116         lu_local_obj_fid(&osi->osi_fid, reg_id);
117         attr->la_valid = LA_MODE;
118         attr->la_mode = S_IFREG | 0644;
119         dof.dof_type = DFT_REGULAR;
120         dto = dt_find_or_create(env, osp->opd_storage, &osi->osi_fid,
121                                 &dof, attr);
122         if (IS_ERR(dto))
123                 RETURN(dto);
124
125         rc = dt_attr_get(env, dto, attr, NULL);
126         if (rc) {
127                 CERROR("%s: can't be initialized: rc = %d\n",
128                        osp->opd_obd->obd_name, rc);
129                 lu_object_put(env, &dto->do_lu);
130                 RETURN(ERR_PTR(rc));
131         }
132         RETURN(dto);
133 }
134
135 static int osp_write_local_file(const struct lu_env *env,
136                                 struct osp_device *osp,
137                                 struct dt_object *dt_obj,
138                                 struct lu_buf *buf,
139                                 loff_t offset)
140 {
141         struct thandle *th;
142         int rc;
143
144         th = dt_trans_create(env, osp->opd_storage);
145         if (IS_ERR(th))
146                 RETURN(PTR_ERR(th));
147
148         rc = dt_declare_record_write(env, dt_obj, buf->lb_len, offset, th);
149         if (rc)
150                 GOTO(out, rc);
151         rc = dt_trans_start_local(env, osp->opd_storage, th);
152         if (rc)
153                 GOTO(out, rc);
154
155         rc = dt_record_write(env, dt_obj, buf, &offset, th);
156 out:
157         dt_trans_stop(env, osp->opd_storage, th);
158         RETURN(rc);
159 }
160
161 static int osp_init_last_objid(const struct lu_env *env, struct osp_device *osp)
162 {
163         struct osp_thread_info  *osi = osp_env_info(env);
164         struct lu_fid           *fid = &osp->opd_last_used_fid;
165         struct dt_object        *dto;
166         int                     rc;
167         ENTRY;
168
169         dto = osp_find_or_create(env, osp, &osi->osi_attr, MDD_LOV_OBJ_OID);
170         if (IS_ERR(dto))
171                 RETURN(PTR_ERR(dto));
172         /* object will be released in device cleanup path */
173         if (osi->osi_attr.la_size >=
174             sizeof(osi->osi_id) * (osp->opd_index + 1)) {
175                 osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
176                                    osp->opd_index);
177                 rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
178                 if (rc != 0)
179                         GOTO(out, rc);
180         } else {
181                 fid->f_oid = 0;
182                 osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
183                                    osp->opd_index);
184                 rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
185                                           osi->osi_off);
186         }
187         osp->opd_last_used_oid_file = dto;
188         RETURN(0);
189 out:
190         /* object will be released in device cleanup path */
191         CERROR("%s: can't initialize lov_objid: rc = %d\n",
192                osp->opd_obd->obd_name, rc);
193         lu_object_put(env, &dto->do_lu);
194         osp->opd_last_used_oid_file = NULL;
195         RETURN(rc);
196 }
197
198 static int osp_init_last_seq(const struct lu_env *env, struct osp_device *osp)
199 {
200         struct osp_thread_info  *osi = osp_env_info(env);
201         struct lu_fid           *fid = &osp->opd_last_used_fid;
202         struct dt_object        *dto;
203         int                     rc;
204         ENTRY;
205
206         dto = osp_find_or_create(env, osp, &osi->osi_attr, MDD_LOV_OBJ_OSEQ);
207         if (IS_ERR(dto))
208                 RETURN(PTR_ERR(dto));
209
210         /* object will be released in device cleanup path */
211         if (osi->osi_attr.la_size >=
212             sizeof(osi->osi_id) * (osp->opd_index + 1)) {
213                 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
214                                    osp->opd_index);
215                 rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
216                 if (rc != 0)
217                         GOTO(out, rc);
218         } else {
219                 fid->f_seq = 0;
220                 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
221                                     osp->opd_index);
222                 rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
223                                           osi->osi_off);
224         }
225         osp->opd_last_used_seq_file = dto;
226         RETURN(0);
227 out:
228         /* object will be released in device cleanup path */
229         CERROR("%s: can't initialize lov_seq: rc = %d\n",
230                osp->opd_obd->obd_name, rc);
231         lu_object_put(env, &dto->do_lu);
232         osp->opd_last_used_seq_file = NULL;
233         RETURN(rc);
234 }
235
236 static int osp_last_used_init(const struct lu_env *env, struct osp_device *osp)
237 {
238         struct osp_thread_info *osi = osp_env_info(env);
239         int                  rc;
240         ENTRY;
241
242         fid_zero(&osp->opd_last_used_fid);
243         rc = osp_init_last_objid(env, osp);
244         if (rc < 0) {
245                 CERROR("%s: Can not get ids %d from old objid!\n",
246                        osp->opd_obd->obd_name, rc);
247                 RETURN(rc);
248         }
249
250         rc = osp_init_last_seq(env, osp);
251         if (rc < 0) {
252                 CERROR("%s: Can not get ids %d from old objid!\n",
253                        osp->opd_obd->obd_name, rc);
254                 GOTO(out, rc);
255         }
256
257         if (fid_oid(&osp->opd_last_used_fid) != 0 &&
258             fid_seq(&osp->opd_last_used_fid) == 0) {
259                 /* Just upgrade from the old version,
260                  * set the seq to be IDIF */
261                 osp->opd_last_used_fid.f_seq =
262                    fid_idif_seq(fid_oid(&osp->opd_last_used_fid),
263                                 osp->opd_index);
264                 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off,
265                                     &osp->opd_last_used_fid.f_seq,
266                                     osp->opd_index);
267                 rc = osp_write_local_file(env, osp, osp->opd_last_used_seq_file,
268                                           &osi->osi_lb, osi->osi_off);
269                 if (rc) {
270                         CERROR("%s : Can not write seq file: rc = %d\n",
271                                osp->opd_obd->obd_name, rc);
272                         GOTO(out, rc);
273                 }
274         }
275
276         if (!fid_is_zero(&osp->opd_last_used_fid) &&
277                  !fid_is_sane(&osp->opd_last_used_fid)) {
278                 CERROR("%s: Got invalid FID "DFID"\n", osp->opd_obd->obd_name,
279                         PFID(&osp->opd_last_used_fid));
280                 GOTO(out, rc = -EINVAL);
281         }
282
283         CDEBUG(D_INFO, "%s: Init last used fid "DFID"\n",
284                osp->opd_obd->obd_name, PFID(&osp->opd_last_used_fid));
285 out:
286         if (rc != 0) {
287                 if (osp->opd_last_used_oid_file != NULL) {
288                         lu_object_put(env, &osp->opd_last_used_oid_file->do_lu);
289                         osp->opd_last_used_oid_file = NULL;
290                 }
291                 if (osp->opd_last_used_seq_file != NULL) {
292                         lu_object_put(env, &osp->opd_last_used_seq_file->do_lu);
293                         osp->opd_last_used_seq_file = NULL;
294                 }
295         }
296
297         RETURN(rc);
298 }
299
300 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
301 {
302         /* release last_used file */
303         if (d->opd_last_used_oid_file != NULL) {
304                 lu_object_put(env, &d->opd_last_used_oid_file->do_lu);
305                 d->opd_last_used_oid_file = NULL;
306         }
307
308         if (d->opd_last_used_seq_file != NULL) {
309                 lu_object_put(env, &d->opd_last_used_seq_file->do_lu);
310                 d->opd_last_used_seq_file = NULL;
311         }
312 }
313
314 static int osp_disconnect(struct osp_device *d)
315 {
316         struct obd_import *imp;
317         int rc = 0;
318
319         imp = d->opd_obd->u.cli.cl_import;
320
321         /* Mark import deactivated now, so we don't try to reconnect if any
322          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
323          * fully deactivate the import, or that would drop all requests. */
324         LASSERT(imp != NULL);
325         spin_lock(&imp->imp_lock);
326         imp->imp_deactive = 1;
327         spin_unlock(&imp->imp_lock);
328
329         ptlrpc_deactivate_import(imp);
330
331         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
332          * delete it regardless.  (It's safe to delete an import that was
333          * never added.) */
334         (void)ptlrpc_pinger_del_import(imp);
335
336         rc = ptlrpc_disconnect_import(imp, 0);
337         if (rc == -ETIMEDOUT || rc == -ENOTCONN || rc == -ESHUTDOWN)
338                 rc = 0;
339         if (rc)
340                 CERROR("%s: can't disconnect: rc = %d\n",
341                        d->opd_obd->obd_name, rc);
342
343         ptlrpc_invalidate_import(imp);
344
345         RETURN(rc);
346 }
347
348 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
349 {
350         int                      rc = 0;
351         ENTRY;
352
353         LASSERT(env);
354
355         rc = osp_disconnect(d);
356
357         if (!d->opd_connect_mdt) {
358                 /* stop precreate thread */
359                 osp_precreate_fini(d);
360
361                 /* stop sync thread */
362                 osp_sync_fini(d);
363
364                 /* release last_used file */
365                 osp_last_used_fini(env, d);
366         }
367
368         obd_fid_fini(d->opd_obd);
369
370         RETURN(rc);
371 }
372
373 static int osp_process_config(const struct lu_env *env,
374                               struct lu_device *dev, struct lustre_cfg *lcfg)
375 {
376         struct osp_device               *d = lu2osp_dev(dev);
377         struct lprocfs_static_vars       lvars = { 0 };
378         int                              rc;
379
380         ENTRY;
381
382         switch (lcfg->lcfg_command) {
383         case LCFG_PRE_CLEANUP:
384                 rc = osp_disconnect(d);
385                 break;
386         case LCFG_CLEANUP:
387                 lu_dev_del_linkage(dev->ld_site, dev);
388                 rc = osp_shutdown(env, d);
389                 break;
390         case LCFG_PARAM:
391                 lprocfs_osp_init_vars(&lvars);
392
393                 LASSERT(d->opd_obd);
394                 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
395                                               lcfg, d->opd_obd);
396                 if (rc > 0)
397                         rc = 0;
398                 if (rc == -ENOSYS) {
399                         /* class_process_proc_param() haven't found matching
400                          * parameter and returned ENOSYS so that layer(s)
401                          * below could use that. But OSP is the bottom, so
402                          * just ignore it */
403                         CERROR("%s: unknown param %s\n",
404                                (char *)lustre_cfg_string(lcfg, 0),
405                                (char *)lustre_cfg_string(lcfg, 1));
406                         rc = 0;
407                 }
408                 break;
409         default:
410                 CERROR("%s: unknown command %u\n",
411                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
412                 rc = 0;
413                 break;
414         }
415
416         RETURN(rc);
417 }
418
419 static int osp_recovery_complete(const struct lu_env *env,
420                                  struct lu_device *dev)
421 {
422         struct osp_device       *osp = lu2osp_dev(dev);
423         int                      rc = 0;
424
425         ENTRY;
426         osp->opd_recovery_completed = 1;
427         if (!osp->opd_connect_mdt)
428                 cfs_waitq_signal(&osp->opd_pre_waitq);
429         RETURN(rc);
430 }
431
432 const struct lu_device_operations osp_lu_ops = {
433         .ldo_object_alloc       = osp_object_alloc,
434         .ldo_process_config     = osp_process_config,
435         .ldo_recovery_complete  = osp_recovery_complete,
436 };
437
438 /**
439  * provides with statfs from corresponded OST
440  *
441  */
442 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
443                       struct obd_statfs *sfs)
444 {
445         struct osp_device *d = dt2osp_dev(dev);
446
447         ENTRY;
448
449         if (unlikely(d->opd_imp_active == 0))
450                 RETURN(-ENOTCONN);
451
452         /* return recently updated data */
453         *sfs = d->opd_statfs;
454
455         /*
456          * layer above osp (usually lod) can use ffree to estimate
457          * how many objects are available for immediate creation
458          */
459
460         spin_lock(&d->opd_pre_lock);
461         LASSERTF(fid_seq(&d->opd_pre_last_created_fid) ==
462                  fid_seq(&d->opd_pre_used_fid),
463                  "last_created "DFID", next_fid "DFID"\n",
464                  PFID(&d->opd_pre_last_created_fid),
465                  PFID(&d->opd_pre_used_fid));
466         sfs->os_fprecreated = fid_oid(&d->opd_pre_last_created_fid) -
467                               fid_oid(&d->opd_pre_used_fid);
468         sfs->os_fprecreated -= d->opd_pre_reserved;
469         spin_unlock(&d->opd_pre_lock);
470
471         LASSERT(sfs->os_fprecreated <= OST_MAX_PRECREATE * 2);
472
473         CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
474                LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
475                sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
476                sfs->os_files, sfs->os_ffree);
477         RETURN(0);
478 }
479
480 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
481 {
482         ENTRY;
483
484         /*
485          * XXX: wake up sync thread, command it to start flushing asap?
486          */
487
488         RETURN(0);
489 }
490
491 const struct dt_device_operations osp_dt_ops = {
492         .dt_statfs      = osp_statfs,
493         .dt_sync        = osp_sync,
494         .dt_trans_start = osp_trans_start,
495         .dt_trans_stop  = osp_trans_stop,
496 };
497
498 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
499                               const char *nextdev)
500 {
501         struct obd_connect_data *data = NULL;
502         struct obd_device       *obd;
503         int                      rc;
504
505         ENTRY;
506
507         LASSERT(m->opd_storage_exp == NULL);
508
509         OBD_ALLOC_PTR(data);
510         if (data == NULL)
511                 RETURN(-ENOMEM);
512
513         obd = class_name2obd(nextdev);
514         if (obd == NULL) {
515                 CERROR("%s: can't locate next device: %s\n",
516                        m->opd_obd->obd_name, nextdev);
517                 GOTO(out, rc = -ENOTCONN);
518         }
519
520         rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
521                          NULL);
522         if (rc) {
523                 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
524                        m->opd_obd->obd_name, nextdev, rc);
525                 GOTO(out, rc);
526         }
527
528         m->opd_dt_dev.dd_lu_dev.ld_site =
529                 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
530         LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
531         m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
532
533 out:
534         OBD_FREE_PTR(data);
535         RETURN(rc);
536 }
537
538 static int osp_init0(const struct lu_env *env, struct osp_device *m,
539                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
540 {
541         struct obd_device       *obd;
542         struct obd_import       *imp;
543         class_uuid_t            uuid;
544         char                    *src, *tgt, *mdt, *osdname = NULL;
545         int                     rc, idx;
546
547         ENTRY;
548
549         obd = class_name2obd(lustre_cfg_string(cfg, 0));
550         if (obd == NULL) {
551                 CERROR("Cannot find obd with name %s\n",
552                        lustre_cfg_string(cfg, 0));
553                 RETURN(-ENODEV);
554         }
555         m->opd_obd = obd;
556
557         /* There is no record in the MDT configuration for the local disk
558          * device, so we have to extract this from elsewhere in the profile.
559          * The only information we get at setup is from the OSC records:
560          * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
561          * Note that 1.8 generated configs are missing the -MDTxxxx part.
562          * We need to reconstruct the name of the underlying OSD from this:
563          * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd".  We
564          * also need to determine the OST index from this - will be used
565          * to calculate the offset in shared lov_objids file later */
566
567         src = lustre_cfg_string(cfg, 0);
568         if (src == NULL)
569                 RETURN(-EINVAL);
570
571         tgt = strrchr(src, '-');
572         if (tgt == NULL) {
573                 CERROR("%s: invalid target name %s\n",
574                        m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
575                 RETURN(-EINVAL);
576         }
577
578         if (strncmp(tgt, "-osc", 4) == 0) {
579                 /* Old OSC name fsname-OSTXXXX-osc */
580                 for (tgt--; tgt > src && *tgt != '-'; tgt--)
581                         ;
582                 if (tgt == src) {
583                         CERROR("%s: invalid target name %s\n",
584                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
585                         RETURN(-EINVAL);
586                 }
587
588                 if (strncmp(tgt, "-OST", 4) != 0) {
589                         CERROR("%s: invalid target name %s\n",
590                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
591                         RETURN(-EINVAL);
592                 }
593
594                 idx = simple_strtol(tgt + 4, &mdt, 16);
595                 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
596                         CERROR("%s: invalid OST index in '%s'\n",
597                                m->opd_obd->obd_name, src);
598                         RETURN(-EINVAL);
599                 }
600                 m->opd_index = idx;
601                 m->opd_group = 0;
602                 idx = tgt - src;
603         } else {
604                 /* New OSC name fsname-OSTXXXX-osc-MDTXXXX */
605                 if (strncmp(tgt, "-MDT", 4) != 0 &&
606                          strncmp(tgt, "-OST", 4) != 0) {
607                         CERROR("%s: invalid target name %s\n",
608                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
609                         RETURN(-EINVAL);
610                 }
611
612                 idx = simple_strtol(tgt + 4, &mdt, 16);
613                 if (*mdt != '\0' || idx > INT_MAX || idx < 0) {
614                         CERROR("%s: invalid OST index in '%s'\n",
615                                m->opd_obd->obd_name, src);
616                         RETURN(-EINVAL);
617                 }
618
619                 /* Get MDT index from the name and set it to opd_group,
620                  * which will be used by OSP to connect with OST */
621                 m->opd_group = idx;
622                 if (tgt - src <= 12) {
623                         CERROR("%s: invalid mdt index retrieve from %s\n",
624                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
625                         RETURN(-EINVAL);
626                 }
627
628                 if (strncmp(tgt - 12, "-MDT", 4) == 0)
629                         m->opd_connect_mdt = 1;
630
631                 idx = simple_strtol(tgt - 8, &mdt, 16);
632                 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
633                         CERROR("%s: invalid OST index in '%s'\n",
634                                m->opd_obd->obd_name, src);
635                         RETURN(-EINVAL);
636                 }
637
638                 m->opd_index = idx;
639                 idx = tgt - src - 12;
640         }
641         /* check the fsname length, and after this everything else will fit */
642         if (idx > MTI_NAME_MAXLEN) {
643                 CERROR("%s: fsname too long in '%s'\n",
644                        m->opd_obd->obd_name, src);
645                 RETURN(-EINVAL);
646         }
647
648         OBD_ALLOC(osdname, MAX_OBD_NAME);
649         if (osdname == NULL)
650                 RETURN(-ENOMEM);
651
652         memcpy(osdname, src, idx); /* copy just the fsname part */
653         osdname[idx] = '\0';
654
655         mdt = strstr(mdt, "-MDT");
656         if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
657                 strcat(osdname, "-MDT0000");
658         else
659                 strcat(osdname, mdt);
660         strcat(osdname, "-osd");
661         CDEBUG(D_HA, "%s: connect to %s (%s)\n", obd->obd_name, osdname, src);
662
663         if (m->opd_connect_mdt) {
664                 struct client_obd *cli = &m->opd_obd->u.cli;
665
666                 OBD_ALLOC(cli->cl_rpc_lock, sizeof(*cli->cl_rpc_lock));
667                 if (!cli->cl_rpc_lock)
668                         GOTO(out_fini, rc = -ENOMEM);
669                 osp_init_rpc_lock(cli->cl_rpc_lock);
670         }
671
672         m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
673         m->opd_dt_dev.dd_ops = &osp_dt_ops;
674         obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
675
676         rc = osp_connect_to_osd(env, m, osdname);
677         if (rc)
678                 GOTO(out_fini, rc);
679
680         rc = ptlrpcd_addref();
681         if (rc)
682                 GOTO(out_disconnect, rc);
683
684         rc = client_obd_setup(obd, cfg);
685         if (rc) {
686                 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
687                 GOTO(out_ref, rc);
688         }
689
690         osp_lprocfs_init(m);
691
692         if (!m->opd_connect_mdt) {
693                 /* Initialize last id from the storage - will be
694                  * used in orphan cleanup. */
695                 rc = osp_last_used_init(env, m);
696                 if (rc)
697                         GOTO(out_proc, rc);
698
699                 rc = obd_fid_init(m->opd_obd, NULL, LUSTRE_SEQ_DATA);
700                 if (rc) {
701                         CERROR("%s: fid init error: rc = %d\n",
702                                m->opd_obd->obd_name, rc);
703                         GOTO(out_last_used, rc);
704                 }
705
706                 /* Initialize precreation thread, it handles new
707                  * connections as well. */
708                 rc = osp_init_precreate(m);
709                 if (rc)
710                         GOTO(out_last_used, rc);
711                 /*
712                  * Initialize synhronization mechanism taking
713                  * care of propogating changes to OST in near
714                  * transactional manner.
715                  */
716                 rc = osp_sync_init(env, m);
717                 if (rc)
718                         GOTO(out_precreat, rc);
719
720         }
721         /*
722          * Initiate connect to OST
723          */
724         ll_generate_random_uuid(uuid);
725         class_uuid_unparse(uuid, &m->opd_cluuid);
726
727         imp = obd->u.cli.cl_import;
728
729         rc = ptlrpc_init_import(imp);
730         if (rc)
731                 GOTO(out, rc);
732         if (osdname)
733                 OBD_FREE(osdname, MAX_OBD_NAME);
734         RETURN(0);
735
736 out:
737         if (!m->opd_connect_mdt)
738                 /* stop sync thread */
739                 osp_sync_fini(m);
740 out_precreat:
741         /* stop precreate thread */
742         if (!m->opd_connect_mdt)
743                 osp_precreate_fini(m);
744 out_last_used:
745         if (!m->opd_connect_mdt)
746                 osp_last_used_fini(env, m);
747 out_proc:
748         ptlrpc_lprocfs_unregister_obd(obd);
749         lprocfs_obd_cleanup(obd);
750         obd_cleanup_client_import(obd);
751         if (m->opd_symlink)
752                 lprocfs_remove(&m->opd_symlink);
753         client_obd_cleanup(obd);
754 out_ref:
755         ptlrpcd_decref();
756 out_disconnect:
757         if (m->opd_connect_mdt) {
758                 struct client_obd *cli = &m->opd_obd->u.cli;
759                 if (cli->cl_rpc_lock != NULL) {
760                         OBD_FREE_PTR(cli->cl_rpc_lock);
761                         cli->cl_rpc_lock = NULL;
762                 }
763         }
764         obd_disconnect(m->opd_storage_exp);
765 out_fini:
766         if (osdname)
767                 OBD_FREE(osdname, MAX_OBD_NAME);
768         RETURN(rc);
769 }
770
771 static struct lu_device *osp_device_free(const struct lu_env *env,
772                                          struct lu_device *lu)
773 {
774         struct osp_device *m = lu2osp_dev(lu);
775
776         ENTRY;
777
778         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
779                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
780                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
781         }
782         dt_device_fini(&m->opd_dt_dev);
783         OBD_FREE_PTR(m);
784         RETURN(NULL);
785 }
786
787 static struct lu_device *osp_device_alloc(const struct lu_env *env,
788                                           struct lu_device_type *t,
789                                           struct lustre_cfg *lcfg)
790 {
791         struct osp_device *m;
792         struct lu_device  *l;
793
794         OBD_ALLOC_PTR(m);
795         if (m == NULL) {
796                 l = ERR_PTR(-ENOMEM);
797         } else {
798                 int rc;
799
800                 l = osp2lu_dev(m);
801                 dt_device_init(&m->opd_dt_dev, t);
802                 rc = osp_init0(env, m, t, lcfg);
803                 if (rc != 0) {
804                         osp_device_free(env, l);
805                         l = ERR_PTR(rc);
806                 }
807         }
808         return l;
809 }
810
811 static struct lu_device *osp_device_fini(const struct lu_env *env,
812                                          struct lu_device *d)
813 {
814         struct osp_device *m = lu2osp_dev(d);
815         struct obd_import *imp;
816         int                rc;
817
818         ENTRY;
819
820         if (m->opd_storage_exp)
821                 obd_disconnect(m->opd_storage_exp);
822
823         imp = m->opd_obd->u.cli.cl_import;
824
825         if (imp->imp_rq_pool) {
826                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
827                 imp->imp_rq_pool = NULL;
828         }
829
830         obd_cleanup_client_import(m->opd_obd);
831
832         if (m->opd_symlink)
833                 lprocfs_remove(&m->opd_symlink);
834
835         LASSERT(m->opd_obd);
836         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
837         lprocfs_obd_cleanup(m->opd_obd);
838
839         if (m->opd_connect_mdt) {
840                 struct client_obd *cli = &m->opd_obd->u.cli;
841                 if (cli->cl_rpc_lock != NULL) {
842                         OBD_FREE_PTR(cli->cl_rpc_lock);
843                         cli->cl_rpc_lock = NULL;
844                 }
845         }
846
847         rc = client_obd_cleanup(m->opd_obd);
848         LASSERTF(rc == 0, "error %d\n", rc);
849
850         ptlrpcd_decref();
851
852         RETURN(NULL);
853 }
854
855 static int osp_reconnect(const struct lu_env *env,
856                          struct obd_export *exp, struct obd_device *obd,
857                          struct obd_uuid *cluuid,
858                          struct obd_connect_data *data,
859                          void *localdata)
860 {
861         return 0;
862 }
863
864 /*
865  * we use exports to track all LOD users
866  */
867 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
868                            struct obd_device *obd, struct obd_uuid *cluuid,
869                            struct obd_connect_data *data, void *localdata)
870 {
871         struct osp_device       *osp = lu2osp_dev(obd->obd_lu_dev);
872         struct obd_connect_data *ocd;
873         struct obd_import       *imp;
874         struct lustre_handle     conn;
875         int                      rc;
876
877         ENTRY;
878
879         CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
880
881         rc = class_connect(&conn, obd, cluuid);
882         if (rc)
883                 RETURN(rc);
884
885         *exp = class_conn2export(&conn);
886         /* Why should there ever be more than 1 connect? */
887         osp->opd_connects++;
888         LASSERT(osp->opd_connects == 1);
889
890         osp->opd_exp = *exp;
891
892         imp = osp->opd_obd->u.cli.cl_import;
893         imp->imp_dlm_handle = conn;
894
895         LASSERT(data != NULL);
896         LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
897         ocd = &imp->imp_connect_data;
898         *ocd = *data;
899
900         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
901
902         ocd->ocd_version = LUSTRE_VERSION_CODE;
903         ocd->ocd_index = data->ocd_index;
904         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
905
906         rc = ptlrpc_connect_import(imp);
907         if (rc) {
908                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
909                 GOTO(out, rc);
910         }
911
912         ptlrpc_pinger_add_import(imp);
913
914         if (osp->opd_connect_mdt && data->ocd_index == 0) {
915                 /* set seq controller export for MDC0 if exists */
916                 struct seq_server_site *ss;
917
918                 ss = lu_site2seq(osp2lu_dev(osp)->ld_site);
919                 ss->ss_control_exp = class_export_get(*exp);
920                 ss->ss_server_fld->lsf_control_exp = *exp;
921         }
922 out:
923         RETURN(rc);
924 }
925
926 /*
927  * once last export (we don't count self-export) disappeared
928  * osp can be released
929  */
930 static int osp_obd_disconnect(struct obd_export *exp)
931 {
932         struct obd_device *obd = exp->exp_obd;
933         struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
934         int                rc;
935         ENTRY;
936
937         /* Only disconnect the underlying layers on the final disconnect. */
938         LASSERT(osp->opd_connects == 1);
939         osp->opd_connects--;
940
941         rc = class_disconnect(exp);
942         if (rc) {
943                 CERROR("%s: class disconnect error: rc = %d\n",
944                        obd->obd_name, rc);
945                 RETURN(rc);
946         }
947
948         /* destroy the device */
949         class_manual_cleanup(obd);
950
951         RETURN(rc);
952 }
953
954 /*
955  * lprocfs helpers still use OBD API, let's keep obd_statfs() support
956  */
957 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
958                           struct obd_statfs *osfs, __u64 max_age, __u32 flags)
959 {
960         struct obd_statfs       *msfs;
961         struct ptlrpc_request   *req;
962         struct obd_import       *imp = NULL;
963         int                      rc;
964
965         ENTRY;
966
967         /* Since the request might also come from lprocfs, so we need
968          * sync this with client_disconnect_export Bug15684 */
969         down_read(&exp->exp_obd->u.cli.cl_sem);
970         if (exp->exp_obd->u.cli.cl_import)
971                 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
972         up_read(&exp->exp_obd->u.cli.cl_sem);
973         if (!imp)
974                 RETURN(-ENODEV);
975
976         /* We could possibly pass max_age in the request (as an absolute
977          * timestamp or a "seconds.usec ago") so the target can avoid doing
978          * extra calls into the filesystem if that isn't necessary (e.g.
979          * during mount that would help a bit).  Having relative timestamps
980          * is not so great if request processing is slow, while absolute
981          * timestamps are not ideal because they need time synchronization. */
982         req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
983
984         class_import_put(imp);
985
986         if (req == NULL)
987                 RETURN(-ENOMEM);
988
989         rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
990         if (rc) {
991                 ptlrpc_request_free(req);
992                 RETURN(rc);
993         }
994         ptlrpc_request_set_replen(req);
995         req->rq_request_portal = OST_CREATE_PORTAL;
996         ptlrpc_at_set_req_timeout(req);
997
998         if (flags & OBD_STATFS_NODELAY) {
999                 /* procfs requests not want stat in wait for avoid deadlock */
1000                 req->rq_no_resend = 1;
1001                 req->rq_no_delay = 1;
1002         }
1003
1004         rc = ptlrpc_queue_wait(req);
1005         if (rc)
1006                 GOTO(out, rc);
1007
1008         msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
1009         if (msfs == NULL)
1010                 GOTO(out, rc = -EPROTO);
1011
1012         *osfs = *msfs;
1013
1014         EXIT;
1015 out:
1016         ptlrpc_req_finished(req);
1017         return rc;
1018 }
1019
1020 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
1021                             enum obd_import_event event)
1022 {
1023         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
1024
1025         switch (event) {
1026         case IMP_EVENT_DISCON:
1027                 d->opd_got_disconnected = 1;
1028                 d->opd_imp_connected = 0;
1029                 if (d->opd_connect_mdt)
1030                         break;
1031                 osp_pre_update_status(d, -ENODEV);
1032                 cfs_waitq_signal(&d->opd_pre_waitq);
1033                 CDEBUG(D_HA, "got disconnected\n");
1034                 break;
1035         case IMP_EVENT_INACTIVE:
1036                 d->opd_imp_active = 0;
1037                 if (d->opd_connect_mdt)
1038                         break;
1039                 osp_pre_update_status(d, -ENODEV);
1040                 cfs_waitq_signal(&d->opd_pre_waitq);
1041                 CDEBUG(D_HA, "got inactive\n");
1042                 break;
1043         case IMP_EVENT_ACTIVE:
1044                 d->opd_imp_active = 1;
1045                 if (d->opd_got_disconnected)
1046                         d->opd_new_connection = 1;
1047                 d->opd_imp_connected = 1;
1048                 d->opd_imp_seen_connected = 1;
1049                 if (d->opd_connect_mdt)
1050                         break;
1051                 cfs_waitq_signal(&d->opd_pre_waitq);
1052                 __osp_sync_check_for_work(d);
1053                 CDEBUG(D_HA, "got connected\n");
1054                 break;
1055         case IMP_EVENT_INVALIDATE:
1056                 if (obd->obd_namespace == NULL)
1057                         break;
1058                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
1059                 break;
1060         case IMP_EVENT_OCD:
1061         case IMP_EVENT_DEACTIVATE:
1062         case IMP_EVENT_ACTIVATE:
1063                 break;
1064         default:
1065                 CERROR("%s: unsupported import event: %#x\n",
1066                        obd->obd_name, event);
1067         }
1068         return 0;
1069 }
1070
1071 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
1072                          void *karg, void *uarg)
1073 {
1074         struct obd_device       *obd = exp->exp_obd;
1075         struct osp_device       *d;
1076         struct obd_ioctl_data   *data = karg;
1077         int                      rc = 0;
1078
1079         ENTRY;
1080
1081         LASSERT(obd->obd_lu_dev);
1082         d = lu2osp_dev(obd->obd_lu_dev);
1083         LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
1084
1085         if (!cfs_try_module_get(THIS_MODULE)) {
1086                 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
1087                 return -EINVAL;
1088         }
1089
1090         switch (cmd) {
1091         case OBD_IOC_CLIENT_RECOVER:
1092                 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
1093                                            data->ioc_inlbuf1, 0);
1094                 if (rc > 0)
1095                         rc = 0;
1096                 break;
1097         case IOC_OSC_SET_ACTIVE:
1098                 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
1099                                               data->ioc_offset);
1100                 break;
1101         case OBD_IOC_PING_TARGET:
1102                 rc = ptlrpc_obd_ping(obd);
1103                 break;
1104         default:
1105                 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
1106                        cmd, cfs_curproc_comm());
1107                 rc = -ENOTTY;
1108         }
1109         cfs_module_put(THIS_MODULE);
1110         return rc;
1111 }
1112
1113 static int osp_obd_health_check(const struct lu_env *env,
1114                                 struct obd_device *obd)
1115 {
1116         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
1117
1118         ENTRY;
1119
1120         /*
1121          * 1.8/2.0 behaviour is that OST being connected once at least
1122          * is considired "healthy". and one "healty" OST is enough to
1123          * allow lustre clients to connect to MDS
1124          */
1125         LASSERT(d);
1126         RETURN(!d->opd_imp_seen_connected);
1127 }
1128
1129 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
1130 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
1131 static void osp_key_exit(const struct lu_context *ctx,
1132                          struct lu_context_key *key, void *data)
1133 {
1134         struct osp_thread_info *info = data;
1135
1136         info->osi_attr.la_valid = 0;
1137 }
1138
1139 struct lu_context_key osp_thread_key = {
1140         .lct_tags = LCT_MD_THREAD,
1141         .lct_init = osp_key_init,
1142         .lct_fini = osp_key_fini,
1143         .lct_exit = osp_key_exit
1144 };
1145
1146 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
1147 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
1148
1149 struct lu_context_key osp_txn_key = {
1150         .lct_tags = LCT_OSP_THREAD,
1151         .lct_init = osp_txn_key_init,
1152         .lct_fini = osp_txn_key_fini
1153 };
1154 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
1155
1156 static struct lu_device_type_operations osp_device_type_ops = {
1157         .ldto_init           = osp_type_init,
1158         .ldto_fini           = osp_type_fini,
1159
1160         .ldto_start          = osp_type_start,
1161         .ldto_stop           = osp_type_stop,
1162
1163         .ldto_device_alloc   = osp_device_alloc,
1164         .ldto_device_free    = osp_device_free,
1165
1166         .ldto_device_fini    = osp_device_fini
1167 };
1168
1169 static struct lu_device_type osp_device_type = {
1170         .ldt_tags     = LU_DEVICE_DT,
1171         .ldt_name     = LUSTRE_OSP_NAME,
1172         .ldt_ops      = &osp_device_type_ops,
1173         .ldt_ctx_tags = LCT_MD_THREAD
1174 };
1175
1176 static struct obd_ops osp_obd_device_ops = {
1177         .o_owner        = THIS_MODULE,
1178         .o_add_conn     = client_import_add_conn,
1179         .o_del_conn     = client_import_del_conn,
1180         .o_reconnect    = osp_reconnect,
1181         .o_connect      = osp_obd_connect,
1182         .o_disconnect   = osp_obd_disconnect,
1183         .o_health_check = osp_obd_health_check,
1184         .o_import_event = osp_import_event,
1185         .o_iocontrol    = osp_iocontrol,
1186         .o_statfs       = osp_obd_statfs,
1187         .o_fid_init     = client_fid_init,
1188         .o_fid_fini     = client_fid_fini,
1189 };
1190
1191 struct llog_operations osp_mds_ost_orig_logops;
1192
1193 static int __init osp_mod_init(void)
1194 {
1195         struct lprocfs_static_vars       lvars;
1196         cfs_proc_dir_entry_t            *osc_proc_dir;
1197         int                              rc;
1198
1199         rc = lu_kmem_init(osp_caches);
1200         if (rc)
1201                 return rc;
1202
1203         lprocfs_osp_init_vars(&lvars);
1204
1205         rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
1206                                  LUSTRE_OSP_NAME, &osp_device_type);
1207
1208         /* create "osc" entry in procfs for compatibility purposes */
1209         if (rc != 0) {
1210                 lu_kmem_fini(osp_caches);
1211                 return rc;
1212         }
1213
1214         lprocfs_lwp_init_vars(&lvars);
1215
1216         rc = class_register_type(&lwp_obd_device_ops, NULL, lvars.module_vars,
1217                                  LUSTRE_LWP_NAME, &lwp_device_type);
1218         if (rc != 0) {
1219                 class_unregister_type(LUSTRE_OSP_NAME);
1220                 lu_kmem_fini(osp_caches);
1221                 return rc;
1222         }
1223
1224         /* Note: add_rec/delcare_add_rec will be only used by catalogs */
1225         osp_mds_ost_orig_logops = llog_osd_ops;
1226         osp_mds_ost_orig_logops.lop_add = llog_cat_add_rec;
1227         osp_mds_ost_orig_logops.lop_declare_add = llog_cat_declare_add_rec;
1228
1229         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1230         if (osc_proc_dir == NULL) {
1231                 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1232                                                 NULL);
1233                 if (IS_ERR(osc_proc_dir))
1234                         CERROR("osp: can't create compat entry \"osc\": %d\n",
1235                                (int) PTR_ERR(osc_proc_dir));
1236         }
1237         return rc;
1238 }
1239
1240 static void __exit osp_mod_exit(void)
1241 {
1242         lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1243
1244         class_unregister_type(LUSTRE_LWP_NAME);
1245         class_unregister_type(LUSTRE_OSP_NAME);
1246         lu_kmem_fini(osp_caches);
1247 }
1248
1249 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1250 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1251 MODULE_LICENSE("GPL");
1252
1253 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);