Whamcloud - gitweb
LU-1267 lfsck: enhance API for MDT-OST consistency
[fs/lustre-release.git] / lustre / osp / osp_dev.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2012, 2013, Intel Corporation.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lustre/osp/osp_dev.c
37  *
38  * Lustre OST Proxy Device
39  *
40  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
41  * Author: Mikhail Pershin <mike.pershin@intel.com>
42  * Author: Di Wang <di.wang@intel.com>
43  */
44
45 #define DEBUG_SUBSYSTEM S_MDS
46
47 #include <obd_class.h>
48 #include <lustre_param.h>
49 #include <lustre_log.h>
50 #include <lustre_mdc.h>
51
52 #include "osp_internal.h"
53
54 /* Slab for OSP object allocation */
55 struct kmem_cache *osp_object_kmem;
56
57 static struct lu_kmem_descr osp_caches[] = {
58         {
59                 .ckd_cache = &osp_object_kmem,
60                 .ckd_name  = "osp_obj",
61                 .ckd_size  = sizeof(struct osp_object)
62         },
63         {
64                 .ckd_cache = NULL
65         }
66 };
67
68 struct lu_object *osp_object_alloc(const struct lu_env *env,
69                                    const struct lu_object_header *hdr,
70                                    struct lu_device *d)
71 {
72         struct lu_object_header *h = NULL;
73         struct osp_object       *o;
74         struct lu_object        *l;
75
76         OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, __GFP_IO);
77         if (o != NULL) {
78                 l = &o->opo_obj.do_lu;
79
80                 /* For data object, OSP obj would always be the top
81                  * object, i.e. hdr is always NULL, see lu_object_alloc.
82                  * But for metadata object, we always build the object
83                  * stack from MDT. i.e. mdt_object will be the top object
84                  * i.e.  hdr != NULL */
85                 if (hdr == NULL) {
86                         /* object for OST */
87                         h = &o->opo_header;
88                         lu_object_header_init(h);
89                         dt_object_init(&o->opo_obj, h, d);
90                         lu_object_add_top(h, l);
91                 } else {
92                         dt_object_init(&o->opo_obj, h, d);
93                 }
94
95                 l->lo_ops = &osp_lu_obj_ops;
96
97                 return l;
98         } else {
99                 return NULL;
100         }
101 }
102
103 static struct dt_object
104 *osp_find_or_create(const struct lu_env *env, struct osp_device *osp,
105                     struct lu_attr *attr, __u32 reg_id)
106 {
107         struct osp_thread_info *osi = osp_env_info(env);
108         struct dt_object_format dof = { 0 };
109         struct dt_object       *dto;
110         int                  rc;
111         ENTRY;
112
113         lu_local_obj_fid(&osi->osi_fid, reg_id);
114         attr->la_valid = LA_MODE;
115         attr->la_mode = S_IFREG | 0644;
116         dof.dof_type = DFT_REGULAR;
117         dto = dt_find_or_create(env, osp->opd_storage, &osi->osi_fid,
118                                 &dof, attr);
119         if (IS_ERR(dto))
120                 RETURN(dto);
121
122         rc = dt_attr_get(env, dto, attr, NULL);
123         if (rc) {
124                 CERROR("%s: can't be initialized: rc = %d\n",
125                        osp->opd_obd->obd_name, rc);
126                 lu_object_put(env, &dto->do_lu);
127                 RETURN(ERR_PTR(rc));
128         }
129         RETURN(dto);
130 }
131
132 static int osp_write_local_file(const struct lu_env *env,
133                                 struct osp_device *osp,
134                                 struct dt_object *dt_obj,
135                                 struct lu_buf *buf,
136                                 loff_t offset)
137 {
138         struct thandle *th;
139         int rc;
140
141         th = dt_trans_create(env, osp->opd_storage);
142         if (IS_ERR(th))
143                 RETURN(PTR_ERR(th));
144
145         rc = dt_declare_record_write(env, dt_obj, buf->lb_len, offset, th);
146         if (rc)
147                 GOTO(out, rc);
148         rc = dt_trans_start_local(env, osp->opd_storage, th);
149         if (rc)
150                 GOTO(out, rc);
151
152         rc = dt_record_write(env, dt_obj, buf, &offset, th);
153 out:
154         dt_trans_stop(env, osp->opd_storage, th);
155         RETURN(rc);
156 }
157
158 static int osp_init_last_objid(const struct lu_env *env, struct osp_device *osp)
159 {
160         struct osp_thread_info  *osi = osp_env_info(env);
161         struct lu_fid           *fid = &osp->opd_last_used_fid;
162         struct dt_object        *dto;
163         int                     rc;
164         ENTRY;
165
166         dto = osp_find_or_create(env, osp, &osi->osi_attr, MDD_LOV_OBJ_OID);
167         if (IS_ERR(dto))
168                 RETURN(PTR_ERR(dto));
169         /* object will be released in device cleanup path */
170         if (osi->osi_attr.la_size >=
171             sizeof(osi->osi_id) * (osp->opd_index + 1)) {
172                 osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
173                                    osp->opd_index);
174                 rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
175                 if (rc != 0)
176                         GOTO(out, rc);
177         } else {
178                 fid->f_oid = 0;
179                 osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
180                                    osp->opd_index);
181                 rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
182                                           osi->osi_off);
183         }
184         osp->opd_last_used_oid_file = dto;
185         RETURN(0);
186 out:
187         /* object will be released in device cleanup path */
188         CERROR("%s: can't initialize lov_objid: rc = %d\n",
189                osp->opd_obd->obd_name, rc);
190         lu_object_put(env, &dto->do_lu);
191         osp->opd_last_used_oid_file = NULL;
192         RETURN(rc);
193 }
194
195 static int osp_init_last_seq(const struct lu_env *env, struct osp_device *osp)
196 {
197         struct osp_thread_info  *osi = osp_env_info(env);
198         struct lu_fid           *fid = &osp->opd_last_used_fid;
199         struct dt_object        *dto;
200         int                     rc;
201         ENTRY;
202
203         dto = osp_find_or_create(env, osp, &osi->osi_attr, MDD_LOV_OBJ_OSEQ);
204         if (IS_ERR(dto))
205                 RETURN(PTR_ERR(dto));
206
207         /* object will be released in device cleanup path */
208         if (osi->osi_attr.la_size >=
209             sizeof(osi->osi_id) * (osp->opd_index + 1)) {
210                 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
211                                    osp->opd_index);
212                 rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
213                 if (rc != 0)
214                         GOTO(out, rc);
215         } else {
216                 fid->f_seq = 0;
217                 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
218                                     osp->opd_index);
219                 rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
220                                           osi->osi_off);
221         }
222         osp->opd_last_used_seq_file = dto;
223         RETURN(0);
224 out:
225         /* object will be released in device cleanup path */
226         CERROR("%s: can't initialize lov_seq: rc = %d\n",
227                osp->opd_obd->obd_name, rc);
228         lu_object_put(env, &dto->do_lu);
229         osp->opd_last_used_seq_file = NULL;
230         RETURN(rc);
231 }
232
233 static int osp_last_used_init(const struct lu_env *env, struct osp_device *osp)
234 {
235         struct osp_thread_info *osi = osp_env_info(env);
236         int                  rc;
237         ENTRY;
238
239         fid_zero(&osp->opd_last_used_fid);
240         rc = osp_init_last_objid(env, osp);
241         if (rc < 0) {
242                 CERROR("%s: Can not get ids %d from old objid!\n",
243                        osp->opd_obd->obd_name, rc);
244                 RETURN(rc);
245         }
246
247         rc = osp_init_last_seq(env, osp);
248         if (rc < 0) {
249                 CERROR("%s: Can not get ids %d from old objid!\n",
250                        osp->opd_obd->obd_name, rc);
251                 GOTO(out, rc);
252         }
253
254         if (fid_oid(&osp->opd_last_used_fid) != 0 &&
255             fid_seq(&osp->opd_last_used_fid) == 0) {
256                 /* Just upgrade from the old version,
257                  * set the seq to be IDIF */
258                 osp->opd_last_used_fid.f_seq =
259                    fid_idif_seq(fid_oid(&osp->opd_last_used_fid),
260                                 osp->opd_index);
261                 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off,
262                                     &osp->opd_last_used_fid.f_seq,
263                                     osp->opd_index);
264                 rc = osp_write_local_file(env, osp, osp->opd_last_used_seq_file,
265                                           &osi->osi_lb, osi->osi_off);
266                 if (rc) {
267                         CERROR("%s : Can not write seq file: rc = %d\n",
268                                osp->opd_obd->obd_name, rc);
269                         GOTO(out, rc);
270                 }
271         }
272
273         if (!fid_is_zero(&osp->opd_last_used_fid) &&
274                  !fid_is_sane(&osp->opd_last_used_fid)) {
275                 CERROR("%s: Got invalid FID "DFID"\n", osp->opd_obd->obd_name,
276                         PFID(&osp->opd_last_used_fid));
277                 GOTO(out, rc = -EINVAL);
278         }
279
280         CDEBUG(D_INFO, "%s: Init last used fid "DFID"\n",
281                osp->opd_obd->obd_name, PFID(&osp->opd_last_used_fid));
282 out:
283         if (rc != 0) {
284                 if (osp->opd_last_used_oid_file != NULL) {
285                         lu_object_put(env, &osp->opd_last_used_oid_file->do_lu);
286                         osp->opd_last_used_oid_file = NULL;
287                 }
288                 if (osp->opd_last_used_seq_file != NULL) {
289                         lu_object_put(env, &osp->opd_last_used_seq_file->do_lu);
290                         osp->opd_last_used_seq_file = NULL;
291                 }
292         }
293
294         RETURN(rc);
295 }
296
297 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
298 {
299         /* release last_used file */
300         if (d->opd_last_used_oid_file != NULL) {
301                 lu_object_put(env, &d->opd_last_used_oid_file->do_lu);
302                 d->opd_last_used_oid_file = NULL;
303         }
304
305         if (d->opd_last_used_seq_file != NULL) {
306                 lu_object_put(env, &d->opd_last_used_seq_file->do_lu);
307                 d->opd_last_used_seq_file = NULL;
308         }
309 }
310
311 static int osp_disconnect(struct osp_device *d)
312 {
313         struct obd_import *imp;
314         int rc = 0;
315
316         imp = d->opd_obd->u.cli.cl_import;
317
318         /* Mark import deactivated now, so we don't try to reconnect if any
319          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
320          * fully deactivate the import, or that would drop all requests. */
321         LASSERT(imp != NULL);
322         spin_lock(&imp->imp_lock);
323         imp->imp_deactive = 1;
324         spin_unlock(&imp->imp_lock);
325
326         ptlrpc_deactivate_import(imp);
327
328         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
329          * delete it regardless.  (It's safe to delete an import that was
330          * never added.) */
331         (void)ptlrpc_pinger_del_import(imp);
332
333         rc = ptlrpc_disconnect_import(imp, 0);
334         if (rc == -ETIMEDOUT || rc == -ENOTCONN || rc == -ESHUTDOWN)
335                 rc = 0;
336         if (rc)
337                 CERROR("%s: can't disconnect: rc = %d\n",
338                        d->opd_obd->obd_name, rc);
339
340         ptlrpc_invalidate_import(imp);
341
342         RETURN(rc);
343 }
344
345 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
346 {
347         int                      rc = 0;
348         ENTRY;
349
350         LASSERT(env);
351
352         rc = osp_disconnect(d);
353
354         if (!d->opd_connect_mdt) {
355                 /* stop sync thread */
356                 osp_sync_fini(d);
357
358                 /* stop precreate thread */
359                 osp_precreate_fini(d);
360
361                 /* release last_used file */
362                 osp_last_used_fini(env, d);
363         }
364
365         obd_fid_fini(d->opd_obd);
366
367         RETURN(rc);
368 }
369
370 static int osp_process_config(const struct lu_env *env,
371                               struct lu_device *dev, struct lustre_cfg *lcfg)
372 {
373         struct osp_device               *d = lu2osp_dev(dev);
374         struct lprocfs_static_vars       lvars = { 0 };
375         int                              rc;
376
377         ENTRY;
378
379         switch (lcfg->lcfg_command) {
380         case LCFG_PRE_CLEANUP:
381                 rc = osp_disconnect(d);
382                 break;
383         case LCFG_CLEANUP:
384                 lu_dev_del_linkage(dev->ld_site, dev);
385                 rc = osp_shutdown(env, d);
386                 break;
387         case LCFG_PARAM:
388                 lprocfs_osp_init_vars(&lvars);
389
390                 LASSERT(d->opd_obd);
391                 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
392                                               lcfg, d->opd_obd);
393                 if (rc > 0)
394                         rc = 0;
395                 if (rc == -ENOSYS) {
396                         /* class_process_proc_param() haven't found matching
397                          * parameter and returned ENOSYS so that layer(s)
398                          * below could use that. But OSP is the bottom, so
399                          * just ignore it */
400                         CERROR("%s: unknown param %s\n",
401                                (char *)lustre_cfg_string(lcfg, 0),
402                                (char *)lustre_cfg_string(lcfg, 1));
403                         rc = 0;
404                 }
405                 break;
406         default:
407                 CERROR("%s: unknown command %u\n",
408                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
409                 rc = 0;
410                 break;
411         }
412
413         RETURN(rc);
414 }
415
416 static int osp_recovery_complete(const struct lu_env *env,
417                                  struct lu_device *dev)
418 {
419         struct osp_device       *osp = lu2osp_dev(dev);
420         int                      rc = 0;
421
422         ENTRY;
423         osp->opd_recovery_completed = 1;
424
425         if (!osp->opd_connect_mdt && osp->opd_pre != NULL)
426                 wake_up(&osp->opd_pre_waitq);
427         RETURN(rc);
428 }
429
430 const struct lu_device_operations osp_lu_ops = {
431         .ldo_object_alloc       = osp_object_alloc,
432         .ldo_process_config     = osp_process_config,
433         .ldo_recovery_complete  = osp_recovery_complete,
434 };
435
436 /**
437  * provides with statfs from corresponded OST
438  *
439  */
440 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
441                       struct obd_statfs *sfs)
442 {
443         struct osp_device *d = dt2osp_dev(dev);
444
445         ENTRY;
446
447         if (unlikely(d->opd_imp_active == 0))
448                 RETURN(-ENOTCONN);
449
450         if (d->opd_pre == NULL)
451                 RETURN(0);
452
453         /* return recently updated data */
454         *sfs = d->opd_statfs;
455
456         /*
457          * layer above osp (usually lod) can use ffree to estimate
458          * how many objects are available for immediate creation
459          */
460         spin_lock(&d->opd_pre_lock);
461         LASSERTF(fid_seq(&d->opd_pre_last_created_fid) ==
462                  fid_seq(&d->opd_pre_used_fid),
463                  "last_created "DFID", next_fid "DFID"\n",
464                  PFID(&d->opd_pre_last_created_fid),
465                  PFID(&d->opd_pre_used_fid));
466         sfs->os_fprecreated = fid_oid(&d->opd_pre_last_created_fid) -
467                               fid_oid(&d->opd_pre_used_fid);
468         sfs->os_fprecreated -= d->opd_pre_reserved;
469         spin_unlock(&d->opd_pre_lock);
470
471         LASSERT(sfs->os_fprecreated <= OST_MAX_PRECREATE * 2);
472
473         CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
474                LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
475                sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
476                sfs->os_files, sfs->os_ffree);
477         RETURN(0);
478 }
479
480 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
481 {
482         ENTRY;
483
484         /*
485          * XXX: wake up sync thread, command it to start flushing asap?
486          */
487
488         RETURN(0);
489 }
490
491 const struct dt_device_operations osp_dt_ops = {
492         .dt_statfs       = osp_statfs,
493         .dt_sync         = osp_sync,
494         .dt_trans_create = osp_trans_create,
495         .dt_trans_start  = osp_trans_start,
496         .dt_trans_stop   = osp_trans_stop,
497 };
498
499 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
500                               const char *nextdev)
501 {
502         struct obd_connect_data *data = NULL;
503         struct obd_device       *obd;
504         int                      rc;
505
506         ENTRY;
507
508         LASSERT(m->opd_storage_exp == NULL);
509
510         OBD_ALLOC_PTR(data);
511         if (data == NULL)
512                 RETURN(-ENOMEM);
513
514         obd = class_name2obd(nextdev);
515         if (obd == NULL) {
516                 CERROR("%s: can't locate next device: %s\n",
517                        m->opd_obd->obd_name, nextdev);
518                 GOTO(out, rc = -ENOTCONN);
519         }
520
521         rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
522                          NULL);
523         if (rc) {
524                 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
525                        m->opd_obd->obd_name, nextdev, rc);
526                 GOTO(out, rc);
527         }
528
529         m->opd_dt_dev.dd_lu_dev.ld_site =
530                 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
531         LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
532         m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
533
534 out:
535         OBD_FREE_PTR(data);
536         RETURN(rc);
537 }
538
539 static int osp_init0(const struct lu_env *env, struct osp_device *m,
540                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
541 {
542         struct obd_device       *obd;
543         struct obd_import       *imp;
544         class_uuid_t            uuid;
545         char                    *src, *tgt, *mdt, *osdname = NULL;
546         int                     rc;
547         long                    idx;
548
549         ENTRY;
550
551         mutex_init(&m->opd_async_requests_mutex);
552         obd = class_name2obd(lustre_cfg_string(cfg, 0));
553         if (obd == NULL) {
554                 CERROR("Cannot find obd with name %s\n",
555                        lustre_cfg_string(cfg, 0));
556                 RETURN(-ENODEV);
557         }
558         m->opd_obd = obd;
559
560         /* There is no record in the MDT configuration for the local disk
561          * device, so we have to extract this from elsewhere in the profile.
562          * The only information we get at setup is from the OSC records:
563          * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
564          * Note that 1.8 generated configs are missing the -MDTxxxx part.
565          * We need to reconstruct the name of the underlying OSD from this:
566          * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd".  We
567          * also need to determine the OST index from this - will be used
568          * to calculate the offset in shared lov_objids file later */
569
570         src = lustre_cfg_string(cfg, 0);
571         if (src == NULL)
572                 RETURN(-EINVAL);
573
574         tgt = strrchr(src, '-');
575         if (tgt == NULL) {
576                 CERROR("%s: invalid target name %s\n",
577                        m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
578                 RETURN(-EINVAL);
579         }
580
581         if (strncmp(tgt, "-osc", 4) == 0) {
582                 /* Old OSC name fsname-OSTXXXX-osc */
583                 for (tgt--; tgt > src && *tgt != '-'; tgt--)
584                         ;
585                 if (tgt == src) {
586                         CERROR("%s: invalid target name %s\n",
587                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
588                         RETURN(-EINVAL);
589                 }
590
591                 if (strncmp(tgt, "-OST", 4) != 0) {
592                         CERROR("%s: invalid target name %s\n",
593                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
594                         RETURN(-EINVAL);
595                 }
596
597                 idx = simple_strtol(tgt + 4, &mdt, 16);
598                 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
599                         CERROR("%s: invalid OST index in '%s'\n",
600                                m->opd_obd->obd_name, src);
601                         RETURN(-EINVAL);
602                 }
603                 m->opd_index = idx;
604                 m->opd_group = 0;
605                 idx = tgt - src;
606         } else {
607                 /* New OSC name fsname-OSTXXXX-osc-MDTXXXX */
608                 if (strncmp(tgt, "-MDT", 4) != 0 &&
609                     strncmp(tgt, "-OST", 4) != 0) {
610                         CERROR("%s: invalid target name %s\n",
611                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
612                         RETURN(-EINVAL);
613                 }
614
615                 idx = simple_strtol(tgt + 4, &mdt, 16);
616                 if (*mdt != '\0' || idx > INT_MAX || idx < 0) {
617                         CERROR("%s: invalid OST index in '%s'\n",
618                                m->opd_obd->obd_name, src);
619                         RETURN(-EINVAL);
620                 }
621
622                 /* Get MDT index from the name and set it to opd_group,
623                  * which will be used by OSP to connect with OST */
624                 m->opd_group = idx;
625                 if (tgt - src <= 12) {
626                         CERROR("%s: invalid mdt index retrieve from %s\n",
627                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
628                         RETURN(-EINVAL);
629                 }
630
631                 if (strncmp(tgt - 12, "-MDT", 4) == 0)
632                         m->opd_connect_mdt = 1;
633
634                 idx = simple_strtol(tgt - 8, &mdt, 16);
635                 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
636                         CERROR("%s: invalid OST index in '%s'\n",
637                                m->opd_obd->obd_name, src);
638                         RETURN(-EINVAL);
639                 }
640
641                 m->opd_index = idx;
642                 idx = tgt - src - 12;
643         }
644         /* check the fsname length, and after this everything else will fit */
645         if (idx > MTI_NAME_MAXLEN) {
646                 CERROR("%s: fsname too long in '%s'\n",
647                        m->opd_obd->obd_name, src);
648                 RETURN(-EINVAL);
649         }
650
651         OBD_ALLOC(osdname, MAX_OBD_NAME);
652         if (osdname == NULL)
653                 RETURN(-ENOMEM);
654
655         memcpy(osdname, src, idx); /* copy just the fsname part */
656         osdname[idx] = '\0';
657
658         mdt = strstr(mdt, "-MDT");
659         if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
660                 strcat(osdname, "-MDT0000");
661         else
662                 strcat(osdname, mdt);
663         strcat(osdname, "-osd");
664         CDEBUG(D_HA, "%s: connect to %s (%s)\n", obd->obd_name, osdname, src);
665
666         if (m->opd_connect_mdt) {
667                 struct client_obd *cli = &m->opd_obd->u.cli;
668
669                 OBD_ALLOC(cli->cl_rpc_lock, sizeof(*cli->cl_rpc_lock));
670                 if (!cli->cl_rpc_lock)
671                         GOTO(out_fini, rc = -ENOMEM);
672                 osp_init_rpc_lock(cli->cl_rpc_lock);
673         }
674
675         m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
676         m->opd_dt_dev.dd_ops = &osp_dt_ops;
677         obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
678
679         rc = osp_connect_to_osd(env, m, osdname);
680         if (rc)
681                 GOTO(out_fini, rc);
682
683         rc = ptlrpcd_addref();
684         if (rc)
685                 GOTO(out_disconnect, rc);
686
687         rc = client_obd_setup(obd, cfg);
688         if (rc) {
689                 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
690                 GOTO(out_ref, rc);
691         }
692
693         osp_lprocfs_init(m);
694
695         rc = obd_fid_init(m->opd_obd, NULL, m->opd_connect_mdt ?
696                           LUSTRE_SEQ_METADATA : LUSTRE_SEQ_DATA);
697         if (rc) {
698                 CERROR("%s: fid init error: rc = %d\n",
699                        m->opd_obd->obd_name, rc);
700                 GOTO(out_proc, rc);
701         }
702
703         if (!m->opd_connect_mdt) {
704                 /* Initialize last id from the storage - will be
705                  * used in orphan cleanup. */
706                 rc = osp_last_used_init(env, m);
707                 if (rc)
708                         GOTO(out_proc, rc);
709
710
711                 /* Initialize precreation thread, it handles new
712                  * connections as well. */
713                 rc = osp_init_precreate(m);
714                 if (rc)
715                         GOTO(out_last_used, rc);
716                 /*
717                  * Initialize synhronization mechanism taking
718                  * care of propogating changes to OST in near
719                  * transactional manner.
720                  */
721                 rc = osp_sync_init(env, m);
722                 if (rc)
723                         GOTO(out_precreat, rc);
724         }
725
726         /*
727          * Initiate connect to OST
728          */
729         ll_generate_random_uuid(uuid);
730         class_uuid_unparse(uuid, &m->opd_cluuid);
731
732         imp = obd->u.cli.cl_import;
733
734         rc = ptlrpc_init_import(imp);
735         if (rc)
736                 GOTO(out, rc);
737         if (osdname)
738                 OBD_FREE(osdname, MAX_OBD_NAME);
739         RETURN(0);
740
741 out:
742         if (!m->opd_connect_mdt)
743                 /* stop sync thread */
744                 osp_sync_fini(m);
745 out_precreat:
746         /* stop precreate thread */
747         if (!m->opd_connect_mdt)
748                 osp_precreate_fini(m);
749 out_last_used:
750         if (!m->opd_connect_mdt)
751                 osp_last_used_fini(env, m);
752 out_proc:
753         ptlrpc_lprocfs_unregister_obd(obd);
754         lprocfs_obd_cleanup(obd);
755         if (m->opd_symlink)
756                 lprocfs_remove(&m->opd_symlink);
757         client_obd_cleanup(obd);
758 out_ref:
759         ptlrpcd_decref();
760 out_disconnect:
761         if (m->opd_connect_mdt) {
762                 struct client_obd *cli = &m->opd_obd->u.cli;
763                 if (cli->cl_rpc_lock != NULL) {
764                         OBD_FREE_PTR(cli->cl_rpc_lock);
765                         cli->cl_rpc_lock = NULL;
766                 }
767         }
768         obd_disconnect(m->opd_storage_exp);
769 out_fini:
770         if (osdname)
771                 OBD_FREE(osdname, MAX_OBD_NAME);
772         RETURN(rc);
773 }
774
775 static struct lu_device *osp_device_free(const struct lu_env *env,
776                                          struct lu_device *lu)
777 {
778         struct osp_device *m = lu2osp_dev(lu);
779
780         ENTRY;
781
782         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
783                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
784                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
785         }
786         dt_device_fini(&m->opd_dt_dev);
787         OBD_FREE_PTR(m);
788         RETURN(NULL);
789 }
790
791 static struct lu_device *osp_device_alloc(const struct lu_env *env,
792                                           struct lu_device_type *t,
793                                           struct lustre_cfg *lcfg)
794 {
795         struct osp_device *m;
796         struct lu_device  *l;
797
798         OBD_ALLOC_PTR(m);
799         if (m == NULL) {
800                 l = ERR_PTR(-ENOMEM);
801         } else {
802                 int rc;
803
804                 l = osp2lu_dev(m);
805                 dt_device_init(&m->opd_dt_dev, t);
806                 rc = osp_init0(env, m, t, lcfg);
807                 if (rc != 0) {
808                         osp_device_free(env, l);
809                         l = ERR_PTR(rc);
810                 }
811         }
812         return l;
813 }
814
815 static struct lu_device *osp_device_fini(const struct lu_env *env,
816                                          struct lu_device *d)
817 {
818         struct osp_device *m = lu2osp_dev(d);
819         struct obd_import *imp;
820         int                rc;
821
822         ENTRY;
823
824         if (m->opd_async_requests != NULL) {
825                 out_destroy_update_req(m->opd_async_requests);
826                 m->opd_async_requests = NULL;
827         }
828
829         if (m->opd_storage_exp)
830                 obd_disconnect(m->opd_storage_exp);
831
832         imp = m->opd_obd->u.cli.cl_import;
833
834         if (imp->imp_rq_pool) {
835                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
836                 imp->imp_rq_pool = NULL;
837         }
838
839         if (m->opd_symlink)
840                 lprocfs_remove(&m->opd_symlink);
841
842         LASSERT(m->opd_obd);
843         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
844         lprocfs_obd_cleanup(m->opd_obd);
845
846         if (m->opd_connect_mdt) {
847                 struct client_obd *cli = &m->opd_obd->u.cli;
848                 if (cli->cl_rpc_lock != NULL) {
849                         OBD_FREE_PTR(cli->cl_rpc_lock);
850                         cli->cl_rpc_lock = NULL;
851                 }
852         }
853
854         rc = client_obd_cleanup(m->opd_obd);
855         LASSERTF(rc == 0, "error %d\n", rc);
856
857         ptlrpcd_decref();
858
859         RETURN(NULL);
860 }
861
862 static int osp_reconnect(const struct lu_env *env,
863                          struct obd_export *exp, struct obd_device *obd,
864                          struct obd_uuid *cluuid,
865                          struct obd_connect_data *data,
866                          void *localdata)
867 {
868         return 0;
869 }
870
871 static int osp_prepare_fid_client(struct osp_device *osp)
872 {
873         LASSERT(osp->opd_obd->u.cli.cl_seq != NULL);
874         if (osp->opd_obd->u.cli.cl_seq->lcs_exp != NULL)
875                 return 0;
876
877         LASSERT(osp->opd_exp != NULL);
878         osp->opd_obd->u.cli.cl_seq->lcs_exp =
879                                 class_export_get(osp->opd_exp);
880         return 0;
881 }
882
883 /*
884  * we use exports to track all LOD users
885  */
886 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
887                            struct obd_device *obd, struct obd_uuid *cluuid,
888                            struct obd_connect_data *data, void *localdata)
889 {
890         struct osp_device       *osp = lu2osp_dev(obd->obd_lu_dev);
891         struct obd_connect_data *ocd;
892         struct obd_import       *imp;
893         struct lustre_handle     conn;
894         int                      rc;
895
896         ENTRY;
897
898         CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
899
900         rc = class_connect(&conn, obd, cluuid);
901         if (rc)
902                 RETURN(rc);
903
904         *exp = class_conn2export(&conn);
905         /* Why should there ever be more than 1 connect? */
906         osp->opd_connects++;
907         LASSERT(osp->opd_connects == 1);
908
909         osp->opd_exp = *exp;
910
911         imp = osp->opd_obd->u.cli.cl_import;
912         imp->imp_dlm_handle = conn;
913
914         LASSERT(data != NULL);
915         LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
916         ocd = &imp->imp_connect_data;
917         *ocd = *data;
918
919         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
920
921         ocd->ocd_version = LUSTRE_VERSION_CODE;
922         ocd->ocd_index = data->ocd_index;
923         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
924
925         rc = ptlrpc_connect_import(imp);
926         if (rc) {
927                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
928                 GOTO(out, rc);
929         }
930
931         ptlrpc_pinger_add_import(imp);
932 out:
933         RETURN(rc);
934 }
935
936 /*
937  * once last export (we don't count self-export) disappeared
938  * osp can be released
939  */
940 static int osp_obd_disconnect(struct obd_export *exp)
941 {
942         struct obd_device *obd = exp->exp_obd;
943         struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
944         int                rc;
945         ENTRY;
946
947         /* Only disconnect the underlying layers on the final disconnect. */
948         LASSERT(osp->opd_connects == 1);
949         osp->opd_connects--;
950
951         rc = class_disconnect(exp);
952         if (rc) {
953                 CERROR("%s: class disconnect error: rc = %d\n",
954                        obd->obd_name, rc);
955                 RETURN(rc);
956         }
957
958         /* destroy the device */
959         class_manual_cleanup(obd);
960
961         RETURN(rc);
962 }
963
964 /*
965  * lprocfs helpers still use OBD API, let's keep obd_statfs() support
966  */
967 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
968                           struct obd_statfs *osfs, __u64 max_age, __u32 flags)
969 {
970         struct obd_statfs       *msfs;
971         struct ptlrpc_request   *req;
972         struct obd_import       *imp = NULL;
973         int                      rc;
974
975         ENTRY;
976
977         /* Since the request might also come from lprocfs, so we need
978          * sync this with client_disconnect_export Bug15684 */
979         down_read(&exp->exp_obd->u.cli.cl_sem);
980         if (exp->exp_obd->u.cli.cl_import)
981                 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
982         up_read(&exp->exp_obd->u.cli.cl_sem);
983         if (!imp)
984                 RETURN(-ENODEV);
985
986         /* We could possibly pass max_age in the request (as an absolute
987          * timestamp or a "seconds.usec ago") so the target can avoid doing
988          * extra calls into the filesystem if that isn't necessary (e.g.
989          * during mount that would help a bit).  Having relative timestamps
990          * is not so great if request processing is slow, while absolute
991          * timestamps are not ideal because they need time synchronization. */
992         req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
993
994         class_import_put(imp);
995
996         if (req == NULL)
997                 RETURN(-ENOMEM);
998
999         rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
1000         if (rc) {
1001                 ptlrpc_request_free(req);
1002                 RETURN(rc);
1003         }
1004         ptlrpc_request_set_replen(req);
1005         req->rq_request_portal = OST_CREATE_PORTAL;
1006         ptlrpc_at_set_req_timeout(req);
1007
1008         if (flags & OBD_STATFS_NODELAY) {
1009                 /* procfs requests not want stat in wait for avoid deadlock */
1010                 req->rq_no_resend = 1;
1011                 req->rq_no_delay = 1;
1012         }
1013
1014         rc = ptlrpc_queue_wait(req);
1015         if (rc)
1016                 GOTO(out, rc);
1017
1018         msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
1019         if (msfs == NULL)
1020                 GOTO(out, rc = -EPROTO);
1021
1022         *osfs = *msfs;
1023
1024         EXIT;
1025 out:
1026         ptlrpc_req_finished(req);
1027         return rc;
1028 }
1029
1030 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
1031                             enum obd_import_event event)
1032 {
1033         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
1034
1035         switch (event) {
1036         case IMP_EVENT_DISCON:
1037                 d->opd_got_disconnected = 1;
1038                 d->opd_imp_connected = 0;
1039                 if (d->opd_connect_mdt)
1040                         break;
1041
1042                 if (d->opd_pre != NULL) {
1043                         osp_pre_update_status(d, -ENODEV);
1044                         wake_up(&d->opd_pre_waitq);
1045                 }
1046
1047                 CDEBUG(D_HA, "got disconnected\n");
1048                 break;
1049         case IMP_EVENT_INACTIVE:
1050                 d->opd_imp_active = 0;
1051                 if (d->opd_connect_mdt)
1052                         break;
1053
1054                 if (d->opd_pre != NULL) {
1055                         osp_pre_update_status(d, -ENODEV);
1056                         wake_up(&d->opd_pre_waitq);
1057                 }
1058
1059                 CDEBUG(D_HA, "got inactive\n");
1060                 break;
1061         case IMP_EVENT_ACTIVE:
1062                 d->opd_imp_active = 1;
1063
1064                 if (osp_prepare_fid_client(d) != 0)
1065                         break;
1066
1067                 if (d->opd_got_disconnected)
1068                         d->opd_new_connection = 1;
1069                 d->opd_imp_connected = 1;
1070                 d->opd_imp_seen_connected = 1;
1071                 if (d->opd_connect_mdt)
1072                         break;
1073
1074                 if (d->opd_pre != NULL)
1075                         wake_up(&d->opd_pre_waitq);
1076
1077                 __osp_sync_check_for_work(d);
1078                 CDEBUG(D_HA, "got connected\n");
1079                 break;
1080         case IMP_EVENT_INVALIDATE:
1081                 if (obd->obd_namespace == NULL)
1082                         break;
1083                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
1084                 break;
1085         case IMP_EVENT_OCD:
1086         case IMP_EVENT_DEACTIVATE:
1087         case IMP_EVENT_ACTIVATE:
1088                 break;
1089         default:
1090                 CERROR("%s: unsupported import event: %#x\n",
1091                        obd->obd_name, event);
1092         }
1093         return 0;
1094 }
1095
1096 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
1097                          void *karg, void *uarg)
1098 {
1099         struct obd_device       *obd = exp->exp_obd;
1100         struct osp_device       *d;
1101         struct obd_ioctl_data   *data = karg;
1102         int                      rc = 0;
1103
1104         ENTRY;
1105
1106         LASSERT(obd->obd_lu_dev);
1107         d = lu2osp_dev(obd->obd_lu_dev);
1108         LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
1109
1110         if (!try_module_get(THIS_MODULE)) {
1111                 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
1112                 return -EINVAL;
1113         }
1114
1115         switch (cmd) {
1116         case OBD_IOC_CLIENT_RECOVER:
1117                 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
1118                                            data->ioc_inlbuf1, 0);
1119                 if (rc > 0)
1120                         rc = 0;
1121                 break;
1122         case IOC_OSC_SET_ACTIVE:
1123                 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
1124                                               data->ioc_offset);
1125                 break;
1126         case OBD_IOC_PING_TARGET:
1127                 rc = ptlrpc_obd_ping(obd);
1128                 break;
1129         default:
1130                 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
1131                        cmd, current_comm());
1132                 rc = -ENOTTY;
1133         }
1134         module_put(THIS_MODULE);
1135         return rc;
1136 }
1137
1138 static int osp_obd_get_info(const struct lu_env *env, struct obd_export *exp,
1139                             __u32 keylen, void *key, __u32 *vallen, void *val,
1140                             struct lov_stripe_md *lsm)
1141 {
1142         int rc = -EINVAL;
1143
1144         if (KEY_IS(KEY_OSP_CONNECTED)) {
1145                 struct obd_device       *obd = exp->exp_obd;
1146                 struct osp_device       *osp;
1147
1148                 if (!obd->obd_set_up || obd->obd_stopping)
1149                         RETURN(-EAGAIN);
1150
1151                 osp = lu2osp_dev(obd->obd_lu_dev);
1152                 LASSERT(osp);
1153                 /*
1154                  * 1.8/2.0 behaviour is that OST being connected once at least
1155                  * is considered "healthy". and one "healthy" OST is enough to
1156                  * allow lustre clients to connect to MDS
1157                  */
1158                 RETURN(!osp->opd_imp_seen_connected);
1159         }
1160
1161         RETURN(rc);
1162 }
1163
1164 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
1165 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
1166 static void osp_key_exit(const struct lu_context *ctx,
1167                          struct lu_context_key *key, void *data)
1168 {
1169         struct osp_thread_info *info = data;
1170
1171         info->osi_attr.la_valid = 0;
1172 }
1173
1174 struct lu_context_key osp_thread_key = {
1175         .lct_tags = LCT_MD_THREAD,
1176         .lct_init = osp_key_init,
1177         .lct_fini = osp_key_fini,
1178         .lct_exit = osp_key_exit
1179 };
1180
1181 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
1182 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
1183
1184 struct lu_context_key osp_txn_key = {
1185         .lct_tags = LCT_OSP_THREAD,
1186         .lct_init = osp_txn_key_init,
1187         .lct_fini = osp_txn_key_fini
1188 };
1189 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
1190
1191 static struct lu_device_type_operations osp_device_type_ops = {
1192         .ldto_init           = osp_type_init,
1193         .ldto_fini           = osp_type_fini,
1194
1195         .ldto_start          = osp_type_start,
1196         .ldto_stop           = osp_type_stop,
1197
1198         .ldto_device_alloc   = osp_device_alloc,
1199         .ldto_device_free    = osp_device_free,
1200
1201         .ldto_device_fini    = osp_device_fini
1202 };
1203
1204 static struct lu_device_type osp_device_type = {
1205         .ldt_tags     = LU_DEVICE_DT,
1206         .ldt_name     = LUSTRE_OSP_NAME,
1207         .ldt_ops      = &osp_device_type_ops,
1208         .ldt_ctx_tags = LCT_MD_THREAD | LCT_DT_THREAD,
1209 };
1210
1211 static struct obd_ops osp_obd_device_ops = {
1212         .o_owner        = THIS_MODULE,
1213         .o_add_conn     = client_import_add_conn,
1214         .o_del_conn     = client_import_del_conn,
1215         .o_reconnect    = osp_reconnect,
1216         .o_connect      = osp_obd_connect,
1217         .o_disconnect   = osp_obd_disconnect,
1218         .o_get_info     = osp_obd_get_info,
1219         .o_import_event = osp_import_event,
1220         .o_iocontrol    = osp_iocontrol,
1221         .o_statfs       = osp_obd_statfs,
1222         .o_fid_init     = client_fid_init,
1223         .o_fid_fini     = client_fid_fini,
1224 };
1225
1226 struct llog_operations osp_mds_ost_orig_logops;
1227
1228 static int __init osp_mod_init(void)
1229 {
1230         struct lprocfs_static_vars       lvars;
1231         cfs_proc_dir_entry_t            *osc_proc_dir;
1232         int                              rc;
1233
1234         rc = lu_kmem_init(osp_caches);
1235         if (rc)
1236                 return rc;
1237
1238         lprocfs_osp_init_vars(&lvars);
1239
1240         rc = class_register_type(&osp_obd_device_ops, NULL, NULL,
1241 #ifndef HAVE_ONLY_PROCFS_SEQ
1242                                 lvars.module_vars,
1243 #endif
1244                                 LUSTRE_OSP_NAME, &osp_device_type);
1245
1246         /* create "osc" entry in procfs for compatibility purposes */
1247         if (rc != 0) {
1248                 lu_kmem_fini(osp_caches);
1249                 return rc;
1250         }
1251
1252         lprocfs_lwp_init_vars(&lvars);
1253
1254         rc = class_register_type(&lwp_obd_device_ops, NULL, NULL,
1255 #ifndef HAVE_ONLY_PROCFS_SEQ
1256                                 lvars.module_vars,
1257 #endif
1258                                 LUSTRE_LWP_NAME, &lwp_device_type);
1259         if (rc != 0) {
1260                 class_unregister_type(LUSTRE_OSP_NAME);
1261                 lu_kmem_fini(osp_caches);
1262                 return rc;
1263         }
1264
1265         /* Note: add_rec/delcare_add_rec will be only used by catalogs */
1266         osp_mds_ost_orig_logops = llog_osd_ops;
1267         osp_mds_ost_orig_logops.lop_add = llog_cat_add_rec;
1268         osp_mds_ost_orig_logops.lop_declare_add = llog_cat_declare_add_rec;
1269
1270         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1271         if (osc_proc_dir == NULL) {
1272                 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1273                                                 NULL);
1274                 if (IS_ERR(osc_proc_dir))
1275                         CERROR("osp: can't create compat entry \"osc\": %d\n",
1276                                (int) PTR_ERR(osc_proc_dir));
1277         }
1278         return rc;
1279 }
1280
1281 static void __exit osp_mod_exit(void)
1282 {
1283         lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1284
1285         class_unregister_type(LUSTRE_LWP_NAME);
1286         class_unregister_type(LUSTRE_OSP_NAME);
1287         lu_kmem_fini(osp_caches);
1288 }
1289
1290 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1291 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1292 MODULE_LICENSE("GPL");
1293
1294 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);