Whamcloud - gitweb
LU-1842 quota: add core QSD code
[fs/lustre-release.git] / lustre / osp / osp_dev.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2011, 2012, Intel, Inc.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lustre/osp/osp_dev.c
37  *
38  * Lustre OST Proxy Device
39  *
40  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
41  * Author: Mikhail Pershin <mike.pershin@intel.com>
42  */
43
44 #ifndef EXPORT_SYMTAB
45 # define EXPORT_SYMTAB
46 #endif
47 #define DEBUG_SUBSYSTEM S_MDS
48
49 #include <obd_class.h>
50 #include <lustre_param.h>
51
52 #include "osp_internal.h"
53
54 /* Slab for OSP object allocation */
55 cfs_mem_cache_t *osp_object_kmem;
56
57 static struct lu_kmem_descr osp_caches[] = {
58         {
59                 .ckd_cache = &osp_object_kmem,
60                 .ckd_name  = "osp_obj",
61                 .ckd_size  = sizeof(struct osp_object)
62         },
63         {
64                 .ckd_cache = NULL
65         }
66 };
67
68 struct lu_object *osp_object_alloc(const struct lu_env *env,
69                                    const struct lu_object_header *hdr,
70                                    struct lu_device *d)
71 {
72         struct lu_object_header *h;
73         struct osp_object       *o;
74         struct lu_object        *l;
75
76         LASSERT(hdr == NULL);
77
78         OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
79         if (o != NULL) {
80                 l = &o->opo_obj.do_lu;
81                 h = &o->opo_header;
82
83                 lu_object_header_init(h);
84                 dt_object_init(&o->opo_obj, h, d);
85                 lu_object_add_top(h, l);
86
87                 l->lo_ops = &osp_lu_obj_ops;
88
89                 return l;
90         } else {
91                 return NULL;
92         }
93 }
94
95 /* Update opd_last_used_id along with checking for gap in objid sequence */
96 void osp_update_last_id(struct osp_device *d, obd_id objid)
97 {
98         /*
99          * we might have lost precreated objects due to VBR and precreate
100          * orphans, the gap in objid can be calculated properly only here
101          */
102         if (objid > le64_to_cpu(d->opd_last_used_id)) {
103                 if (objid - le64_to_cpu(d->opd_last_used_id) > 1) {
104                         d->opd_gap_start = le64_to_cpu(d->opd_last_used_id) + 1;
105                         d->opd_gap_count = objid - d->opd_gap_start;
106                         CDEBUG(D_HA, "Gap in objids: %d, start = %llu\n",
107                                d->opd_gap_count, d->opd_gap_start);
108                 }
109                 d->opd_last_used_id = cpu_to_le64(objid);
110         }
111 }
112
113 static int osp_last_used_init(const struct lu_env *env, struct osp_device *m)
114 {
115         struct osp_thread_info  *osi = osp_env_info(env);
116         struct dt_object_format  dof = { 0 };
117         struct dt_object        *o;
118         int                      rc;
119
120         ENTRY;
121
122         osi->osi_attr.la_valid = LA_MODE;
123         osi->osi_attr.la_mode = S_IFREG | 0644;
124         lu_local_obj_fid(&osi->osi_fid, MDD_LOV_OBJ_OID);
125         dof.dof_type = DFT_REGULAR;
126         o = dt_find_or_create(env, m->opd_storage, &osi->osi_fid, &dof,
127                               &osi->osi_attr);
128         if (IS_ERR(o))
129                 RETURN(PTR_ERR(o));
130
131         rc = dt_attr_get(env, o, &osi->osi_attr, NULL);
132         if (rc)
133                 GOTO(out, rc);
134
135         /* object will be released in device cleanup path */
136         m->opd_last_used_file = o;
137
138         if (osi->osi_attr.la_size >= sizeof(osi->osi_id) *
139                                      (m->opd_index + 1)) {
140                 osp_objid_buf_prep(osi, m, m->opd_index);
141                 rc = dt_record_read(env, o, &osi->osi_lb, &osi->osi_off);
142                 if (rc != 0)
143                         GOTO(out, rc);
144         } else {
145                 /* reset value to 0, just to make sure and change file's size */
146                 struct thandle *th;
147
148                 m->opd_last_used_id = 0;
149                 osp_objid_buf_prep(osi, m, m->opd_index);
150
151                 th = dt_trans_create(env, m->opd_storage);
152                 if (IS_ERR(th))
153                         GOTO(out, rc = PTR_ERR(th));
154
155                 rc = dt_declare_record_write(env, m->opd_last_used_file,
156                                              osi->osi_lb.lb_len, osi->osi_off,
157                                              th);
158                 if (rc) {
159                         dt_trans_stop(env, m->opd_storage, th);
160                         GOTO(out, rc);
161                 }
162
163                 rc = dt_trans_start_local(env, m->opd_storage, th);
164                 if (rc) {
165                         dt_trans_stop(env, m->opd_storage, th);
166                         GOTO(out, rc);
167                 }
168
169                 rc = dt_record_write(env, m->opd_last_used_file, &osi->osi_lb,
170                                      &osi->osi_off, th);
171                 dt_trans_stop(env, m->opd_storage, th);
172                 if (rc)
173                         GOTO(out, rc);
174         }
175         RETURN(0);
176 out:
177         CERROR("%s: can't initialize lov_objid: %d\n",
178                m->opd_obd->obd_name, rc);
179         lu_object_put(env, &o->do_lu);
180         m->opd_last_used_file = NULL;
181         return rc;
182 }
183
184 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
185 {
186         if (d->opd_last_used_file != NULL) {
187                 lu_object_put(env, &d->opd_last_used_file->do_lu);
188                 d->opd_last_used_file = NULL;
189         }
190 }
191
192 int osp_disconnect(struct osp_device *d)
193 {
194         struct obd_import *imp;
195         int rc = 0;
196
197         imp = d->opd_obd->u.cli.cl_import;
198
199         /* Mark import deactivated now, so we don't try to reconnect if any
200          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
201          * fully deactivate the import, or that would drop all requests. */
202         LASSERT(imp != NULL);
203         cfs_spin_lock(&imp->imp_lock);
204         imp->imp_deactive = 1;
205         cfs_spin_unlock(&imp->imp_lock);
206
207         ptlrpc_deactivate_import(imp);
208
209         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
210          * delete it regardless.  (It's safe to delete an import that was
211          * never added.) */
212         (void)ptlrpc_pinger_del_import(imp);
213
214         rc = ptlrpc_disconnect_import(imp, 0);
215         if (rc)
216                 CERROR("%s: can't disconnect: rc = %d\n",
217                        d->opd_obd->obd_name, rc);
218
219         ptlrpc_invalidate_import(imp);
220
221         RETURN(rc);
222 }
223
224 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
225 {
226         int                      rc = 0;
227         ENTRY;
228
229         if (is_osp_on_ost(d->opd_obd->obd_name)) {
230                 rc = osp_disconnect(d);
231                 RETURN(rc);
232         }
233
234         LASSERT(env);
235         /* release last_used file */
236         osp_last_used_fini(env, d);
237
238         rc = osp_disconnect(d);
239
240         /* stop precreate thread */
241         osp_precreate_fini(d);
242
243         /* stop sync thread */
244         osp_sync_fini(d);
245
246         RETURN(rc);
247 }
248
249 static int osp_process_config(const struct lu_env *env,
250                               struct lu_device *dev, struct lustre_cfg *lcfg)
251 {
252         struct osp_device               *d = lu2osp_dev(dev);
253         struct lprocfs_static_vars       lvars = { 0 };
254         int                              rc;
255
256         ENTRY;
257
258         switch (lcfg->lcfg_command) {
259         case LCFG_CLEANUP:
260                 if (!is_osp_on_ost(d->opd_obd->obd_name))
261                         lu_dev_del_linkage(dev->ld_site, dev);
262                 rc = osp_shutdown(env, d);
263                 break;
264         case LCFG_PARAM:
265                 lprocfs_osp_init_vars(&lvars);
266
267                 LASSERT(d->opd_obd);
268                 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
269                                               lcfg, d->opd_obd);
270                 if (rc > 0)
271                         rc = 0;
272                 if (rc == -ENOSYS) {
273                         /* class_process_proc_param() haven't found matching
274                          * parameter and returned ENOSYS so that layer(s)
275                          * below could use that. But OSP is the bottom, so
276                          * just ignore it */
277                         CERROR("%s: unknown param %s\n",
278                                (char *)lustre_cfg_string(lcfg, 0),
279                                (char *)lustre_cfg_string(lcfg, 1));
280                         rc = 0;
281                 }
282                 break;
283         default:
284                 CERROR("%s: unknown command %u\n",
285                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
286                 rc = 0;
287                 break;
288         }
289
290         RETURN(rc);
291 }
292
293 static int osp_recovery_complete(const struct lu_env *env,
294                                  struct lu_device *dev)
295 {
296         struct osp_device       *osp = lu2osp_dev(dev);
297         int                      rc = 0;
298
299         ENTRY;
300         osp->opd_recovery_completed = 1;
301         cfs_waitq_signal(&osp->opd_pre_waitq);
302         RETURN(rc);
303 }
304
305 const struct lu_device_operations osp_lu_ops = {
306         .ldo_object_alloc       = osp_object_alloc,
307         .ldo_process_config     = osp_process_config,
308         .ldo_recovery_complete  = osp_recovery_complete,
309 };
310
311 /**
312  * provides with statfs from corresponded OST
313  *
314  */
315 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
316                       struct obd_statfs *sfs)
317 {
318         struct osp_device *d = dt2osp_dev(dev);
319
320         ENTRY;
321
322         if (unlikely(d->opd_imp_active == 0)) {
323                 /*
324                  * in case of inactive OST we return nulls
325                  * so that caller can understand this device
326                  * is unusable for new objects
327                  *
328                  * XXX: shouldn't we take normal statfs and fill
329                  * just few specific fields with zeroes?
330                  */
331                 memset(sfs, 0, sizeof(*sfs));
332                 sfs->os_bsize = 4096;
333                 RETURN(0);
334         }
335
336         /* return recently updated data */
337         *sfs = d->opd_statfs;
338
339         /*
340          * layer above osp (usually lod) can use ffree to estimate
341          * how many objects are available for immediate creation
342          */
343         cfs_spin_lock(&d->opd_pre_lock);
344         sfs->os_ffree = d->opd_pre_last_created - d->opd_pre_next;
345         cfs_spin_unlock(&d->opd_pre_lock);
346
347         CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
348                LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
349                sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
350                sfs->os_files, sfs->os_ffree);
351         RETURN(0);
352 }
353
354 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
355 {
356         ENTRY;
357
358         /*
359          * XXX: wake up sync thread, command it to start flushing asap?
360          */
361
362         RETURN(0);
363 }
364
365 const struct dt_device_operations osp_dt_ops = {
366         .dt_statfs      = osp_statfs,
367         .dt_sync        = osp_sync,
368 };
369
370 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
371                               const char *nextdev)
372 {
373         struct obd_connect_data *data = NULL;
374         struct obd_device       *obd;
375         int                      rc;
376
377         ENTRY;
378
379         LASSERT(m->opd_storage_exp == NULL);
380
381         OBD_ALLOC_PTR(data);
382         if (data == NULL)
383                 RETURN(-ENOMEM);
384
385         obd = class_name2obd(nextdev);
386         if (obd == NULL) {
387                 CERROR("%s: can't locate next device: %s\n",
388                        m->opd_obd->obd_name, nextdev);
389                 GOTO(out, rc = -ENOTCONN);
390         }
391
392         rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
393                          NULL);
394         if (rc) {
395                 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
396                        m->opd_obd->obd_name, nextdev, rc);
397                 GOTO(out, rc);
398         }
399
400         m->opd_dt_dev.dd_lu_dev.ld_site =
401                 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
402         LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
403         m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
404
405 out:
406         OBD_FREE_PTR(data);
407         RETURN(rc);
408 }
409
410 static int osp_init0(const struct lu_env *env, struct osp_device *m,
411                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
412 {
413         struct lprocfs_static_vars       lvars = { 0 };
414         struct proc_dir_entry           *osc_proc_dir;
415         struct obd_import               *imp;
416         class_uuid_t                     uuid;
417         char                            *src, *ost, *mdt, *osdname = NULL;
418         int                              rc, idx;
419
420         ENTRY;
421
422         m->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
423         if (m->opd_obd == NULL) {
424                 CERROR("Cannot find obd with name %s\n",
425                        lustre_cfg_string(cfg, 0));
426                 RETURN(-ENODEV);
427         }
428
429         /* There is no record in the MDT configuration for the local disk
430          * device, so we have to extract this from elsewhere in the profile.
431          * The only information we get at setup is from the OSC records:
432          * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
433          * Note that 1.8 generated configs are missing the -MDTxxxx part.
434          * We need to reconstruct the name of the underlying OSD from this:
435          * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd".  We
436          * also need to determine the OST index from this - will be used
437          * to calculate the offset in shared lov_objids file later */
438
439         src = lustre_cfg_string(cfg, 0);
440         if (src == NULL)
441                 RETURN(-EINVAL);
442
443         ost = strstr(src, "-OST");
444         if (ost == NULL)
445                 RETURN(-EINVAL);
446
447         idx = simple_strtol(ost + 4, &mdt, 16);
448         if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
449                 CERROR("%s: invalid OST index in '%s'\n",
450                        m->opd_obd->obd_name, src);
451                 GOTO(out_fini, rc = -EINVAL);
452         }
453         m->opd_index = idx;
454
455         idx = ost - src;
456         /* check the fsname length, and after this everything else will fit */
457         if (idx > MTI_NAME_MAXLEN) {
458                 CERROR("%s: fsname too long in '%s'\n",
459                        m->opd_obd->obd_name, src);
460                 GOTO(out_fini, rc = -EINVAL);
461         }
462
463         OBD_ALLOC(osdname, MAX_OBD_NAME);
464         if (osdname == NULL)
465                 GOTO(out_fini, rc = -ENOMEM);
466
467         memcpy(osdname, src, idx); /* copy just the fsname part */
468         osdname[idx] = '\0';
469
470         mdt = strstr(mdt, "-MDT");
471         if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
472                 strcat(osdname, "-MDT0000");
473         else
474                 strcat(osdname, mdt);
475         strcat(osdname, "-osd");
476         CDEBUG(D_HA, "%s: connect to %s (%s)\n",
477                m->opd_obd->obd_name, osdname, src);
478
479         m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
480         m->opd_dt_dev.dd_ops = &osp_dt_ops;
481         m->opd_obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
482
483         rc = osp_connect_to_osd(env, m, osdname);
484         if (rc)
485                 GOTO(out_fini, rc);
486
487         rc = ptlrpcd_addref();
488         if (rc)
489                 GOTO(out_disconnect, rc);
490
491         rc = client_obd_setup(m->opd_obd, cfg);
492         if (rc) {
493                 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
494                 GOTO(out_ref, rc);
495         }
496
497         lprocfs_osp_init_vars(&lvars);
498         if (lprocfs_obd_setup(m->opd_obd, lvars.obd_vars) == 0)
499                 ptlrpc_lprocfs_register_obd(m->opd_obd);
500
501         /* for compatibility we link old procfs's OSC entries to osp ones */
502         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
503         if (osc_proc_dir) {
504                 cfs_proc_dir_entry_t    *symlink = NULL;
505                 char                    *name;
506
507                 OBD_ALLOC(name, strlen(m->opd_obd->obd_name) + 1);
508                 if (name == NULL)
509                         GOTO(out, rc = -ENOMEM);
510
511                 strcpy(name, m->opd_obd->obd_name);
512                 if (strstr(name, "osc"))
513                         symlink = lprocfs_add_symlink(name, osc_proc_dir,
514                                                       "../osp/%s",
515                                                       m->opd_obd->obd_name);
516                 OBD_FREE(name, strlen(m->opd_obd->obd_name) + 1);
517                 m->opd_symlink = symlink;
518         }
519
520         /*
521          * Initialize last id from the storage - will be used in orphan cleanup
522          */
523         rc = osp_last_used_init(env, m);
524         if (rc)
525                 GOTO(out_proc, rc);
526
527         /*
528          * Initialize precreation thread, it handles new connections as well
529          */
530         rc = osp_init_precreate(m);
531         if (rc)
532                 GOTO(out_last_used, rc);
533
534         /*
535          * Initialize synhronization mechanism taking care of propogating
536          * changes to OST in near transactional manner
537          */
538         rc = osp_sync_init(env, m);
539         if (rc)
540                 GOTO(out_precreat, rc);
541
542         /*
543          * Initiate connect to OST
544          */
545         ll_generate_random_uuid(uuid);
546         class_uuid_unparse(uuid, &m->opd_cluuid);
547
548         imp = m->opd_obd->u.cli.cl_import;
549
550         rc = ptlrpc_init_import(imp);
551         if (rc)
552                 GOTO(out, rc);
553         if (osdname)
554                 OBD_FREE(osdname, MAX_OBD_NAME);
555         RETURN(0);
556
557 out:
558         /* stop sync thread */
559         osp_sync_fini(m);
560 out_precreat:
561         /* stop precreate thread */
562         osp_precreate_fini(m);
563 out_last_used:
564         osp_last_used_fini(env, m);
565 out_proc:
566         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
567         lprocfs_obd_cleanup(m->opd_obd);
568         class_destroy_import(m->opd_obd->u.cli.cl_import);
569         client_obd_cleanup(m->opd_obd);
570 out_ref:
571         ptlrpcd_decref();
572 out_disconnect:
573         obd_disconnect(m->opd_storage_exp);
574 out_fini:
575         if (osdname)
576                 OBD_FREE(osdname, MAX_OBD_NAME);
577         RETURN(rc);
578 }
579
580 static struct lu_device *osp_device_free(const struct lu_env *env,
581                                          struct lu_device *lu)
582 {
583         struct osp_device *m = lu2osp_dev(lu);
584
585         ENTRY;
586
587         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
588                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
589                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
590         }
591         dt_device_fini(&m->opd_dt_dev);
592         OBD_FREE_PTR(m);
593         RETURN(NULL);
594 }
595
596 static struct lu_device *osp_device_alloc(const struct lu_env *env,
597                                           struct lu_device_type *t,
598                                           struct lustre_cfg *lcfg)
599 {
600         struct osp_device *m;
601         struct lu_device  *l;
602
603         OBD_ALLOC_PTR(m);
604         if (m == NULL) {
605                 l = ERR_PTR(-ENOMEM);
606         } else {
607                 int rc;
608
609                 l = osp2lu_dev(m);
610                 dt_device_init(&m->opd_dt_dev, t);
611                 if (is_osp_on_ost(lustre_cfg_string(lcfg, 0)))
612                         rc = osp_init_for_ost(env, m, t, lcfg);
613                 else
614                         rc = osp_init0(env, m, t, lcfg);
615                 if (rc != 0) {
616                         osp_device_free(env, l);
617                         l = ERR_PTR(rc);
618                 }
619         }
620         return l;
621 }
622
623 static struct lu_device *osp_device_fini(const struct lu_env *env,
624                                          struct lu_device *d)
625 {
626         struct osp_device *m = lu2osp_dev(d);
627         struct obd_import *imp;
628         int                rc;
629
630         ENTRY;
631
632         if (m->opd_storage_exp)
633                 obd_disconnect(m->opd_storage_exp);
634
635         if (is_osp_on_ost(m->opd_obd->obd_name))
636                 osp_fini_for_ost(m);
637
638         imp = m->opd_obd->u.cli.cl_import;
639
640         if (imp->imp_rq_pool) {
641                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
642                 imp->imp_rq_pool = NULL;
643         }
644
645         obd_cleanup_client_import(m->opd_obd);
646
647         if (m->opd_symlink)
648                 lprocfs_remove(&m->opd_symlink);
649
650         LASSERT(m->opd_obd);
651         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
652         lprocfs_obd_cleanup(m->opd_obd);
653
654         rc = client_obd_cleanup(m->opd_obd);
655         LASSERTF(rc == 0, "error %d\n", rc);
656
657         ptlrpcd_decref();
658
659         RETURN(NULL);
660 }
661
662 static int osp_reconnect(const struct lu_env *env,
663                          struct obd_export *exp, struct obd_device *obd,
664                          struct obd_uuid *cluuid,
665                          struct obd_connect_data *data,
666                          void *localdata)
667 {
668         return 0;
669 }
670
671 /*
672  * we use exports to track all LOD users
673  */
674 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
675                            struct obd_device *obd, struct obd_uuid *cluuid,
676                            struct obd_connect_data *data, void *localdata)
677 {
678         struct osp_device       *osp = lu2osp_dev(obd->obd_lu_dev);
679         struct obd_connect_data *ocd;
680         struct obd_import       *imp;
681         struct lustre_handle     conn;
682         int                      rc;
683
684         ENTRY;
685
686         CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
687
688         rc = class_connect(&conn, obd, cluuid);
689         if (rc)
690                 RETURN(rc);
691
692         *exp = class_conn2export(&conn);
693         if (is_osp_on_ost(obd->obd_name))
694                 osp->opd_exp = *exp;
695
696         /* Why should there ever be more than 1 connect? */
697         osp->opd_connects++;
698         LASSERT(osp->opd_connects == 1);
699
700         imp = osp->opd_obd->u.cli.cl_import;
701         imp->imp_dlm_handle = conn;
702
703         ocd = &imp->imp_connect_data;
704         ocd->ocd_connect_flags = OBD_CONNECT_AT |
705                                  OBD_CONNECT_FULL20 |
706                                  OBD_CONNECT_INDEX |
707 #ifdef HAVE_LRU_RESIZE_SUPPORT
708                                  OBD_CONNECT_LRU_RESIZE |
709 #endif
710                                  OBD_CONNECT_MDS |
711                                  OBD_CONNECT_OSS_CAPA |
712                                  OBD_CONNECT_REQPORTAL |
713                                  OBD_CONNECT_SKIP_ORPHAN |
714                                  OBD_CONNECT_VERSION |
715                                  OBD_CONNECT_FID;
716
717         if (is_osp_on_ost(osp->opd_obd->obd_name))
718                 ocd->ocd_connect_flags |= OBD_CONNECT_LIGHTWEIGHT;
719
720         ocd->ocd_version = LUSTRE_VERSION_CODE;
721         LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
722         ocd->ocd_index = data->ocd_index;
723         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
724
725         rc = ptlrpc_connect_import(imp);
726         if (rc) {
727                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
728                 GOTO(out, rc);
729         }
730
731         ptlrpc_pinger_add_import(imp);
732
733 out:
734         RETURN(rc);
735 }
736
737 /*
738  * once last export (we don't count self-export) disappeared
739  * osp can be released
740  */
741 static int osp_obd_disconnect(struct obd_export *exp)
742 {
743         struct obd_device *obd = exp->exp_obd;
744         struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
745         int                rc;
746         ENTRY;
747
748         /* Only disconnect the underlying layers on the final disconnect. */
749         LASSERT(osp->opd_connects == 1);
750         osp->opd_connects--;
751
752         rc = class_disconnect(exp);
753         if (rc) {
754                 CERROR("%s: class disconnect error: rc = %d\n",
755                        obd->obd_name, rc);
756                 RETURN(rc);
757         }
758
759         /* destroy the device */
760         if (!is_osp_on_ost(obd->obd_name))
761                 class_manual_cleanup(obd);
762
763         RETURN(rc);
764 }
765
766 /*
767  * lprocfs helpers still use OBD API, let's keep obd_statfs() support
768  */
769 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
770                           struct obd_statfs *osfs, __u64 max_age, __u32 flags)
771 {
772         struct obd_statfs       *msfs;
773         struct ptlrpc_request   *req;
774         struct obd_import       *imp = NULL;
775         int                      rc;
776
777         ENTRY;
778
779         /* Since the request might also come from lprocfs, so we need
780          * sync this with client_disconnect_export Bug15684 */
781         cfs_down_read(&exp->exp_obd->u.cli.cl_sem);
782         if (exp->exp_obd->u.cli.cl_import)
783                 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
784         cfs_up_read(&exp->exp_obd->u.cli.cl_sem);
785         if (!imp)
786                 RETURN(-ENODEV);
787
788         /* We could possibly pass max_age in the request (as an absolute
789          * timestamp or a "seconds.usec ago") so the target can avoid doing
790          * extra calls into the filesystem if that isn't necessary (e.g.
791          * during mount that would help a bit).  Having relative timestamps
792          * is not so great if request processing is slow, while absolute
793          * timestamps are not ideal because they need time synchronization. */
794         req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
795
796         class_import_put(imp);
797
798         if (req == NULL)
799                 RETURN(-ENOMEM);
800
801         rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
802         if (rc) {
803                 ptlrpc_request_free(req);
804                 RETURN(rc);
805         }
806         ptlrpc_request_set_replen(req);
807         req->rq_request_portal = OST_CREATE_PORTAL;
808         ptlrpc_at_set_req_timeout(req);
809
810         if (flags & OBD_STATFS_NODELAY) {
811                 /* procfs requests not want stat in wait for avoid deadlock */
812                 req->rq_no_resend = 1;
813                 req->rq_no_delay = 1;
814         }
815
816         rc = ptlrpc_queue_wait(req);
817         if (rc)
818                 GOTO(out, rc);
819
820         msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
821         if (msfs == NULL)
822                 GOTO(out, rc = -EPROTO);
823
824         *osfs = *msfs;
825
826         EXIT;
827 out:
828         ptlrpc_req_finished(req);
829         return rc;
830 }
831
832 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
833                             enum obd_import_event event)
834 {
835         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
836
837         switch (event) {
838         case IMP_EVENT_DISCON:
839                 d->opd_got_disconnected = 1;
840                 d->opd_imp_connected = 0;
841                 if (is_osp_on_ost(d->opd_obd->obd_name))
842                         break;
843                 osp_pre_update_status(d, -ENODEV);
844                 cfs_waitq_signal(&d->opd_pre_waitq);
845                 CDEBUG(D_HA, "got disconnected\n");
846                 break;
847         case IMP_EVENT_INACTIVE:
848                 d->opd_imp_active = 0;
849                 if (is_osp_on_ost(d->opd_obd->obd_name))
850                         break;
851                 osp_pre_update_status(d, -ENODEV);
852                 cfs_waitq_signal(&d->opd_pre_waitq);
853                 CDEBUG(D_HA, "got inactive\n");
854                 break;
855         case IMP_EVENT_ACTIVE:
856                 d->opd_imp_active = 1;
857                 if (d->opd_got_disconnected)
858                         d->opd_new_connection = 1;
859                 d->opd_imp_connected = 1;
860                 d->opd_imp_seen_connected = 1;
861                 if (is_osp_on_ost(d->opd_obd->obd_name))
862                         break;
863                 cfs_waitq_signal(&d->opd_pre_waitq);
864                 __osp_sync_check_for_work(d);
865                 CDEBUG(D_HA, "got connected\n");
866                 break;
867         case IMP_EVENT_INVALIDATE:
868                 if (obd->obd_namespace == NULL)
869                         break;
870                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
871                 break;
872         case IMP_EVENT_OCD:
873                 break;
874         default:
875                 CERROR("%s: unsupported import event: %#x\n",
876                        obd->obd_name, event);
877         }
878         return 0;
879 }
880
881 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
882                          void *karg, void *uarg)
883 {
884         struct obd_device       *obd = exp->exp_obd;
885         struct osp_device       *d;
886         struct obd_ioctl_data   *data = karg;
887         int                      rc = 0;
888
889         ENTRY;
890
891         LASSERT(obd->obd_lu_dev);
892         d = lu2osp_dev(obd->obd_lu_dev);
893         LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
894
895         if (!cfs_try_module_get(THIS_MODULE)) {
896                 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
897                 return -EINVAL;
898         }
899
900         switch (cmd) {
901         case OBD_IOC_CLIENT_RECOVER:
902                 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
903                                            data->ioc_inlbuf1, 0);
904                 if (rc > 0)
905                         rc = 0;
906                 break;
907         case IOC_OSC_SET_ACTIVE:
908                 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
909                                               data->ioc_offset);
910                 break;
911         case OBD_IOC_PING_TARGET:
912                 rc = ptlrpc_obd_ping(obd);
913                 break;
914         default:
915                 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
916                        cmd, cfs_curproc_comm());
917                 rc = -ENOTTY;
918         }
919         cfs_module_put(THIS_MODULE);
920         return rc;
921 }
922
923 static int osp_obd_health_check(const struct lu_env *env,
924                                 struct obd_device *obd)
925 {
926         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
927
928         ENTRY;
929
930         /*
931          * 1.8/2.0 behaviour is that OST being connected once at least
932          * is considired "healthy". and one "healty" OST is enough to
933          * allow lustre clients to connect to MDS
934          */
935         LASSERT(d);
936         RETURN(!d->opd_imp_seen_connected);
937 }
938
939 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
940 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
941 static void osp_key_exit(const struct lu_context *ctx,
942                          struct lu_context_key *key, void *data)
943 {
944         struct osp_thread_info *info = data;
945
946         info->osi_attr.la_valid = 0;
947 }
948
949 struct lu_context_key osp_thread_key = {
950         .lct_tags = LCT_MD_THREAD,
951         .lct_init = osp_key_init,
952         .lct_fini = osp_key_fini,
953         .lct_exit = osp_key_exit
954 };
955
956 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
957 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
958
959 struct lu_context_key osp_txn_key = {
960         .lct_tags = LCT_OSP_THREAD,
961         .lct_init = osp_txn_key_init,
962         .lct_fini = osp_txn_key_fini
963 };
964 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
965
966 static struct lu_device_type_operations osp_device_type_ops = {
967         .ldto_init           = osp_type_init,
968         .ldto_fini           = osp_type_fini,
969
970         .ldto_start          = osp_type_start,
971         .ldto_stop           = osp_type_stop,
972
973         .ldto_device_alloc   = osp_device_alloc,
974         .ldto_device_free    = osp_device_free,
975
976         .ldto_device_fini    = osp_device_fini
977 };
978
979 static struct lu_device_type osp_device_type = {
980         .ldt_tags     = LU_DEVICE_DT,
981         .ldt_name     = LUSTRE_OSP_NAME,
982         .ldt_ops      = &osp_device_type_ops,
983         .ldt_ctx_tags = LCT_MD_THREAD
984 };
985
986 static struct obd_ops osp_obd_device_ops = {
987         .o_owner        = THIS_MODULE,
988         .o_add_conn     = client_import_add_conn,
989         .o_del_conn     = client_import_del_conn,
990         .o_reconnect    = osp_reconnect,
991         .o_connect      = osp_obd_connect,
992         .o_disconnect   = osp_obd_disconnect,
993         .o_health_check = osp_obd_health_check,
994         .o_import_event = osp_import_event,
995         .o_iocontrol    = osp_iocontrol,
996         .o_statfs       = osp_obd_statfs,
997 };
998
999 static int __init osp_mod_init(void)
1000 {
1001         struct lprocfs_static_vars       lvars;
1002         cfs_proc_dir_entry_t            *osc_proc_dir;
1003         int                              rc;
1004
1005         rc = lu_kmem_init(osp_caches);
1006         if (rc)
1007                 return rc;
1008
1009         lprocfs_osp_init_vars(&lvars);
1010
1011         rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
1012                                  LUSTRE_OSP_NAME, &osp_device_type);
1013
1014         /* create "osc" entry in procfs for compatibility purposes */
1015         if (rc != 0) {
1016                 lu_kmem_fini(osp_caches);
1017                 return rc;
1018         }
1019
1020         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1021         if (osc_proc_dir == NULL) {
1022                 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1023                                                 NULL);
1024                 if (IS_ERR(osc_proc_dir))
1025                         CERROR("osp: can't create compat entry \"osc\": %d\n",
1026                                (int) PTR_ERR(osc_proc_dir));
1027         }
1028         return rc;
1029 }
1030
1031 static void __exit osp_mod_exit(void)
1032 {
1033         lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1034
1035         class_unregister_type(LUSTRE_OSP_NAME);
1036         lu_kmem_fini(osp_caches);
1037 }
1038
1039 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1040 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1041 MODULE_LICENSE("GPL");
1042
1043 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);
1044