Whamcloud - gitweb
Revert "b=19325 adjust waiting extent locks during 1st enqueue"
[fs/lustre-release.git] / lustre / lov / lov_dev.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see
20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * Implementation of cl_device and cl_device_type for LOV layer.
37  *
38  *   Author: Nikita Danilov <nikita.danilov@sun.com>
39  */
40
41 #define DEBUG_SUBSYSTEM S_LOV
42
43 /* class_name2obd() */
44 #include <obd_class.h>
45
46 #include "lov_cl_internal.h"
47
48 cfs_mem_cache_t *lov_page_kmem;
49 cfs_mem_cache_t *lov_lock_kmem;
50 cfs_mem_cache_t *lov_object_kmem;
51 cfs_mem_cache_t *lov_thread_kmem;
52 cfs_mem_cache_t *lov_session_kmem;
53 cfs_mem_cache_t *lov_req_kmem;
54
55 cfs_mem_cache_t *lovsub_page_kmem;
56 cfs_mem_cache_t *lovsub_lock_kmem;
57 cfs_mem_cache_t *lovsub_object_kmem;
58 cfs_mem_cache_t *lovsub_req_kmem;
59
60 cfs_mem_cache_t *lov_lock_link_kmem;
61
62 /** Lock class of lov_device::ld_mutex. */
63 cfs_lock_class_key_t cl_lov_device_mutex_class;
64
65 struct lu_kmem_descr lov_caches[] = {
66         {
67                 .ckd_cache = &lov_page_kmem,
68                 .ckd_name  = "lov_page_kmem",
69                 .ckd_size  = sizeof (struct lov_page)
70         },
71         {
72                 .ckd_cache = &lov_lock_kmem,
73                 .ckd_name  = "lov_lock_kmem",
74                 .ckd_size  = sizeof (struct lov_lock)
75         },
76         {
77                 .ckd_cache = &lov_object_kmem,
78                 .ckd_name  = "lov_object_kmem",
79                 .ckd_size  = sizeof (struct lov_object)
80         },
81         {
82                 .ckd_cache = &lov_thread_kmem,
83                 .ckd_name  = "lov_thread_kmem",
84                 .ckd_size  = sizeof (struct lov_thread_info)
85         },
86         {
87                 .ckd_cache = &lov_session_kmem,
88                 .ckd_name  = "lov_session_kmem",
89                 .ckd_size  = sizeof (struct lov_session)
90         },
91         {
92                 .ckd_cache = &lov_req_kmem,
93                 .ckd_name  = "lov_req_kmem",
94                 .ckd_size  = sizeof (struct lov_req)
95         },
96         {
97                 .ckd_cache = &lovsub_page_kmem,
98                 .ckd_name  = "lovsub_page_kmem",
99                 .ckd_size  = sizeof (struct lovsub_page)
100         },
101         {
102                 .ckd_cache = &lovsub_lock_kmem,
103                 .ckd_name  = "lovsub_lock_kmem",
104                 .ckd_size  = sizeof (struct lovsub_lock)
105         },
106         {
107                 .ckd_cache = &lovsub_object_kmem,
108                 .ckd_name  = "lovsub_object_kmem",
109                 .ckd_size  = sizeof (struct lovsub_object)
110         },
111         {
112                 .ckd_cache = &lovsub_req_kmem,
113                 .ckd_name  = "lovsub_req_kmem",
114                 .ckd_size  = sizeof (struct lovsub_req)
115         },
116         {
117                 .ckd_cache = &lov_lock_link_kmem,
118                 .ckd_name  = "lov_lock_link_kmem",
119                 .ckd_size  = sizeof (struct lov_lock_link)
120         },
121         {
122                 .ckd_cache = NULL
123         }
124 };
125
126 /*****************************************************************************
127  *
128  * Lov transfer operations.
129  *
130  */
131
132 static void lov_req_completion(const struct lu_env *env,
133                                const struct cl_req_slice *slice, int ioret)
134 {
135         struct lov_req *lr;
136
137         ENTRY;
138         lr = cl2lov_req(slice);
139         OBD_SLAB_FREE_PTR(lr, lov_req_kmem);
140         EXIT;
141 }
142
143 static const struct cl_req_operations lov_req_ops = {
144         .cro_completion = lov_req_completion
145 };
146
147 /*****************************************************************************
148  *
149  * Lov device and device type functions.
150  *
151  */
152
153 static void *lov_key_init(const struct lu_context *ctx,
154                           struct lu_context_key *key)
155 {
156         struct lov_thread_info *info;
157
158         OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, CFS_ALLOC_IO);
159         if (info != NULL)
160                 CFS_INIT_LIST_HEAD(&info->lti_closure.clc_list);
161         else
162                 info = ERR_PTR(-ENOMEM);
163         return info;
164 }
165
166 static void lov_key_fini(const struct lu_context *ctx,
167                          struct lu_context_key *key, void *data)
168 {
169         struct lov_thread_info *info = data;
170         LINVRNT(cfs_list_empty(&info->lti_closure.clc_list));
171         OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
172 }
173
174 struct lu_context_key lov_key = {
175         .lct_tags = LCT_CL_THREAD,
176         .lct_init = lov_key_init,
177         .lct_fini = lov_key_fini
178 };
179
180 static void *lov_session_key_init(const struct lu_context *ctx,
181                                   struct lu_context_key *key)
182 {
183         struct lov_session *info;
184
185         OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, CFS_ALLOC_IO);
186         if (info == NULL)
187                 info = ERR_PTR(-ENOMEM);
188         return info;
189 }
190
191 static void lov_session_key_fini(const struct lu_context *ctx,
192                                  struct lu_context_key *key, void *data)
193 {
194         struct lov_session *info = data;
195         OBD_SLAB_FREE_PTR(info, lov_session_kmem);
196 }
197
198 struct lu_context_key lov_session_key = {
199         .lct_tags = LCT_SESSION,
200         .lct_init = lov_session_key_init,
201         .lct_fini = lov_session_key_fini
202 };
203
204 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
205 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
206
207 static struct lu_device *lov_device_fini(const struct lu_env *env,
208                                          struct lu_device *d)
209 {
210         int i;
211         struct lov_device *ld = lu2lov_dev(d);
212
213         LASSERT(ld->ld_lov != NULL);
214         if (ld->ld_target == NULL)
215                 RETURN(NULL);
216
217         lov_foreach_target(ld, i) {
218                 struct lovsub_device *lsd;
219
220                 lsd = ld->ld_target[i];
221                 if (lsd != NULL) {
222                         cl_stack_fini(env, lovsub2cl_dev(lsd));
223                         ld->ld_target[i] = NULL;
224                 }
225         }
226         RETURN(NULL);
227 }
228
229 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
230                            const char *name, struct lu_device *next)
231 {
232         struct lov_device *ld = lu2lov_dev(d);
233         int i;
234         int rc = 0;
235
236         LASSERT(d->ld_site != NULL);
237         if (ld->ld_target == NULL)
238                 RETURN(rc);
239
240         lov_foreach_target(ld, i) {
241                 struct lovsub_device *lsd;
242                 struct cl_device     *cl;
243                 struct lov_tgt_desc  *desc;
244
245                 desc = ld->ld_lov->lov_tgts[i];
246                 if (desc == NULL)
247                         continue;
248
249                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
250                                    desc->ltd_obd->obd_lu_dev);
251                 if (IS_ERR(cl)) {
252                         rc = PTR_ERR(cl);
253                         break;
254                 }
255                 lsd = cl2lovsub_dev(cl);
256                 lsd->acid_idx = i;
257                 lsd->acid_super = ld;
258                 ld->ld_target[i] = lsd;
259         }
260
261         if (rc)
262                 lov_device_fini(env, d);
263         else
264                 ld->ld_flags |= LOV_DEV_INITIALIZED;
265
266         RETURN(rc);
267 }
268
269 static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
270                         struct cl_req *req)
271 {
272         struct lov_req *lr;
273         int result;
274
275         ENTRY;
276         OBD_SLAB_ALLOC_PTR_GFP(lr, lov_req_kmem, CFS_ALLOC_IO);
277         if (lr != NULL) {
278                 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
279                 result = 0;
280         } else
281                 result = -ENOMEM;
282         RETURN(result);
283 }
284
285 static const struct cl_device_operations lov_cl_ops = {
286         .cdo_req_init = lov_req_init
287 };
288
289 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
290 {
291         int i;
292
293         for (i = 0; i < nr; ++i) {
294                 struct lov_device_emerg *em;
295
296                 em = emrg[i];
297                 if (em != NULL) {
298                         LASSERT(em->emrg_page_list.pl_nr == 0);
299                         if (em->emrg_env != NULL)
300                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
301                         OBD_FREE_PTR(em);
302                 }
303         }
304         OBD_FREE(emrg, nr * sizeof emrg[0]);
305 }
306
307 static struct lu_device *lov_device_free(const struct lu_env *env,
308                                          struct lu_device *d)
309 {
310         struct lov_device *ld = lu2lov_dev(d);
311         const int          nr = ld->ld_target_nr;
312
313         cl_device_fini(lu2cl_dev(d));
314         if (ld->ld_target != NULL)
315                 OBD_FREE(ld->ld_target, nr * sizeof ld->ld_target[0]);
316         if (ld->ld_emrg != NULL)
317                 lov_emerg_free(ld->ld_emrg, nr);
318         OBD_FREE_PTR(ld);
319         return NULL;
320 }
321
322 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
323                               __u32 index)
324 {
325         struct lov_device *ld = lu2lov_dev(dev);
326         ENTRY;
327
328         if (ld->ld_target[index] != NULL) {
329                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
330                 ld->ld_target[index] = NULL;
331         }
332         EXIT;
333 }
334
335 static struct lov_device_emerg **lov_emerg_alloc(int nr)
336 {
337         struct lov_device_emerg **emerg;
338         int i;
339         int result;
340
341         OBD_ALLOC(emerg, nr * sizeof emerg[0]);
342         if (emerg == NULL)
343                 return ERR_PTR(-ENOMEM);
344         for (result = i = 0; i < nr && result == 0; i++) {
345                 struct lov_device_emerg *em;
346
347                 OBD_ALLOC_PTR(em);
348                 if (em != NULL) {
349                         emerg[i] = em;
350                         cl_page_list_init(&em->emrg_page_list);
351                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
352                                                     LCT_REMEMBER|LCT_NOREF);
353                         if (!IS_ERR(em->emrg_env))
354                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
355                         else {
356                                 result = PTR_ERR(em->emrg_env);
357                                 em->emrg_env = NULL;
358                         }
359                 } else
360                         result = -ENOMEM;
361         }
362         if (result != 0) {
363                 lov_emerg_free(emerg, nr);
364                 emerg = ERR_PTR(result);
365         }
366         return emerg;
367 }
368
369 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
370 {
371         int   result;
372         __u32 tgt_size;
373         __u32 sub_size;
374
375         ENTRY;
376         result = 0;
377         tgt_size = dev->ld_lov->lov_tgt_size;
378         sub_size = dev->ld_target_nr;
379         if (sub_size < tgt_size) {
380                 struct lovsub_device    **newd;
381                 struct lov_device_emerg **emerg;
382                 const size_t              sz   = sizeof newd[0];
383
384                 emerg = lov_emerg_alloc(tgt_size);
385                 if (IS_ERR(emerg))
386                         RETURN(PTR_ERR(emerg));
387
388                 OBD_ALLOC(newd, tgt_size * sz);
389                 if (newd != NULL) {
390                         cfs_mutex_lock(&dev->ld_mutex);
391                         if (sub_size > 0) {
392                                 memcpy(newd, dev->ld_target, sub_size * sz);
393                                 OBD_FREE(dev->ld_target, sub_size * sz);
394                         }
395                         dev->ld_target    = newd;
396                         dev->ld_target_nr = tgt_size;
397
398                         if (dev->ld_emrg != NULL)
399                                 lov_emerg_free(dev->ld_emrg, sub_size);
400                         dev->ld_emrg = emerg;
401                         cfs_mutex_unlock(&dev->ld_mutex);
402                 } else {
403                         lov_emerg_free(emerg, tgt_size);
404                         result = -ENOMEM;
405                 }
406         }
407         RETURN(result);
408 }
409
410 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
411                              __u32 index)
412 {
413         struct obd_device    *obd = dev->ld_obd;
414         struct lov_device    *ld  = lu2lov_dev(dev);
415         struct lov_tgt_desc  *tgt;
416         struct lovsub_device *lsd;
417         struct cl_device     *cl;
418         int rc;
419         ENTRY;
420
421         obd_getref(obd);
422
423         tgt = obd->u.lov.lov_tgts[index];
424         LASSERT(tgt != NULL);
425         LASSERT(tgt->ltd_obd != NULL);
426
427         if (!tgt->ltd_obd->obd_set_up) {
428                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
429                 RETURN(-EINVAL);
430         }
431
432         rc = lov_expand_targets(env, ld);
433         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
434                 LASSERT(dev->ld_site != NULL);
435
436                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
437                                    tgt->ltd_obd->obd_lu_dev);
438                 if (!IS_ERR(cl)) {
439                         lsd = cl2lovsub_dev(cl);
440                         lsd->acid_idx = index;
441                         lsd->acid_super = ld;
442                         ld->ld_target[index] = lsd;
443                 } else {
444                         CERROR("add failed (%d), deleting %s\n", rc,
445                                obd_uuid2str(&tgt->ltd_uuid));
446                         lov_cl_del_target(env, dev, index);
447                         rc = PTR_ERR(cl);
448                 }
449         }
450         obd_putref(obd);
451         RETURN(rc);
452 }
453
454 static int lov_process_config(const struct lu_env *env,
455                               struct lu_device *d, struct lustre_cfg *cfg)
456 {
457         struct obd_device *obd = d->ld_obd;
458         int cmd;
459         int rc;
460         int gen;
461         __u32 index;
462
463         obd_getref(obd);
464
465         cmd = cfg->lcfg_command;
466         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
467         if (rc == 0) {
468                 switch(cmd) {
469                 case LCFG_LOV_ADD_OBD:
470                 case LCFG_LOV_ADD_INA:
471                         rc = lov_cl_add_target(env, d, index);
472                         if (rc != 0)
473                                 lov_del_target(d->ld_obd, index, 0, 0);
474                         break;
475                 case LCFG_LOV_DEL_OBD:
476                         lov_cl_del_target(env, d, index);
477                         break;
478                 }
479         }
480         obd_putref(obd);
481         RETURN(rc);
482 }
483
484 static const struct lu_device_operations lov_lu_ops = {
485         .ldo_object_alloc      = lov_object_alloc,
486         .ldo_process_config    = lov_process_config,
487 };
488
489 static struct lu_device *lov_device_alloc(const struct lu_env *env,
490                                           struct lu_device_type *t,
491                                           struct lustre_cfg *cfg)
492 {
493         struct lu_device *d;
494         struct lov_device *ld;
495         struct obd_device *obd;
496         int rc;
497
498         OBD_ALLOC_PTR(ld);
499         if (ld == NULL)
500                 RETURN(ERR_PTR(-ENOMEM));
501
502         cl_device_init(&ld->ld_cl, t);
503         d = lov2lu_dev(ld);
504         d->ld_ops        = &lov_lu_ops;
505         ld->ld_cl.cd_ops = &lov_cl_ops;
506
507         cfs_mutex_init(&ld->ld_mutex);
508         cfs_lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
509
510         /* setup the LOV OBD */
511         obd = class_name2obd(lustre_cfg_string(cfg, 0));
512         LASSERT(obd != NULL);
513         rc = lov_setup(obd, cfg);
514         if (rc) {
515                 lov_device_free(env, d);
516                 RETURN(ERR_PTR(rc));
517         }
518
519         ld->ld_lov = &obd->u.lov;
520         RETURN(d);
521 }
522
523 static const struct lu_device_type_operations lov_device_type_ops = {
524         .ldto_init = lov_type_init,
525         .ldto_fini = lov_type_fini,
526
527         .ldto_start = lov_type_start,
528         .ldto_stop  = lov_type_stop,
529
530         .ldto_device_alloc = lov_device_alloc,
531         .ldto_device_free  = lov_device_free,
532
533         .ldto_device_init    = lov_device_init,
534         .ldto_device_fini    = lov_device_fini
535 };
536
537 struct lu_device_type lov_device_type = {
538         .ldt_tags     = LU_DEVICE_CL,
539         .ldt_name     = LUSTRE_LOV_NAME,
540         .ldt_ops      = &lov_device_type_ops,
541         .ldt_ctx_tags = LCT_CL_THREAD
542 };
543 EXPORT_SYMBOL(lov_device_type);
544
545 /** @} lov */