4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2011, 2012, Whamcloud, Inc.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
38 * Author: Alexey Zhuravlev <bzzz@whamcloud.com>
39 * Author: Mikhail Pershin <tappro@whamcloud.com>
42 #define DEBUG_SUBSYSTEM S_FILTER
44 #include "ofd_internal.h"
46 int ofd_record_write(const struct lu_env *env, struct ofd_device *ofd,
47 struct dt_object *dt, struct lu_buf *buf, loff_t *off)
56 th = dt_trans_create(env, ofd->ofd_osd);
60 rc = dt_declare_record_write(env, dt, buf->lb_len, *off, th);
62 rc = dt_trans_start_local(env, ofd->ofd_osd, th);
64 rc = dt_record_write(env, dt, buf, off, th);
66 dt_trans_stop(env, ofd->ofd_osd, th);
71 obd_id ofd_last_id(struct ofd_device *ofd, obd_seq group)
75 LASSERT(group <= ofd->ofd_max_group);
77 cfs_spin_lock(&ofd->ofd_objid_lock);
78 id = ofd->ofd_last_objids[group];
79 cfs_spin_unlock(&ofd->ofd_objid_lock);
84 void ofd_last_id_set(struct ofd_device *ofd, obd_id id, obd_seq group)
86 LASSERT(group <= ofd->ofd_max_group);
87 cfs_spin_lock(&ofd->ofd_objid_lock);
88 if (ofd->ofd_last_objids[group] < id)
89 ofd->ofd_last_objids[group] = id;
90 cfs_spin_unlock(&ofd->ofd_objid_lock);
93 int ofd_last_id_write(const struct lu_env *env, struct ofd_device *ofd,
96 struct ofd_thread_info *info = ofd_info(env);
102 info->fti_buf.lb_buf = &tmp;
103 info->fti_buf.lb_len = sizeof(tmp);
106 CDEBUG(D_INODE, "%s: write last_objid for group "LPU64": "LPU64"\n",
107 ofd_obd(ofd)->obd_name, group, ofd_last_id(ofd, group));
109 tmp = cpu_to_le64(ofd_last_id(ofd, group));
111 rc = ofd_record_write(env, ofd, ofd->ofd_lastid_obj[group],
112 &info->fti_buf, &info->fti_off);
116 int ofd_last_group_write(const struct lu_env *env, struct ofd_device *ofd)
118 struct ofd_thread_info *info = ofd_info(env);
124 info->fti_buf.lb_buf = &tmp;
125 info->fti_buf.lb_len = sizeof(tmp);
128 tmp = cpu_to_le32(ofd->ofd_max_group);
130 rc = ofd_record_write(env, ofd, ofd->ofd_last_group_file,
131 &info->fti_buf, &info->fti_off);
136 void ofd_group_fini(const struct lu_env *env, struct ofd_device *ofd,
139 LASSERT(ofd->ofd_lastid_obj[group]);
140 lu_object_put(env, &ofd->ofd_lastid_obj[group]->do_lu);
141 ofd->ofd_lastid_obj[group] = NULL;
144 int ofd_group_load(const struct lu_env *env, struct ofd_device *ofd, int group)
146 struct ofd_thread_info *info = ofd_info(env);
147 struct dt_object *dob;
153 /* if group is already initialized */
154 if (ofd->ofd_lastid_obj[group])
157 lu_local_obj_fid(&info->fti_fid, OFD_GROUP0_LAST_OID + group);
158 memset(&info->fti_attr, 0, sizeof(info->fti_attr));
159 info->fti_attr.la_valid = LA_MODE;
160 info->fti_attr.la_mode = S_IFREG | S_IRUGO | S_IWUSR;
161 info->fti_dof.dof_type = dt_mode_to_dft(S_IFREG);
163 /* create object tracking per-group last created
164 * id to be used by orphan recovery mechanism */
165 dob = dt_find_or_create(env, ofd->ofd_osd, &info->fti_fid,
166 &info->fti_dof, &info->fti_attr);
168 RETURN(PTR_ERR(dob));
170 ofd->ofd_lastid_obj[group] = dob;
171 cfs_mutex_init(&ofd->ofd_create_locks[group]);
173 rc = dt_attr_get(env, dob, &info->fti_attr, BYPASS_CAPA);
177 if (info->fti_attr.la_size == 0) {
178 /* object is just created, initialize last id */
179 ofd->ofd_last_objids[group] = OFD_INIT_OBJID;
180 ofd_last_id_set(ofd, OFD_INIT_OBJID, group);
181 ofd_last_id_write(env, ofd, group);
182 ofd_last_group_write(env, ofd);
183 } else if (info->fti_attr.la_size == sizeof(lastid)) {
185 info->fti_buf.lb_buf = &lastid;
186 info->fti_buf.lb_len = sizeof(lastid);
188 rc = dt_record_read(env, dob, &info->fti_buf, &info->fti_off);
190 CERROR("can't read last_id: %d\n", rc);
193 ofd->ofd_last_objids[group] = le64_to_cpu(lastid);
195 CERROR("corrupted size %Lu LAST_ID of group %u\n",
196 (unsigned long long)info->fti_attr.la_size, group);
202 ofd_group_fini(env, ofd, group);
206 /* ofd groups managements */
207 int ofd_groups_init(const struct lu_env *env, struct ofd_device *ofd)
209 struct ofd_thread_info *info = ofd_info(env);
210 unsigned long groups_size;
217 cfs_spin_lock_init(&ofd->ofd_objid_lock);
219 rc = dt_attr_get(env, ofd->ofd_last_group_file,
220 &info->fti_attr, BYPASS_CAPA);
224 groups_size = (unsigned long)info->fti_attr.la_size;
226 if (groups_size == sizeof(last_group)) {
228 info->fti_buf.lb_buf = &last_group;
229 info->fti_buf.lb_len = sizeof(last_group);
231 rc = dt_record_read(env, ofd->ofd_last_group_file,
232 &info->fti_buf, &info->fti_off);
234 CERROR("can't read LAST_GROUP: %d\n", rc);
238 ofd->ofd_max_group = le32_to_cpu(last_group);
239 LASSERT(ofd->ofd_max_group <= OFD_MAX_GROUPS);
240 } else if (groups_size == 0) {
241 ofd->ofd_max_group = 0;
243 CERROR("groups file is corrupted? size = %lu\n", groups_size);
244 GOTO(cleanup, rc = -EIO);
247 for (i = 0; i <= ofd->ofd_max_group; i++) {
248 rc = ofd_group_load(env, ofd, i);
250 CERROR("can't load group %d: %d\n", i, rc);
251 /* Clean all previously set groups */
253 ofd_group_fini(env, ofd, --i);
258 CWARN("%s: %u groups initialized\n",
259 ofd_obd(ofd)->obd_name, ofd->ofd_max_group + 1);
264 int ofd_clients_data_init(const struct lu_env *env, struct ofd_device *ofd,
267 struct obd_device *obd = ofd_obd(ofd);
268 struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd;
269 struct lsd_client_data *lcd = NULL;
270 struct filter_export_data *fed;
273 loff_t off = lsd->lsd_client_start;
275 CLASSERT(offsetof(struct lsd_client_data, lcd_padding) +
276 sizeof(lcd->lcd_padding) == LR_CLIENT_SIZE);
282 for (cl_idx = 0; off < fsize; cl_idx++) {
283 struct obd_export *exp;
286 /* Don't assume off is incremented properly by
287 * fsfilt_read_record(), in case sizeof(*lcd)
288 * isn't the same as fsd->lsd_client_size. */
289 off = lsd->lsd_client_start + cl_idx * lsd->lsd_client_size;
290 rc = lut_client_data_read(env, &ofd->ofd_lut, lcd, &off, cl_idx);
292 CERROR("error reading FILT %s idx %d off %llu: rc %d\n",
293 LAST_RCVD, cl_idx, off, rc);
295 break; /* read error shouldn't cause startup to fail */
298 if (lcd->lcd_uuid[0] == '\0') {
299 CDEBUG(D_INFO, "skipping zeroed client at offset %d\n",
304 last_rcvd = lcd->lcd_last_transno;
306 /* These exports are cleaned up by ofd_disconnect(), so they
307 * need to be set up like real exports as ofd_connect() does.
309 exp = class_new_export(obd, (struct obd_uuid *)lcd->lcd_uuid);
311 CDEBUG(D_HA, "RCVRNG CLIENT uuid: %s idx: %d lr: "LPU64
312 " srv lr: "LPU64"\n", lcd->lcd_uuid, cl_idx,
313 last_rcvd, lsd->lsd_last_transno);
316 if (PTR_ERR(exp) == -EALREADY) {
317 /* export already exists, zero out this one */
318 CERROR("Duplicate export %s!\n", lcd->lcd_uuid);
321 GOTO(err_out, rc = PTR_ERR(exp));
324 fed = &exp->exp_filter_data;
325 *fed->fed_ted.ted_lcd = *lcd;
327 ofd_export_stats_init(ofd, exp, NULL);
328 rc = lut_client_add(env, exp, cl_idx);
329 LASSERTF(rc == 0, "rc = %d\n", rc); /* can't fail existing */
330 /* VBR: set export last committed version */
331 exp->exp_last_committed = last_rcvd;
332 cfs_spin_lock(&exp->exp_lock);
333 exp->exp_connecting = 0;
334 exp->exp_in_recovery = 0;
335 cfs_spin_unlock(&exp->exp_lock);
336 obd->obd_max_recoverable_clients++;
337 class_export_put(exp);
339 /* Need to check last_rcvd even for duplicated exports. */
340 CDEBUG(D_OTHER, "client at idx %d has last_rcvd = "LPU64"\n",
343 cfs_spin_lock(&ofd->ofd_lut.lut_translock);
344 if (last_rcvd > lsd->lsd_last_transno)
345 lsd->lsd_last_transno = last_rcvd;
346 cfs_spin_unlock(&ofd->ofd_lut.lut_translock);
354 int ofd_server_data_init(const struct lu_env *env, struct ofd_device *ofd)
356 struct ofd_thread_info *info = ofd_info(env);
357 struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd;
358 struct obd_device *obd = ofd_obd(ofd);
359 unsigned long last_rcvd_size;
362 rc = dt_attr_get(env, ofd->ofd_lut.lut_last_rcvd, &info->fti_attr,
367 last_rcvd_size = (unsigned long)info->fti_attr.la_size;
369 /* ensure padding in the struct is the correct size */
370 CLASSERT (offsetof(struct lr_server_data, lsd_padding) +
371 sizeof(lsd->lsd_padding) == LR_SERVER_SIZE);
373 if (last_rcvd_size == 0) {
374 LCONSOLE_WARN("%s: new disk, initializing\n", obd->obd_name);
376 memcpy(lsd->lsd_uuid, obd->obd_uuid.uuid,
377 sizeof(lsd->lsd_uuid));
378 lsd->lsd_last_transno = 0;
379 lsd->lsd_mount_count = 0;
380 lsd->lsd_server_size = LR_SERVER_SIZE;
381 lsd->lsd_client_start = LR_CLIENT_START;
382 lsd->lsd_client_size = LR_CLIENT_SIZE;
383 lsd->lsd_subdir_count = FILTER_SUBDIR_COUNT;
384 lsd->lsd_feature_incompat = OBD_INCOMPAT_OST;
386 rc = lut_server_data_read(env, &ofd->ofd_lut);
388 CDEBUG(D_INODE,"OBD ofd: error reading %s: rc %d\n",
392 if (strcmp((char *)lsd->lsd_uuid,
393 (char *)obd->obd_uuid.uuid)) {
394 LCONSOLE_ERROR("Trying to start OBD %s using the wrong"
395 " disk %s. Were the /dev/ assignments "
397 obd->obd_uuid.uuid, lsd->lsd_uuid);
398 GOTO(err_fsd, rc = -EINVAL);
402 lsd->lsd_mount_count++;
403 obd->u.obt.obt_mount_count = lsd->lsd_mount_count;
404 obd->u.obt.obt_instance = (__u32)obd->u.obt.obt_mount_count;
405 ofd->ofd_subdir_count = lsd->lsd_subdir_count;
407 if (lsd->lsd_feature_incompat & ~OFD_INCOMPAT_SUPP) {
408 CERROR("%s: unsupported incompat filesystem feature(s) %x\n",
410 lsd->lsd_feature_incompat & ~OFD_INCOMPAT_SUPP);
411 GOTO(err_fsd, rc = -EINVAL);
413 if (lsd->lsd_feature_rocompat & ~OFD_ROCOMPAT_SUPP) {
414 CERROR("%s: unsupported read-only filesystem feature(s) %x\n",
416 lsd->lsd_feature_rocompat & ~OFD_ROCOMPAT_SUPP);
417 /* Do something like remount filesystem read-only */
418 GOTO(err_fsd, rc = -EINVAL);
421 CDEBUG(D_INODE, "%s: server last_transno : "LPU64"\n",
422 obd->obd_name, lsd->lsd_last_transno);
423 CDEBUG(D_INODE, "%s: server mount_count: "LPU64"\n",
424 obd->obd_name, lsd->lsd_mount_count);
425 CDEBUG(D_INODE, "%s: server data size: %u\n",
426 obd->obd_name, lsd->lsd_server_size);
427 CDEBUG(D_INODE, "%s: per-client data start: %u\n",
428 obd->obd_name, lsd->lsd_client_start);
429 CDEBUG(D_INODE, "%s: per-client data size: %u\n",
430 obd->obd_name, lsd->lsd_client_size);
431 CDEBUG(D_INODE, "%s: server subdir_count: %u\n",
432 obd->obd_name, lsd->lsd_subdir_count);
433 CDEBUG(D_INODE, "%s: last_rcvd clients: %lu\n", obd->obd_name,
434 last_rcvd_size <= lsd->lsd_client_start ? 0 :
435 (last_rcvd_size - lsd->lsd_client_start) /
436 lsd->lsd_client_size);
438 if (!obd->obd_replayable)
439 CWARN("%s: recovery support OFF\n", obd->obd_name);
441 rc = ofd_clients_data_init(env, ofd, last_rcvd_size);
443 cfs_spin_lock(&ofd->ofd_lut.lut_translock);
444 obd->obd_last_committed = lsd->lsd_last_transno;
445 cfs_spin_unlock(&ofd->ofd_lut.lut_translock);
447 /* save it, so mount count and last_transno is current */
448 rc = lut_server_data_update(env, &ofd->ofd_lut, 0);
455 class_disconnect_exports(obd);
459 int ofd_fs_setup(const struct lu_env *env, struct ofd_device *ofd,
460 struct obd_device *obd)
462 struct ofd_thread_info *info = ofd_info(env);
463 struct dt_object *fo;
468 if (OBD_FAIL_CHECK(OBD_FAIL_MDS_FS_SETUP))
471 rc = ofd_server_data_init(env, ofd);
475 lu_local_obj_fid(&info->fti_fid, OFD_HEALTH_CHECK_OID);
476 memset(&info->fti_attr, 0, sizeof(info->fti_attr));
477 info->fti_attr.la_valid = LA_MODE;
478 info->fti_attr.la_mode = S_IFREG | S_IRUGO | S_IWUSR;
479 info->fti_dof.dof_type = dt_mode_to_dft(S_IFREG);
481 fo = dt_find_or_create(env, ofd->ofd_osd, &info->fti_fid,
482 &info->fti_dof, &info->fti_attr);
484 GOTO(out, rc = PTR_ERR(fo));
486 ofd->ofd_health_check_file = fo;
488 lu_local_obj_fid(&info->fti_fid, OFD_LAST_GROUP_OID);
489 memset(&info->fti_attr, 0, sizeof(info->fti_attr));
490 info->fti_attr.la_valid = LA_MODE;
491 info->fti_attr.la_mode = S_IFREG | S_IRUGO | S_IWUSR;
492 info->fti_dof.dof_type = dt_mode_to_dft(S_IFREG);
494 fo = dt_find_or_create(env, ofd->ofd_osd, &info->fti_fid,
495 &info->fti_dof, &info->fti_attr);
497 GOTO(out_hc, rc = PTR_ERR(fo));
499 ofd->ofd_last_group_file = fo;
501 rc = ofd_groups_init(env, ofd);
507 lu_object_put(env, &ofd->ofd_last_group_file->do_lu);
509 lu_object_put(env, &ofd->ofd_health_check_file->do_lu);
514 void ofd_fs_cleanup(const struct lu_env *env, struct ofd_device *ofd)
520 ofd_info_init(env, NULL);
522 for (i = 0; i <= ofd->ofd_max_group; i++) {
523 if (ofd->ofd_lastid_obj[i]) {
524 ofd_last_id_write(env, ofd, i);
525 ofd_group_fini(env, ofd, i);
529 i = dt_sync(env, ofd->ofd_osd);
531 CERROR("can't sync: %d\n", i);
533 if (ofd->ofd_last_group_file) {
534 lu_object_put(env, &ofd->ofd_last_group_file->do_lu);
535 ofd->ofd_last_group_file = NULL;
538 if (ofd->ofd_health_check_file) {
539 lu_object_put(env, &ofd->ofd_health_check_file->do_lu);
540 ofd->ofd_health_check_file = NULL;