4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2011, 2012, Whamcloud, Inc.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
36 * lustre/ofd/ofd_obd.c
38 * Author: Andreas Dilger <adilger@whamcloud.com>
39 * Author: Alex Zhuravlev <bzzz@whamcloud.com>
40 * Author: Mike Pershin <tappro@whamcloud.com>
43 #define DEBUG_SUBSYSTEM S_FILTER
45 #include "ofd_internal.h"
46 #include <obd_cksum.h>
48 static int ofd_export_stats_init(struct ofd_device *ofd,
49 struct obd_export *exp, void *client_nid)
51 struct obd_device *obd = ofd_obd(ofd);
52 struct nid_stat *stats;
58 if (obd_uuid_equals(&exp->exp_client_uuid, &obd->obd_uuid))
59 /* Self-export gets no proc entry */
62 rc = lprocfs_exp_setup(exp, client_nid, &newnid);
64 /* Mask error for already created
74 stats = exp->exp_nid_stats;
75 LASSERT(stats != NULL);
77 OBD_ALLOC(stats->nid_brw_stats, sizeof(struct brw_stats));
78 if (stats->nid_brw_stats == NULL)
79 GOTO(clean, rc = -ENOMEM);
81 for (i = 0; i < BRW_LAST; i++)
82 cfs_spin_lock_init(&stats->nid_brw_stats->hist[i].oh_lock);
84 rc = lprocfs_seq_create(stats->nid_proc, "brw_stats", 0644,
85 &ofd_per_nid_stats_fops, stats);
87 CWARN("Error adding the brw_stats file\n");
89 num_stats = (sizeof(*obd->obd_type->typ_dt_ops) / sizeof(void *)) +
92 stats->nid_stats = lprocfs_alloc_stats(num_stats,
93 LPROCFS_STATS_FLAG_NOPERCPU);
94 if (stats->nid_stats == NULL)
97 lprocfs_init_ops_stats(LPROC_OFD_LAST, stats->nid_stats);
98 lprocfs_counter_init(stats->nid_stats, LPROC_OFD_READ_BYTES,
99 LPROCFS_CNTR_AVGMINMAX, "read_bytes", "bytes");
100 lprocfs_counter_init(stats->nid_stats, LPROC_OFD_WRITE_BYTES,
101 LPROCFS_CNTR_AVGMINMAX, "write_bytes", "bytes");
103 rc = lprocfs_register_stats(stats->nid_proc, "stats",
108 rc = lprocfs_nid_ldlm_stats_init(stats);
110 lprocfs_free_stats(&stats->nid_stats);
119 static int ofd_parse_connect_data(const struct lu_env *env,
120 struct obd_export *exp,
121 struct obd_connect_data *data)
123 struct ofd_device *ofd = ofd_exp(exp);
124 struct filter_export_data *fed = &exp->exp_filter_data;
129 CDEBUG(D_RPCTRACE, "%s: cli %s/%p ocd_connect_flags: "LPX64
130 " ocd_version: %x ocd_grant: %d ocd_index: %u\n",
131 exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp,
132 data->ocd_connect_flags, data->ocd_version,
133 data->ocd_grant, data->ocd_index);
135 if (fed->fed_group != 0 && fed->fed_group != data->ocd_group) {
136 CWARN("!!! This export (nid %s) used object group %d "
137 "earlier; now it's trying to use group %d! This could "
138 "be a bug in the MDS. Please report to "
139 "http://bugs.whamcloud.com/\n",
140 obd_export_nid2str(exp), fed->fed_group,
144 fed->fed_group = data->ocd_group;
146 data->ocd_connect_flags &= OST_CONNECT_SUPPORTED;
147 exp->exp_connect_flags = data->ocd_connect_flags;
148 data->ocd_version = LUSTRE_VERSION_CODE;
150 /* Kindly make sure the SKIP_ORPHAN flag is from MDS. */
151 if (data->ocd_connect_flags & OBD_CONNECT_MDS)
152 CDEBUG(D_HA, "%s: Received MDS connection for group %u\n",
153 exp->exp_obd->obd_name, data->ocd_group);
154 else if (data->ocd_connect_flags & OBD_CONNECT_SKIP_ORPHAN)
157 if (data->ocd_connect_flags & OBD_CONNECT_INDEX) {
158 struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd;
159 int index = lsd->lsd_ost_index;
161 if (!(lsd->lsd_feature_compat & OBD_COMPAT_OST)) {
162 /* this will only happen on the first connect */
163 lsd->lsd_ost_index = data->ocd_index;
164 lsd->lsd_feature_compat |= OBD_COMPAT_OST;
165 /* sync is not needed here as lut_client_add will
166 * set exp_need_sync flag */
167 lut_server_data_update(env, &ofd->ofd_lut, 0);
168 } else if (index != data->ocd_index) {
169 LCONSOLE_ERROR_MSG(0x136, "Connection from %s to index"
170 " %u doesn't match actual OST index"
171 " %u in last_rcvd file, bad "
173 obd_export_nid2str(exp), index,
179 if (OBD_FAIL_CHECK(OBD_FAIL_OST_BRW_SIZE)) {
180 data->ocd_brw_size = 65536;
181 } else if (data->ocd_connect_flags & OBD_CONNECT_BRW_SIZE) {
182 data->ocd_brw_size = min(data->ocd_brw_size,
183 (__u32)(PTLRPC_MAX_BRW_PAGES << CFS_PAGE_SHIFT));
184 if (data->ocd_brw_size == 0) {
185 CERROR("%s: cli %s/%p ocd_connect_flags: "LPX64
186 " ocd_version: %x ocd_grant: %d ocd_index: %u "
187 "ocd_brw_size is unexpectedly zero, "
188 "network data corruption?"
189 "Refusing connection of this client\n",
190 exp->exp_obd->obd_name,
191 exp->exp_client_uuid.uuid,
192 exp, data->ocd_connect_flags, data->ocd_version,
193 data->ocd_grant, data->ocd_index);
198 if (data->ocd_connect_flags & OBD_CONNECT_CKSUM) {
199 __u32 cksum_types = data->ocd_cksum_types;
201 /* The client set in ocd_cksum_types the checksum types it
202 * supports. We have to mask off the algorithms that we don't
204 data->ocd_cksum_types &= cksum_types_supported();
206 if (unlikely(data->ocd_cksum_types == 0)) {
207 CERROR("%s: Connect with checksum support but no "
208 "ocd_cksum_types is set\n",
209 exp->exp_obd->obd_name);
213 CDEBUG(D_RPCTRACE, "%s: cli %s supports cksum type %x, return "
214 "%x\n", exp->exp_obd->obd_name, obd_export_nid2str(exp),
215 cksum_types, data->ocd_cksum_types);
217 /* This client does not support OBD_CONNECT_CKSUM
218 * fall back to CRC32 */
219 CDEBUG(D_RPCTRACE, "%s: cli %s does not support "
220 "OBD_CONNECT_CKSUM, CRC32 will be used\n",
221 exp->exp_obd->obd_name, obd_export_nid2str(exp));
224 if (data->ocd_connect_flags & OBD_CONNECT_MAXBYTES)
225 data->ocd_maxbytes = ofd->ofd_dt_conf.ddp_maxbytes;
230 static int ofd_obd_reconnect(const struct lu_env *env, struct obd_export *exp,
231 struct obd_device *obd, struct obd_uuid *cluuid,
232 struct obd_connect_data *data, void *localdata)
234 struct ofd_device *ofd = ofd_dev(obd->obd_lu_dev);
239 if (exp == NULL || obd == NULL || cluuid == NULL)
242 rc = lu_env_refill((struct lu_env *)env);
244 CERROR("Failure to refill session: '%d'\n", rc);
248 ofd_info_init(env, exp);
249 rc = ofd_parse_connect_data(env, exp, data);
251 ofd_export_stats_init(ofd, exp, localdata);
256 static int ofd_obd_connect(const struct lu_env *env, struct obd_export **_exp,
257 struct obd_device *obd, struct obd_uuid *cluuid,
258 struct obd_connect_data *data, void *localdata)
260 struct obd_export *exp;
261 struct ofd_device *ofd;
262 struct lustre_handle conn = { 0 };
267 if (_exp == NULL || obd == NULL || cluuid == NULL)
270 ofd = ofd_dev(obd->obd_lu_dev);
272 rc = class_connect(&conn, obd, cluuid);
276 exp = class_conn2export(&conn);
277 LASSERT(exp != NULL);
279 rc = lu_env_refill((struct lu_env *)env);
281 CERROR("Failure to refill session: '%d'\n", rc);
285 ofd_info_init(env, exp);
287 rc = ofd_parse_connect_data(env, exp, data);
291 group = data->ocd_group;
292 if (obd->obd_replayable) {
293 struct tg_export_data *ted = &exp->exp_target_data;
295 memcpy(ted->ted_lcd->lcd_uuid, cluuid,
296 sizeof(ted->ted_lcd->lcd_uuid));
297 rc = lut_client_new(env, exp);
300 ofd_export_stats_init(ofd, exp, localdata);
306 if (group > ofd->ofd_max_group) {
307 ofd->ofd_max_group = group;
308 rc = ofd_group_load(env, ofd, group);
312 class_disconnect(exp);
320 static int ofd_obd_disconnect(struct obd_export *exp)
328 class_export_get(exp);
330 rc = server_disconnect_export(exp);
332 rc = lu_env_init(&env, LCT_DT_THREAD);
336 /* Do not erase record for recoverable client. */
337 if (exp->exp_obd->obd_replayable &&
338 (!exp->exp_obd->obd_fail || exp->exp_failed))
339 lut_client_del(&env, exp);
342 class_export_put(exp);
346 static int ofd_init_export(struct obd_export *exp)
350 cfs_spin_lock_init(&exp->exp_filter_data.fed_lock);
351 CFS_INIT_LIST_HEAD(&exp->exp_filter_data.fed_mod_list);
352 cfs_spin_lock(&exp->exp_lock);
353 exp->exp_connecting = 1;
354 cfs_spin_unlock(&exp->exp_lock);
356 /* self-export doesn't need client data and ldlm initialization */
357 if (unlikely(obd_uuid_equals(&exp->exp_obd->obd_uuid,
358 &exp->exp_client_uuid)))
361 rc = lut_client_alloc(exp);
363 ldlm_init_export(exp);
365 CERROR("%s: Can't initialize export: rc %d\n",
366 exp->exp_obd->obd_name, rc);
370 static int ofd_destroy_export(struct obd_export *exp)
372 if (exp->exp_filter_data.fed_pending)
373 CERROR("%s: cli %s/%p has %lu pending on destroyed export"
374 "\n", exp->exp_obd->obd_name, exp->exp_client_uuid.uuid,
375 exp, exp->exp_filter_data.fed_pending);
377 target_destroy_export(exp);
379 if (unlikely(obd_uuid_equals(&exp->exp_obd->obd_uuid,
380 &exp->exp_client_uuid)))
383 ldlm_destroy_export(exp);
384 lut_client_free(exp);
386 LASSERT(cfs_list_empty(&exp->exp_filter_data.fed_mod_list));
390 int ofd_obd_postrecov(struct obd_device *obd)
393 struct lu_device *ldev = obd->obd_lu_dev;
398 rc = lu_env_init(&env, LCT_DT_THREAD);
401 ofd_info_init(&env, obd->obd_self_export);
403 rc = ldev->ld_ops->ldo_recovery_complete(&env, ldev);
408 static int ofd_set_mds_conn(struct obd_export *exp, void *val)
414 LCONSOLE_WARN("%s: received MDS connection from %s\n",
415 exp->exp_obd->obd_name, obd_export_nid2str(exp));
419 static int ofd_set_info_async(const struct lu_env *env, struct obd_export *exp,
420 __u32 keylen, void *key, __u32 vallen, void *val,
421 struct ptlrpc_request_set *set)
423 struct ofd_device *ofd = ofd_exp(exp);
428 if (exp->exp_obd == NULL) {
429 CDEBUG(D_IOCTL, "invalid export %p\n", exp);
433 if (KEY_IS(KEY_CAPA_KEY)) {
434 rc = ofd_update_capa_key(ofd, val);
436 CERROR("ofd update capability key failed: %d\n", rc);
437 } else if (KEY_IS(KEY_MDS_CONN)) {
438 rc = ofd_set_mds_conn(exp, val);
440 CERROR("%s: Unsupported key %s\n",
441 exp->exp_obd->obd_name, (char*)key);
448 static int ofd_get_info(const struct lu_env *env, struct obd_export *exp,
449 __u32 keylen, void *key, __u32 *vallen, void *val,
450 struct lov_stripe_md *lsm)
452 struct ofd_device *ofd = ofd_exp(exp);
457 if (exp->exp_obd == NULL) {
458 CDEBUG(D_IOCTL, "invalid client export %p\n", exp);
462 if (KEY_IS(KEY_BLOCKSIZE)) {
463 __u32 *blocksize = val;
465 if (*vallen < sizeof(*blocksize))
467 *blocksize = 1 << ofd->ofd_dt_conf.ddp_block_shift;
469 *vallen = sizeof(*blocksize);
470 } else if (KEY_IS(KEY_BLOCKSIZE_BITS)) {
471 __u32 *blocksize_bits = val;
472 if (blocksize_bits) {
473 if (*vallen < sizeof(*blocksize_bits))
475 *blocksize_bits = ofd->ofd_dt_conf.ddp_block_shift;
477 *vallen = sizeof(*blocksize_bits);
478 } else if (KEY_IS(KEY_LAST_ID)) {
479 obd_id *last_id = val;
481 if (*vallen < sizeof(*last_id))
483 *last_id = ofd_last_id(ofd,
484 exp->exp_filter_data.fed_group);
486 *vallen = sizeof(*last_id);
488 CERROR("Not supported key %s\n", (char*)key);
495 /** helper function for statfs, also used by grant code */
496 int ofd_statfs_internal(const struct lu_env *env, struct ofd_device *ofd,
497 struct obd_statfs *osfs, __u64 max_age, int *from_cache)
501 rc = dt_statfs(env, ofd->ofd_osd, osfs);
508 static int ofd_statfs(const struct lu_env *env, struct obd_export *exp,
509 struct obd_statfs *osfs, __u64 max_age, __u32 flags)
511 struct ofd_device *ofd = ofd_dev(exp->exp_obd->obd_lu_dev);
516 rc = ofd_statfs_internal(env, ofd, osfs, max_age, NULL);
520 if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOSPC,
521 ofd->ofd_lut.lut_lsd.lsd_ost_index))
522 osfs->os_bfree = osfs->os_bavail = 2;
524 if (OBD_FAIL_CHECK_VALUE(OBD_FAIL_OST_ENOINO,
525 ofd->ofd_lut.lut_lsd.lsd_ost_index))
528 /* OS_STATE_READONLY can be set by OSD already */
529 if (ofd->ofd_raid_degraded)
530 osfs->os_state |= OS_STATE_DEGRADED;
536 static int ofd_sync(const struct lu_env *env, struct obd_export *exp,
537 struct obd_info *oinfo, obd_size start, obd_size end,
538 struct ptlrpc_request_set *set)
540 struct ofd_device *ofd = ofd_exp(exp);
545 /* if no objid is specified, it means "sync whole filesystem" */
546 if (oinfo->oi_oa == NULL || !(oinfo->oi_oa->o_valid & OBD_MD_FLID)) {
547 rc = dt_sync(env, ofd->ofd_osd);
556 int ofd_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
557 void *karg, void *uarg)
560 struct ofd_device *ofd = ofd_exp(exp);
561 struct obd_device *obd = ofd_obd(ofd);
566 CDEBUG(D_IOCTL, "handling ioctl cmd %#x\n", cmd);
567 rc = lu_env_init(&env, LCT_LOCAL);
572 case OBD_IOC_ABORT_RECOVERY:
573 CERROR("aborting recovery for device %s\n", obd->obd_name);
574 target_stop_recovery_thread(obd);
577 CDEBUG(D_RPCTRACE, "syncing ost %s\n", obd->obd_name);
578 rc = dt_sync(&env, ofd->ofd_osd);
580 case OBD_IOC_SET_READONLY:
581 rc = dt_sync(&env, ofd->ofd_osd);
583 rc = dt_ro(&env, ofd->ofd_osd);
586 CERROR("Not supported cmd = %d for device %s\n",
595 static int ofd_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
602 case OBD_CLEANUP_EARLY:
604 case OBD_CLEANUP_EXPORTS:
605 target_cleanup_recovery(obd);
611 static int ofd_ping(const struct lu_env *env, struct obd_export *exp)
616 static int ofd_health_check(const struct lu_env *env, struct obd_device *obd)
618 struct ofd_device *ofd = ofd_dev(obd->obd_lu_dev);
619 struct ofd_thread_info *info;
620 #ifdef USE_HEALTH_CHECK_WRITE
625 info = ofd_info_init(env, NULL);
626 rc = dt_statfs(env, ofd->ofd_osd, &info->fti_u.osfs);
630 if (info->fti_u.osfs.os_state == OS_STATE_READONLY)
631 GOTO(out, rc = -EROFS);
633 #ifdef USE_HEALTH_CHECK_WRITE
634 OBD_ALLOC(info->fti_buf.lb_buf, CFS_PAGE_SIZE);
635 if (info->fti_buf.lb_buf == NULL)
636 GOTO(out, rc = -ENOMEM);
638 info->fti_buf.lb_len = CFS_PAGE_SIZE;
641 th = dt_trans_create(env, ofd->ofd_osd);
643 GOTO(out, rc = PTR_ERR(th));
645 rc = dt_declare_record_write(env, ofd->ofd_health_check_file,
646 info->fti_buf.lb_len, info->fti_off, th);
648 th->th_sync = 1; /* sync IO is needed */
649 rc = dt_trans_start_local(env, ofd->ofd_osd, th);
651 rc = dt_record_write(env, ofd->ofd_health_check_file,
652 &info->fti_buf, &info->fti_off,
655 dt_trans_stop(env, ofd->ofd_osd, th);
657 OBD_FREE(info->fti_buf.lb_buf, CFS_PAGE_SIZE);
659 CDEBUG(D_INFO, "write 1 page synchronously for checking io rc %d\n",rc);
665 static int ofd_obd_notify(struct obd_device *obd, struct obd_device *unused,
666 enum obd_notify_event ev, void *data)
669 case OBD_NOTIFY_CONFIG:
670 LASSERT(obd->obd_no_conn);
671 cfs_spin_lock(&obd->obd_dev_lock);
672 obd->obd_no_conn = 0;
673 cfs_spin_unlock(&obd->obd_dev_lock);
676 CDEBUG(D_INFO, "%s: Unhandled notification %#x\n",
682 struct obd_ops ofd_obd_ops = {
683 .o_owner = THIS_MODULE,
684 .o_connect = ofd_obd_connect,
685 .o_reconnect = ofd_obd_reconnect,
686 .o_disconnect = ofd_obd_disconnect,
687 .o_set_info_async = ofd_set_info_async,
688 .o_get_info = ofd_get_info,
689 .o_statfs = ofd_statfs,
690 .o_init_export = ofd_init_export,
691 .o_destroy_export = ofd_destroy_export,
692 .o_postrecov = ofd_obd_postrecov,
694 .o_iocontrol = ofd_iocontrol,
695 .o_precleanup = ofd_precleanup,
697 .o_health_check = ofd_health_check,
698 .o_notify = ofd_obd_notify,