unsigned int lut_syncjournal:1,
lut_sync_lock_cancel:2,
/* e.g. OST node */
- lut_no_reconstruct:1;
+ lut_no_reconstruct:1,
+ /* enforce recovery for local clients */
+ lut_local_recovery:1;
/** last_rcvd file */
struct dt_object *lut_last_rcvd;
/* transaction callbacks */
#define LMD_FLG_HSM 0x4000 /* Start coordinator */
#define LMD_FLG_DEV_RDONLY 0x8000 /* discard modification quitely */
#define LMD_FLG_NO_PRECREATE 0x10000 /* do not allow OST object creation */
+#define LMD_FLG_LOCAL_RECOV 0x20000 /* force recovery for local clients */
#define lmd_is_client(x) ((x)->lmd_flags & LMD_FLG_CLIENT)
exp_need_mne_swab:1,
/* The export already got final replay ping
* request. */
- exp_replay_done:1;
+ exp_replay_done:1,
+ /* local client with recovery disabled */
+ exp_no_recovery:1;
/* also protected by exp_lock */
enum lustre_sec_part exp_sp_peer;
struct sptlrpc_flavor exp_flvr; /* current */
RETURN(-EFAULT);
} else {
lsi = s2lsi(lmi->lmi_sb);
+ LASSERT(lsi->lsi_lmd);
/* CMD is supported only in IAM mode */
LASSERT(num);
node_id = simple_strtol(num, NULL, 10);
obd->u.obt.obt_magic = OBT_MAGIC;
- if (lsi->lsi_lmd != NULL &&
- lsi->lsi_lmd->lmd_flags & LMD_FLG_SKIP_LFSCK)
+ if (lsi->lsi_lmd->lmd_flags & LMD_FLG_SKIP_LFSCK)
m->mdt_skip_lfsck = 1;
}
if (ldlm_timeout == LDLM_TIMEOUT_DEFAULT)
ldlm_timeout = MDS_LDLM_TIMEOUT_DEFAULT;
+ if ((lsi->lsi_lmd->lmd_flags & LMD_FLG_LOCAL_RECOV))
+ m->mdt_lut.lut_local_recovery = 1;
+
RETURN(0);
err_procfs:
mdt_tunables_fini(m);
}
LUSTRE_RW_ATTR(commit_on_sharing);
+static ssize_t local_recovery_show(struct kobject *kobj,
+ struct attribute *attr, char *buf)
+{
+ struct obd_device *obd = container_of(kobj, struct obd_device,
+ obd_kset.kobj);
+
+ return scnprintf(buf, PAGE_SIZE, "%u\n",
+ obd->u.obt.obt_lut->lut_local_recovery);
+}
+
+static ssize_t local_recovery_store(struct kobject *kobj,
+ struct attribute *attr,
+ const char *buffer, size_t count)
+{
+ struct obd_device *obd = container_of(kobj, struct obd_device,
+ obd_kset.kobj);
+ bool val;
+ int rc;
+
+ rc = kstrtobool(buffer, &val);
+ if (rc)
+ return rc;
+
+ obd->u.obt.obt_lut->lut_local_recovery = !!val;
+ return count;
+}
+LUSTRE_RW_ATTR(local_recovery);
+
static int mdt_root_squash_seq_show(struct seq_file *m, void *data)
{
struct obd_device *obd = m->private;
&lustre_attr_enable_dir_migration.attr,
&lustre_attr_enable_remote_rename.attr,
&lustre_attr_commit_on_sharing.attr,
+ &lustre_attr_local_recovery.attr,
&lustre_attr_async_commit_count.attr,
&lustre_attr_sync_count.attr,
&lustre_attr_dom_lock.attr,
flag2str(exp, in_recovery);
flag2str(exp, disconnected);
flag2str(exp, connecting);
+ flag2str(exp, no_recovery);
return 0;
}
strlcat(lmd->lmd_params, " ", LMD_PARAMS_MAXLEN);
s3 = s1 + 6 + length;
clear++;
+ } else if (strncmp(s1, "localrecov", 10) == 0) {
+ lmd->lmd_flags |= LMD_FLG_LOCAL_RECOV;
+ clear++;
} else if (strncmp(s1, "osd=", 4) == 0) {
rc = lmd_parse_string(&lmd->lmd_osd_type, s1 + 4);
if (rc)
* \retval negative value on error
*/
static int ofd_stack_init(const struct lu_env *env,
- struct ofd_device *m, struct lustre_cfg *cfg)
+ struct ofd_device *m, struct lustre_cfg *cfg,
+ u32 *lmd_flags)
{
const char *dev = lustre_cfg_string(cfg, 0);
struct lu_device *d;
}
lmd = s2lsi(lmi->lmi_sb)->lsi_lmd;
- if (lmd && lmd->lmd_flags & LMD_FLG_SKIP_LFSCK)
- m->ofd_skip_lfsck = 1;
-
- if (lmd && lmd->lmd_flags & LMD_FLG_NO_PRECREATE)
- m->ofd_no_precreate = 1;
+ if (lmd) {
+ if (lmd->lmd_flags & LMD_FLG_SKIP_LFSCK)
+ m->ofd_skip_lfsck = 1;
+ if (lmd->lmd_flags & LMD_FLG_NO_PRECREATE)
+ m->ofd_no_precreate = 1;
+ *lmd_flags = lmd->lmd_flags;
+ }
/* find bottom osd */
OBD_ALLOC(osdname, MTI_NAME_MAXLEN);
struct lu_fid fid;
struct nm_config_file *nodemap_config;
struct obd_device_target *obt;
+ u32 lmd_flags = 0;
int rc;
ENTRY;
if (info == NULL)
RETURN(-EFAULT);
- rc = ofd_stack_init(env, m, cfg);
+ rc = ofd_stack_init(env, m, cfg, &lmd_flags);
if (rc) {
CERROR("%s: can't init device stack, rc %d\n",
obd->obd_name, rc);
if (rc)
GOTO(err_free_ns, rc);
+ if (lmd_flags & LMD_FLG_SKIP_LFSCK)
+ m->ofd_skip_lfsck = 1;
+ if (lmd_flags & LMD_FLG_LOCAL_RECOV)
+ m->ofd_lut.lut_local_recovery = 1;
+
rc = ofd_tunables_init(m);
if (rc)
GOTO(err_fini_lut, rc);
return rc ? rc : exp->exp_need_sync;
}
+static int tgt_is_local_client(const struct lu_env *env,
+ struct obd_export *exp)
+{
+ struct lu_target *tgt = class_exp2tgt(exp);
+ struct tgt_session_info *tsi = tgt_ses_info(env);
+ struct ptlrpc_request *req = tgt_ses_req(tsi);
+
+ if (tgt->lut_local_recovery)
+ return 0;
+ if (!req)
+ return 0;
+ if (!LNetIsPeerLocal(req->rq_peer.nid))
+ return 0;
+ if (exp_connect_flags(exp) & OBD_CONNECT_MDS)
+ return 0;
+
+ return 1;
+}
+
/**
* Add new client to the last_rcvd upon new connection.
*
if (exp_connect_flags(exp) & OBD_CONNECT_LIGHTWEIGHT)
RETURN(0);
+ if (tgt_is_local_client(env, exp)) {
+ LCONSOLE_WARN("%s: local client %s w/o recovery\n",
+ exp->exp_obd->obd_name, ted->ted_lcd->lcd_uuid);
+ exp->exp_no_recovery = 1;
+ RETURN(0);
+ }
+
/* the bitmap operations can handle cl_idx > sizeof(long) * 8, so
* there's no need for extra complication here
*/
/* XXX if lcd_uuid were a real obd_uuid, I could use obd_uuid_equals */
if (!strcmp((char *)ted->ted_lcd->lcd_uuid,
(char *)tgt->lut_obd->obd_uuid.uuid) ||
- exp_connect_flags(exp) & OBD_CONNECT_LIGHTWEIGHT)
+ exp_connect_flags(exp) & OBD_CONNECT_LIGHTWEIGHT ||
+ exp->exp_no_recovery)
RETURN(0);
/* Slot may be not yet assigned, use case is race between Client
#
MDSOPT=${MDSOPT:-}
MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
-MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}
+# use localrecov to enable recovery on local clients, LU-12722
+MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:--olocalrecov}
# <facet_type>_MOUNT_FS_OPTS is the mount options specified when formatting
# the underlying device by argument "--mountfsoptions"
MDS_MOUNT_FS_OPTS=${MDS_MOUNT_FS_OPTS:-}
OSTSIZE=${OSTSIZE:-400000}
OSTOPT=${OSTOPT:-}
OST_FS_MKFS_OPTS=${OST_FS_MKFS_OPTS:-}
-OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-}
+# use localrecov to enable recovery on local clients, LU-12722
+OST_MOUNT_OPTS=${OST_MOUNT_OPTS:--olocalrecov}
OST_MOUNT_FS_OPTS=${OST_MOUNT_FS_OPTS:-}
OST_INDEX_LIST=${OST_INDEX_LIST:-}
# Can specify individual ost devs with
error "format ost1 error"
if ! test -b $dev; then
- mnt_opts=$(csa_add "$OST_MOUNT_OPTS" -o loop)
+ mnt_opts=$(csa_add "$OST_MOUNT_FS_OPTS" -o loop)
fi
echo "mnt_opts $mnt_opts"
do_facet ost1 mount -t "$ost1_FSTYPE" $dev \
#define OBD_FAIL_FLD_QUERY_REQ 0x1103
do_facet mds2 lctl set_param fail_loc=0x1103
- start mds2 $(mdsdevname 2) -o abort_recovery ||
+ local OPTS="$MDS_MOUNT_OPTS -o abort_recovery"
+ start mds2 $(mdsdevname 2) $OPTS ||
error "start MDS with abort_recovery should succeed"
do_facet mds2 lctl set_param fail_loc=0
# cleanup
stop mds2 || error "cleanup: stop mds2 failed"
- start mds2 $(mdsdevname 2) || error "cleanup: start mds2 failed"
+ start mds2 $(mdsdevname 2) $MDS_MOUNT_OPTS ||
+ error "cleanup: start mds2 failed"
zconf_mount $(hostname) $MOUNT || error "cleanup: mount failed"
client_up || error "post-failover df failed"
}
#define OBD_FAIL_MDS_CHANGELOG_INIT 0x151
do_facet $SINGLEMDS lctl set_param fail_loc=0x151
stop $SINGLEMDS || error "stop MDS failed"
- start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) &&
+ start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) $MDS_MOUNT_OPTS &&
error "start MDS should fail"
do_facet $SINGLEMDS lctl set_param fail_loc=0
- start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) ||
+ start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) $MDS_MOUNT_OPTS ||
error "start MDS failed"
}
run_test 111 "mdd setup fail should not cause umount oops"
}
run_test 139 "corrupted catid won't cause crash"
+test_140a() {
+ [ $MDS1_VERSION -lt $(version_code 2.12.58) ] &&
+ skip "Need MDS version at least 2.13.50"
+
+ slr=$(do_facet mds1 \
+ $LCTL get_param -n mdt.$FSNAME-MDT0000.local_recovery)
+ stack_trap "do_facet mds1 $LCTL set_param \
+ mdt.*.local_recovery=$slr" EXIT
+
+ # disable recovery for local clients
+ # so local clients should be marked with no_recovery flag
+ do_facet mds1 $LCTL set_param mdt.*.local_recovery=0
+ mount_mds_client
+
+ local cnt
+ cnt=$(do_facet mds1 $LCTL get_param "mdt.*.exports.*.export" |
+ grep export_flags.*no_recovery | wc -l)
+ echo "$cnt clients with recovery disabled"
+ umount_mds_client
+ [ $cnt -eq 0 ] && error "no clients with recovery disabled"
+
+ # enable recovery for local clients
+ # so no local clients should be marked with no_recovery flag
+ do_facet mds1 $LCTL set_param mdt.*.local_recovery=1
+ mount_mds_client
+
+ cnt=$(do_facet mds1 $LCTL get_param "mdt.*.exports.*.export" |
+ grep export_flags.*no_recovery | wc -l)
+ echo "$cnt clients with recovery disabled"
+ umount_mds_client
+ [ $cnt -eq 0 ] || error "$cnt clients with recovery disabled"
+}
+run_test 140a "local mount is flagged properly"
+
+test_140b() {
+ [ $MDS1_VERSION -lt $(version_code 2.12.58) ] &&
+ skip "Need MDS version at least 2.13.50"
+
+ slr=$(do_facet mds1 \
+ $LCTL get_param -n mdt.$FSNAME-MDT0000.local_recovery)
+ stack_trap "do_facet mds1 $LCTL set_param \
+ mdt.*.local_recovery=$slr" EXIT
+
+ # disable recovery for local clients
+ do_facet mds1 $LCTL set_param mdt.*.local_recovery=0
+
+ mount_mds_client
+ replay_barrier mds1
+ umount_mds_client
+ local before=$SECONDS
+ fail mds1
+ local after=$SECONDS
+ (( $after-$before < $TIMEOUT*2 )) ||
+ error "recovery took too long" $((after-bsfore)) $TIMEOUT
+}
+run_test 140b "local mount is excluded from recovery"
+
complete $SECONDS
check_and_cleanup_lustre
exit_status
$LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
SHOW_LAYOUT_ON_OST="do_facet ost1 \
$LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
-MOUNT_OPTS_SCRUB="-o user_xattr"
-MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
+MOUNT_OPTS_SCRUB="$MDS_MOUNT_OPTS -o user_xattr"
+MOUNT_OPTS_NOSCRUB="$MDS_MOUNT_OPTS -o user_xattr,noscrub"
MOUNT_OPTS_SKIP_LFSCK="-o user_xattr,skip_lfsck"
lfsck_prep() {
fi
}
+# Mount the file system on the MDS
+mount_mds_client() {
+ local mds_HOST=${SINGLEMDS}_HOST
+ echo $mds_HOST
+ do_facet $SINGLEMDS "mkdir -p $MOUNT2"
+ zconf_mount $mds1_HOST $MOUNT2 $MOUNT_OPTS ||
+ error "unable to mount $MOUNT2 on MDS"
+}
+
+# Unmount the file system on the MDS
+umount_mds_client() {
+ local mds_HOST=${SINGLEMDS}_HOST
+ zconf_umount $mds1_HOST $MOUNT2
+ do_facet $SINGLEMDS "rm -rf $MOUNT2"
+}
+
# nodes is comma list
sanity_mount_check_nodes () {
local nodes=$1
local rcmd="do_facet $facet"
local metaea=${TMP}/backup_restore.ea
local metadata=${TMP}/backup_restore.tgz
- local opts=${MDS_MOUNT_OPTS}
+ local opts=${MDS_MOUNT_FS_OPTS}
local svc=${facet}_svc
if ! ${rcmd} test -b ${devname}; then
local devname=$(mdsdevname $(facet_number $facet))
local mntpt=$(facet_mntpt brpt)
local rcmd="do_facet $facet"
- local opts=${MDS_MOUNT_OPTS}
+ local opts=${MDS_MOUNT_FS_OPTS}
if ! ${rcmd} test -b ${devname}; then
opts=$(csa_add "$opts" -o loop)