From 21a9805abde3cbe1ffe046f2d9958d968303b8d3 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Tue, 15 Jan 2013 17:47:44 +0800 Subject: [PATCH] LU-1866 lfsck: LFSCK 1.5 user space control Include the user space command to start/stop the LFSCK for namespace consistency check and repair. And proc interface to query the LFSCK status, statistical informtion, and ect. Test-Parameters: envdefinitions=ENABLE_QUOTA=yes testlist=sanity-lfsck Signed-off-by: Fan Yong Change-Id: Icf4046b313dc1653b5b8b1a3add7fa0329ca7ad0 Reviewed-on: http://review.whamcloud.com/4911 Tested-by: Hudson Reviewed-by: Alex Zhuravlev Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lustre/include/lustre/lustre_lfsck_user.h | 3 +- lustre/include/obd_support.h | 6 + lustre/mdd/mdd_internal.h | 2 + lustre/mdd/mdd_lfsck.c | 407 +++++++++++++++++++++++++++++- lustre/mdd/mdd_lproc.c | 20 ++ lustre/tests/Makefile.am | 2 +- lustre/tests/sanity-lfsck.sh | 322 +++++++++++++++++++++++ lustre/utils/lustre_lfsck.c | 61 +++-- 8 files changed, 794 insertions(+), 29 deletions(-) create mode 100644 lustre/tests/sanity-lfsck.sh diff --git a/lustre/include/lustre/lustre_lfsck_user.h b/lustre/include/lustre/lustre_lfsck_user.h index aaa9330..cb93fde 100644 --- a/lustre/include/lustre/lustre_lfsck_user.h +++ b/lustre/include/lustre/lustre_lfsck_user.h @@ -56,10 +56,11 @@ enum lfsck_type { }; #define LFSCK_VERSION_V1 1 +#define LFSCK_VERSION_V2 2 #define LFSCK_TYPES_ALL ((__u16)(~0)) #define LFSCK_TYPES_DEF ((__u16)0) -#define LFSCK_TYPES_SUPPORTED 0 +#define LFSCK_TYPES_SUPPORTED LT_NAMESPACE #define LFSCK_SPEED_NO_LIMIT 0 #define LFSCK_SPEED_LIMIT_DEF LFSCK_SPEED_NO_LIMIT diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 03eed2f..79ec6aa 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -463,6 +463,12 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_UPDATE_OBJ_NET 0x1500 +/* LFSCK */ +#define OBD_FAIL_LFSCK_DELAY1 0x1600 +#define OBD_FAIL_LFSCK_DELAY2 0x1601 +#define OBD_FAIL_LFSCK_FATAL1 0x1608 +#define OBD_FAIL_LFSCK_FATAL2 0x1609 + /* Assign references to moved code to reduce code changes */ #define OBD_FAIL_PRECHECK(id) CFS_FAIL_PRECHECK(id) #define OBD_FAIL_CHECK(id) CFS_FAIL_CHECK(id) diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h index 8d8f4d8..58fb060 100644 --- a/lustre/mdd/mdd_internal.h +++ b/lustre/mdd/mdd_internal.h @@ -455,6 +455,8 @@ int mdd_lfsck_stop(const struct lu_env *env, struct md_lfsck *lfsck, bool pause); int mdd_lfsck_setup(const struct lu_env *env, struct mdd_device *mdd); void mdd_lfsck_cleanup(const struct lu_env *env, struct mdd_device *mdd); +int mdd_lfsck_dump(const struct lu_env *env, struct md_lfsck *lfsck, + __u16 type, char *buf, int len); /* mdd_device.c */ struct lu_object *mdd_object_alloc(const struct lu_env *env, diff --git a/lustre/mdd/mdd_lfsck.c b/lustre/mdd/mdd_lfsck.c index 04f3a52..349058b 100644 --- a/lustre/mdd/mdd_lfsck.c +++ b/lustre/mdd/mdd_lfsck.c @@ -52,6 +52,31 @@ const char lfsck_bookmark_name[] = "lfsck_bookmark"; const char lfsck_namespace_name[] = "lfsck_namespace"; +static const char *lfsck_status_names[] = { + "init", + "scanning-phase1", + "scanning-phase2", + "completed", + "failed", + "stopped", + "paused", + "crashed", + NULL +}; + +static const char *lfsck_flags_names[] = { + "scanned-once", + "inconsistent", + "upgrade", + NULL +}; + +static const char *lfsck_param_names[] = { + "failout", + "dryrun", + NULL +}; + /* misc functions */ static inline struct mdd_device *mdd_lfsck2mdd(struct md_lfsck *lfsck) @@ -97,6 +122,30 @@ __mdd_lfsck_component_find(struct md_lfsck *lfsck, __u16 type, cfs_list_t *list) return NULL; } +static struct lfsck_component * +mdd_lfsck_component_find(struct md_lfsck *lfsck, __u16 type) +{ + struct lfsck_component *com; + + spin_lock(&lfsck->ml_lock); + com = __mdd_lfsck_component_find(lfsck, type, &lfsck->ml_list_scan); + if (com != NULL) + goto unlock; + + com = __mdd_lfsck_component_find(lfsck, type, + &lfsck->ml_list_double_scan); + if (com != NULL) + goto unlock; + + com = __mdd_lfsck_component_find(lfsck, type, &lfsck->ml_list_idle); + +unlock: + if (com != NULL) + mdd_lfsck_component_get(com); + spin_unlock(&lfsck->ml_lock); + return com; +} + static void mdd_lfsck_component_cleanup(const struct lu_env *env, struct lfsck_component *com) { @@ -108,6 +157,77 @@ static void mdd_lfsck_component_cleanup(const struct lu_env *env, mdd_lfsck_component_put(env, com); } +static int lfsck_bits_dump(char **buf, int *len, int bits, const char *names[], + const char *prefix) +{ + int save = *len; + int flag; + int rc; + int i; + + rc = snprintf(*buf, *len, "%s:%c", prefix, bits != 0 ? ' ' : '\n'); + if (rc <= 0) + return -ENOSPC; + + *buf += rc; + *len -= rc; + for (i = 0, flag = 1; bits != 0; i++, flag = 1 << i) { + if (flag & bits) { + bits &= ~flag; + rc = snprintf(*buf, *len, "%s%c", names[i], + bits != 0 ? ',' : '\n'); + if (rc <= 0) + return -ENOSPC; + + *buf += rc; + *len -= rc; + } + } + return save - *len; +} + +static int lfsck_time_dump(char **buf, int *len, __u64 time, const char *prefix) +{ + int rc; + + if (time != 0) + rc = snprintf(*buf, *len, "%s: "LPU64" seconds\n", prefix, + cfs_time_current_sec() - time); + else + rc = snprintf(*buf, *len, "%s: N/A\n", prefix); + if (rc <= 0) + return -ENOSPC; + + *buf += rc; + *len -= rc; + return rc; +} + +static int lfsck_pos_dump(char **buf, int *len, struct lfsck_position *pos, + const char *prefix) +{ + int rc; + + if (fid_is_zero(&pos->lp_dir_parent)) { + if (pos->lp_oit_cookie == 0) + rc = snprintf(*buf, *len, "%s: N/A, N/A, N/A\n", + prefix); + else + rc = snprintf(*buf, *len, "%s: "LPU64", N/A, N/A\n", + prefix, pos->lp_oit_cookie); + } else { + rc = snprintf(*buf, *len, "%s: "LPU64", "DFID", "LPU64"\n", + prefix, pos->lp_oit_cookie, + PFID(&pos->lp_dir_parent), pos->lp_dir_cookie); + } + if (rc <= 0) + return -ENOSPC; + + *buf += rc; + *len -= rc; + return rc; +} + static void mdd_lfsck_pos_fill(const struct lu_env *env, struct md_lfsck *lfsck, struct lfsck_position *pos, bool oit_processed, bool dir_processed) @@ -362,7 +482,7 @@ static int mdd_lfsck_bookmark_init(const struct lu_env *env, memset(mb, 0, sizeof(mb)); mb->lb_magic = LFSCK_BOOKMARK_MAGIC; - mb->lb_version = LFSCK_VERSION_V1; + mb->lb_version = LFSCK_VERSION_V2; mutex_lock(&lfsck->ml_mutex); rc = mdd_lfsck_bookmark_store(env, lfsck); mutex_unlock(&lfsck->ml_mutex); @@ -919,12 +1039,239 @@ static int mdd_lfsck_namespace_post(const struct lu_env *env, return rc; } -/* XXX: to be implemented in other patch. */ static int mdd_lfsck_namespace_dump(const struct lu_env *env, struct lfsck_component *com, char *buf, int len) { - return 0; + struct md_lfsck *lfsck = com->lc_lfsck; + struct lfsck_bookmark *bk = &lfsck->ml_bookmark_ram; + struct lfsck_namespace *ns = + (struct lfsck_namespace *)com->lc_file_ram; + int save = len; + int ret = -ENOSPC; + int rc; + + down_read(&com->lc_sem); + rc = snprintf(buf, len, + "name: lfsck_namespace\n" + "magic: 0x%x\n" + "version: %d\n" + "status: %s\n", + ns->ln_magic, + bk->lb_version, + lfsck_status_names[ns->ln_status]); + if (rc <= 0) + goto out; + + buf += rc; + len -= rc; + rc = lfsck_bits_dump(&buf, &len, ns->ln_flags, lfsck_flags_names, + "flags"); + if (rc < 0) + goto out; + + rc = lfsck_bits_dump(&buf, &len, bk->lb_param, lfsck_param_names, + "param"); + if (rc < 0) + goto out; + + rc = lfsck_time_dump(&buf, &len, ns->ln_time_last_complete, + "time_since_last_completed"); + if (rc < 0) + goto out; + + rc = lfsck_time_dump(&buf, &len, ns->ln_time_latest_start, + "time_since_latest_start"); + if (rc < 0) + goto out; + + rc = lfsck_time_dump(&buf, &len, ns->ln_time_last_checkpoint, + "time_since_last_checkpoint"); + if (rc < 0) + goto out; + + rc = lfsck_pos_dump(&buf, &len, &ns->ln_pos_latest_start, + "latest_start_position"); + if (rc < 0) + goto out; + + rc = lfsck_pos_dump(&buf, &len, &ns->ln_pos_last_checkpoint, + "last_checkpoint_position"); + if (rc < 0) + goto out; + + rc = lfsck_pos_dump(&buf, &len, &ns->ln_pos_first_inconsistent, + "first_failure_position"); + if (rc < 0) + goto out; + + if (ns->ln_status == LS_SCANNING_PHASE1) { + struct lfsck_position pos; + cfs_duration_t duration = cfs_time_current() - + lfsck->ml_time_last_checkpoint; + __u64 checked = ns->ln_items_checked + com->lc_new_checked; + __u64 speed = checked; + __u64 new_checked = com->lc_new_checked * CFS_HZ; + __u32 rtime = ns->ln_run_time_phase1 + + cfs_duration_sec(duration + HALF_SEC); + + if (duration != 0) + do_div(new_checked, duration); + if (rtime != 0) + do_div(speed, rtime); + rc = snprintf(buf, len, + "checked_phase1: "LPU64"\n" + "checked_phase2: "LPU64"\n" + "updated_phase1: "LPU64"\n" + "updated_phase2: "LPU64"\n" + "failed_phase1: "LPU64"\n" + "failed_phase2: "LPU64"\n" + "dirs: "LPU64"\n" + "M-linked: "LPU64"\n" + "nlinks_repaired: "LPU64"\n" + "lost_found: "LPU64"\n" + "success_count: %u\n" + "run_time_phase1: %u seconds\n" + "run_time_phase2: %u seconds\n" + "average_speed_phase1: "LPU64" items/sec\n" + "average_speed_phase2: N/A\n" + "real-time_speed_phase1: "LPU64" items/sec\n" + "real-time_speed_phase2: N/A\n", + checked, + ns->ln_objs_checked_phase2, + ns->ln_items_repaired, + ns->ln_objs_repaired_phase2, + ns->ln_items_failed, + ns->ln_objs_failed_phase2, + ns->ln_dirs_checked, + ns->ln_mlinked_checked, + ns->ln_objs_nlink_repaired, + ns->ln_objs_lost_found, + ns->ln_success_count, + rtime, + ns->ln_run_time_phase2, + speed, + new_checked); + if (rc <= 0) + goto out; + + buf += rc; + len -= rc; + mdd_lfsck_pos_fill(env, lfsck, &pos, true, true); + rc = lfsck_pos_dump(&buf, &len, &pos, "current_position"); + if (rc <= 0) + goto out; + } else if (ns->ln_status == LS_SCANNING_PHASE2) { + cfs_duration_t duration = cfs_time_current() - + lfsck->ml_time_last_checkpoint; + __u64 checked = ns->ln_objs_checked_phase2 + + com->lc_new_checked; + __u64 speed1 = ns->ln_items_checked; + __u64 speed2 = checked; + __u64 new_checked = com->lc_new_checked * CFS_HZ; + __u32 rtime = ns->ln_run_time_phase2 + + cfs_duration_sec(duration + HALF_SEC); + + if (duration != 0) + do_div(new_checked, duration); + if (ns->ln_run_time_phase1 != 0) + do_div(speed1, ns->ln_run_time_phase1); + if (rtime != 0) + do_div(speed2, rtime); + rc = snprintf(buf, len, + "checked_phase1: "LPU64"\n" + "checked_phase2: "LPU64"\n" + "updated_phase1: "LPU64"\n" + "updated_phase2: "LPU64"\n" + "failed_phase1: "LPU64"\n" + "failed_phase2: "LPU64"\n" + "dirs: "LPU64"\n" + "M-linked: "LPU64"\n" + "nlinks_repaired: "LPU64"\n" + "lost_found: "LPU64"\n" + "success_count: %u\n" + "run_time_phase1: %u seconds\n" + "run_time_phase2: %u seconds\n" + "average_speed_phase1: "LPU64" items/sec\n" + "average_speed_phase2: "LPU64" objs/sec\n" + "real-time_speed_phase1: N/A\n" + "real-time_speed_phase2: "LPU64" objs/sec\n" + "current_position: "DFID"\n", + ns->ln_items_checked, + checked, + ns->ln_items_repaired, + ns->ln_objs_repaired_phase2, + ns->ln_items_failed, + ns->ln_objs_failed_phase2, + ns->ln_dirs_checked, + ns->ln_mlinked_checked, + ns->ln_objs_nlink_repaired, + ns->ln_objs_lost_found, + ns->ln_success_count, + ns->ln_run_time_phase1, + rtime, + speed1, + speed2, + new_checked, + PFID(&ns->ln_fid_latest_scanned_phase2)); + if (rc <= 0) + goto out; + + buf += rc; + len -= rc; + } else { + __u64 speed1 = ns->ln_items_checked; + __u64 speed2 = ns->ln_objs_checked_phase2; + + if (ns->ln_run_time_phase1 != 0) + do_div(speed1, ns->ln_run_time_phase1); + if (ns->ln_run_time_phase2 != 0) + do_div(speed2, ns->ln_run_time_phase2); + rc = snprintf(buf, len, + "checked_phase1: "LPU64"\n" + "checked_phase2: "LPU64"\n" + "updated_phase1: "LPU64"\n" + "updated_phase2: "LPU64"\n" + "failed_phase1: "LPU64"\n" + "failed_phase2: "LPU64"\n" + "dirs: "LPU64"\n" + "M-linked: "LPU64"\n" + "nlinks_repaired: "LPU64"\n" + "lost_found: "LPU64"\n" + "success_count: %u\n" + "run_time_phase1: %u seconds\n" + "run_time_phase2: %u seconds\n" + "average_speed_phase1: "LPU64" items/sec\n" + "average_speed_phase2: "LPU64" objs/sec\n" + "real-time_speed_phase1: N/A\n" + "real-time_speed_phase2: N/A\n" + "current_position: N/A\n", + ns->ln_items_checked, + ns->ln_objs_checked_phase2, + ns->ln_items_repaired, + ns->ln_objs_repaired_phase2, + ns->ln_items_failed, + ns->ln_objs_failed_phase2, + ns->ln_dirs_checked, + ns->ln_mlinked_checked, + ns->ln_objs_nlink_repaired, + ns->ln_objs_lost_found, + ns->ln_success_count, + ns->ln_run_time_phase1, + ns->ln_run_time_phase2, + speed1, + speed2); + if (rc <= 0) + goto out; + + buf += rc; + len -= rc; + } + ret = save - len; + +out: + up_read(&com->lc_sem); + return ret; } /* XXX: to be implemented in other patch. */ @@ -1422,6 +1769,17 @@ static int mdd_lfsck_dir_engine(const struct lu_env *env, do { struct mdd_object *child; + if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_DELAY2) && + cfs_fail_val > 0) { + struct l_wait_info lwi; + + lwi = LWI_TIMEOUT(cfs_time_seconds(cfs_fail_val), + NULL, NULL); + l_wait_event(thread->t_ctl_waitq, + !thread_is_running(thread), + &lwi); + } + lfsck->ml_new_scanned++; rc = iops->rec(env, di, (struct dt_rec *)ent, lfsck->ml_args_dir); @@ -1466,6 +1824,13 @@ checkpoint: if (unlikely(!thread_is_running(thread))) RETURN(0); + if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_FATAL2)) { + spin_lock(&lfsck->ml_lock); + thread_set_flags(thread, SVC_STOPPING); + spin_unlock(&lfsck->ml_lock); + RETURN(-EINVAL); + } + rc = iops->next(env, di); } while (rc == 0); @@ -1501,6 +1866,17 @@ static int mdd_lfsck_oit_engine(const struct lu_env *env, if (unlikely(lfsck->ml_oit_over)) RETURN(1); + if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_DELAY1) && + cfs_fail_val > 0) { + struct l_wait_info lwi; + + lwi = LWI_TIMEOUT(cfs_time_seconds(cfs_fail_val), + NULL, NULL); + l_wait_event(thread->t_ctl_waitq, + !thread_is_running(thread), + &lwi); + } + lfsck->ml_new_scanned++; rc = iops->rec(env, di, (struct dt_rec *)fid, 0); if (rc != 0) { @@ -1539,6 +1915,13 @@ checkpoint: /* Rate control. */ mdd_lfsck_control_speed(lfsck); + if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_FATAL1)) { + spin_lock(&lfsck->ml_lock); + thread_set_flags(thread, SVC_STOPPING); + spin_unlock(&lfsck->ml_lock); + RETURN(-EINVAL); + } + rc = iops->next(env, di); if (rc > 0) lfsck->ml_oit_over = 1; @@ -1658,6 +2041,24 @@ int mdd_lfsck_set_speed(const struct lu_env *env, struct md_lfsck *lfsck, return rc; } +int mdd_lfsck_dump(const struct lu_env *env, struct md_lfsck *lfsck, + __u16 type, char *buf, int len) +{ + struct lfsck_component *com; + int rc; + + if (!lfsck->ml_initialized) + return -ENODEV; + + com = mdd_lfsck_component_find(lfsck, type); + if (com == NULL) + return -ENOTSUPP; + + rc = com->lc_ops->lfsck_dump(env, com, buf, len); + mdd_lfsck_component_put(env, com); + return rc; +} + int mdd_lfsck_start(const struct lu_env *env, struct md_lfsck *lfsck, struct lfsck_start *start) { diff --git a/lustre/mdd/mdd_lproc.c b/lustre/mdd/mdd_lproc.c index 4d8e42d..4d64e43 100644 --- a/lustre/mdd/mdd_lproc.c +++ b/lustre/mdd/mdd_lproc.c @@ -303,6 +303,25 @@ static int lprocfs_wr_lfsck_speed_limit(struct file *file, const char *buffer, return rc != 0 ? rc : count; } +static int lprocfs_rd_lfsck_namespace(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct lu_env env; + struct mdd_device *mdd = data; + int rc; + + LASSERT(mdd != NULL); + *eof = 1; + + rc = lu_env_init(&env, LCT_MD_THREAD | LCT_DT_THREAD); + if (rc != 0) + return rc; + + rc = mdd_lfsck_dump(&env, &mdd->mdd_lfsck, LT_NAMESPACE, page, count); + lu_env_fini(&env); + return rc; +} + static struct lprocfs_vars lprocfs_mdd_obd_vars[] = { { "atime_diff", lprocfs_rd_atime_diff, lprocfs_wr_atime_diff, 0 }, { "changelog_mask", lprocfs_rd_changelog_mask, @@ -311,6 +330,7 @@ static struct lprocfs_vars lprocfs_mdd_obd_vars[] = { { "sync_permission", lprocfs_rd_sync_perm, lprocfs_wr_sync_perm, 0 }, { "lfsck_speed_limit", lprocfs_rd_lfsck_speed_limit, lprocfs_wr_lfsck_speed_limit, 0 }, + { "lfsck_namespace", lprocfs_rd_lfsck_namespace, 0, 0 }, { 0 } }; diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 77c781f..1ae7af5 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -32,7 +32,7 @@ noinst_SCRIPTS += sgpdd-survey.sh maloo_upload.sh auster setup-nfs.sh noinst_SCRIPTS += mds-survey.sh parallel-scale-nfs.sh large-lun.sh noinst_SCRIPTS += parallel-scale-nfsv3.sh parallel-scale-nfsv4.sh noinst_SCRIPTS += posix.sh sanity-scrub.sh scrub-performance.sh ha.sh -noinst_SCRIPTS += sanity-quota-old.sh +noinst_SCRIPTS += sanity-quota-old.sh sanity-lfsck.sh noinst_SCRIPTS += resolveip noinst_SCRIPTS += sanity-hsm.sh nobase_noinst_SCRIPTS = cfg/local.sh diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh new file mode 100644 index 0000000..3a550cf --- /dev/null +++ b/lustre/tests/sanity-lfsck.sh @@ -0,0 +1,322 @@ +#!/bin/bash +# +# Run select tests by setting ONLY, or as arguments to the script. +# Skip specific tests by setting EXCEPT. +# + +set -e + +ONLY=${ONLY:-"$*"} +ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT" +[ "$SLOW" = "no" ] && EXCEPT_SLOW="" +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! + +LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging + +[ $(facet_fstype $SINGLEMDS) != ldiskfs ] && + skip "test LFSCK only for ldiskfs" && exit 0 +require_dsh_mds || exit 0 + +SAVED_MDSSIZE=${MDSSIZE} +SAVED_OSTSIZE=${OSTSIZE} +# use small MDS + OST size to speed formatting time +# do not use too small MDSSIZE/OSTSIZE, which affect the default journal size +MDSSIZE=100000 +OSTSIZE=100000 + +check_and_setup_lustre +build_test_filter + +$LCTL set_param debug=+lfsck > /dev/null || true + +MDT_DEV="${FSNAME}-MDT0000" +MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/}) +START_NAMESPACE="do_facet $SINGLEMDS \ + $LCTL lfsck_start -M ${MDT_DEV} -t namespace" +STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}" +SHOW_NAMESPACE="do_facet $SINGLEMDS \ + $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace" +MOUNT_OPTS_SCRUB="-o user_xattr" +MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub" + +lfsck_prep() { + local ndirs=$1 + local nfiles=$2 + + echo "formatall" + formatall > /dev/null + + echo "setupall" + setupall > /dev/null + + echo "preparing... ${nfiles} * ${ndirs} files will be created." + mkdir -p $DIR/$tdir + cp $LUSTRE/tests/*.sh $DIR/$tdir/ + for ((i=0; i<${ndirs}; i++)); do + mkdir $DIR/$tdir/d${i} + touch $DIR/$tdir/f${i} + for ((j=0; j<${nfiles}; j++)); do + touch $DIR/$tdir/d${i}/f${j} + done + mkdir $DIR/$tdir/e${i} + done + + echo "prepared." + cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!" + echo "stop $SINGLEMDS" + stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!" +} + +test_0() { + lfsck_prep 10 10 + echo "start $SINGLEMDS" + start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || + error "(1) Fail to start MDS!" + + #define OBD_FAIL_LFSCK_DELAY1 0x1600 + do_facet $SINGLEMDS $LCTL set_param fail_val=3 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600 + $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!" + + $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)" + + local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(4) Expect 'scanning-phase1', but got '$STATUS'" + + $STOP_LFSCK || error "(5) Fail to stop LFSCK!" + + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "stopped" ] || + error "(6) Expect 'stopped', but got '$STATUS'" + + $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!" + + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(8) Expect 'scanning-phase1', but got '$STATUS'" + + do_facet $SINGLEMDS $LCTL set_param fail_loc=0 + do_facet $SINGLEMDS $LCTL set_param fail_val=0 + sleep 3 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "completed" ] || + error "(9) Expect 'completed', but got '$STATUS'" + + local repaired=$($SHOW_NAMESPACE | + awk '/^updated_phase1/ { print $2 }') + [ $repaired -eq 0 ] || + error "(10) Expect nothing to be repaired, but got: $repaired" +} +run_test 0 "Control LFSCK manually" + +test_6a() { + lfsck_prep 10 10 + echo "start $SINGLEMDS" + start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || + error "(1) Fail to start MDS!" + + #define OBD_FAIL_LFSCK_DELAY1 0x1600 + do_facet $SINGLEMDS $LCTL set_param fail_val=1 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600 + $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!" + + local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(3) Expect 'scanning-phase1', but got '$STATUS'" + + # Sleep 3 sec to guarantee at least one object processed by LFSCK + sleep 3 + # Fail the LFSCK to guarantee there is at least one checkpoint + #define OBD_FAIL_LFSCK_FATAL1 0x1608 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608 + sleep 3 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "failed" ] || + error "(4) Expect 'failed', but got '$STATUS'" + + local POSITION0=$($SHOW_NAMESPACE | + awk '/^last_checkpoint_position/ { print $2 }' | + tr -d ',') + + #define OBD_FAIL_LFSCK_DELAY1 0x1600 + do_facet $SINGLEMDS $LCTL set_param fail_val=1 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600 + $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!" + + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(6) Expect 'scanning-phase1', but got '$STATUS'" + + local POSITION1=$($SHOW_NAMESPACE | + awk '/^latest_start_position/ { print $2 }' | + tr -d ',') + [ $POSITION0 -lt $POSITION1 ] || + error "(7) Expect larger than: $POSITION0, but got $POSITION1" + + do_facet $SINGLEMDS $LCTL set_param fail_loc=0 + do_facet $SINGLEMDS $LCTL set_param fail_val=0 + sleep 3 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "completed" ] || + error "(8) Expect 'completed', but got '$STATUS'" +} +run_test 6a "LFSCK resumes from last checkpoint (1)" + +test_6b() { + lfsck_prep 10 10 + echo "start $SINGLEMDS" + start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || + error "(1) Fail to start MDS!" + + #define OBD_FAIL_LFSCK_DELAY2 0x1601 + do_facet $SINGLEMDS $LCTL set_param fail_val=1 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601 + $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!" + + local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(3) Expect 'scanning-phase1', but got '$STATUS'" + + # Sleep 3 sec to guarantee at least one object processed by LFSCK + sleep 3 + # Fail the LFSCK to guarantee there is at least one checkpoint + #define OBD_FAIL_LFSCK_FATAL2 0x1609 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609 + sleep 3 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "failed" ] || + error "(4) Expect 'failed', but got '$STATUS'" + + local POSITION0=$($SHOW_NAMESPACE | + awk '/^last_checkpoint_position/ { print $4 }') + + #define OBD_FAIL_LFSCK_DELAY2 0x1601 + do_facet $SINGLEMDS $LCTL set_param fail_val=1 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601 + $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!" + + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(6) Expect 'scanning-phase1', but got '$STATUS'" + + local POSITION1=$($SHOW_NAMESPACE | + awk '/^latest_start_position/ { print $4 }') + if [ $POSITION0 -gt $POSITION1 ]; then + [ $POSITION1 -eq 0 -a $POSITINO0 -eq $((POSITION1 + 1)) ] || + error "(7) Expect larger than: $POSITION0, but got $POSITION1" + fi + + do_facet $SINGLEMDS $LCTL set_param fail_loc=0 + do_facet $SINGLEMDS $LCTL set_param fail_val=0 + sleep 3 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "completed" ] || + error "(8) Expect 'completed', but got '$STATUS'" +} +run_test 6b "LFSCK resumes from last checkpoint (2)" + +test_7a() +{ + lfsck_prep 10 10 + echo "start $SINGLEMDS" + start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || + error "(1) Fail to start MDS!" + + #define OBD_FAIL_LFSCK_DELAY2 0x1601 + do_facet $SINGLEMDS $LCTL set_param fail_val=1 + do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601 + $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!" + + local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(3) Expect 'scanning-phase1', but got '$STATUS'" + + # Sleep 3 sec to guarantee at least one object processed by LFSCK + sleep 3 + echo "stop $SINGLEMDS" + stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!" + + echo "start $SINGLEMDS" + start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || + error "(5) Fail to start MDS!" + + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(6) Expect 'scanning-phase1', but got '$STATUS'" + + do_facet $SINGLEMDS $LCTL set_param fail_loc=0 + do_facet $SINGLEMDS $LCTL set_param fail_val=0 + sleep 3 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "completed" ] || + error "(7) Expect 'completed', but got '$STATUS'" +} +run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)" + +test_9a() { + if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then + skip "Testing on UP system, the speed may be inaccurate." + return 0 + fi + + lfsck_prep 70 70 + echo "start $SINGLEMDS" + start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || + error "(1) Fail to start MDS!" + + local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "init" ] || + error "(2) Expect 'init', but got '$STATUS'" + + $START_NAMESPACE -s 100 || error "(3) Fail to start LFSCK!" + + sleep 10 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "(3) Expect 'scanning-phase1', but got '$STATUS'" + + local SPEED=$($SHOW_NAMESPACE | + awk '/^average_speed_phase1/ { print $2 }') + # (100 * (10 + 1)) / 10 = 110 + [ $SPEED -lt 120 ] || + error "(4) Unexpected speed $SPEED, should not more than 120" + + # adjust speed limit + do_facet $SINGLEMDS \ + $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300 + sleep 10 + + SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }') + # (100 * (10 - 1) + 300 * (10 - 1)) / 20 = 180 + [ $SPEED -lt 170 ] && + error "(5) Unexpected speed $SPEED, should not less than 170" + + # (100 * (10 + 1) + 300 * (10 + 1)) / 20 = 220 + [ $SPEED -lt 230 ] || + error "(6) Unexpected speed $SPEED, should not more than 230" + + do_facet $SINGLEMDS \ + $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0 + sleep 5 + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "completed" ] || + error "(7) Expect 'completed', but got '$STATUS'" +} +run_test 9a "LFSCK speed control (1)" + +$LCTL set_param debug=-lfsck > /dev/null || true + +# restore MDS/OST size +MDSSIZE=${SAVED_MDSSIZE} +OSTSIZE=${SAVED_OSTSIZE} + +# cleanup the system at last +formatall + +complete $SECONDS +exit_status diff --git a/lustre/utils/lustre_lfsck.c b/lustre/utils/lustre_lfsck.c index 61fd534..4ed27eb 100644 --- a/lustre/utils/lustre_lfsck.c +++ b/lustre/utils/lustre_lfsck.c @@ -62,17 +62,32 @@ static struct option long_opt_stop[] = { {0, 0, 0, 0} }; -struct lfsck_types_names { - char *name; - __u16 type; +struct lfsck_type_name { + char *name; + int namelen; + enum lfsck_type type; }; -static struct lfsck_types_names lfsck_types_names[3] = { - { "layout", LT_LAYOUT }, - { "DNE", LT_DNE }, - { 0, 0 } +static struct lfsck_type_name lfsck_types_names[] = { + { "layout", 6, LT_LAYOUT }, + { "DNE", 3, LT_DNE }, + { "namespace", 9, LT_NAMESPACE}, + { 0, 0, 0 } }; +static inline int lfsck_name2type(const char *name, int namelen) +{ + int i = 0; + + while (lfsck_types_names[i].name != NULL) { + if (namelen == lfsck_types_names[i].namelen && + strncmp(lfsck_types_names[i].name, name, namelen) == 0) + return lfsck_types_names[i].type; + i++; + } + return 0; +} + static void usage_start(void) { fprintf(stderr, "Start LFSCK.\n" @@ -128,7 +143,7 @@ int jt_lfsck_start(int argc, char **argv) char device[MAX_OBD_NAME]; struct lfsck_start start; char *optstring = "M:e:hn:rs:t:"; - int opt, index, rc, val, i; + int opt, index, rc, val, i, type; memset(&data, 0, sizeof(data)); memset(&start, 0, sizeof(start)); @@ -197,29 +212,25 @@ int jt_lfsck_start(int argc, char **argv) c = *p; *p = 0; - for (i = 0; i < 3; i++) { - if (strcmp(str, - lfsck_types_names[i].name) - == 0) { - start.ls_active |= - lfsck_types_names[i].type; - break; - } + type = lfsck_name2type(str, strlen(str)); + if (type == 0) { + fprintf(stderr, "Invalid type (%s).\n" + "The valid value should be " + "'layout', 'DNE' or " + "'namespace'.\n", str); + *p = c; + return -EINVAL; } + *p = c; str = p; - if (i >= 3 ) { - fprintf(stderr, "Invalid LFSCK type.\n" - "The valid value should be " - "'layout' or 'DNE'.\n"); - return -EINVAL; - } + start.ls_active |= type; } if (start.ls_active == 0) { fprintf(stderr, "Miss LFSCK type(s).\n" "The valid value should be " - "'layout' or 'DNE'.\n"); + "'layout', 'DNE' or 'namespace'.\n"); return -EINVAL; } break; @@ -256,11 +267,13 @@ int jt_lfsck_start(int argc, char **argv) printf("Started LFSCK on the MDT device %s", device); } else { printf("Started LFSCK on the MDT device %s:", device); - for (i = 0; i < 2; i++) { + i = 0; + while (lfsck_types_names[i].name != NULL) { if (start.ls_active & lfsck_types_names[i].type) { printf(" %s", lfsck_types_names[i].name); start.ls_active &= ~lfsck_types_names[i].type; } + i++; } if (start.ls_active != 0) printf(" unknown(0x%x)", start.ls_active); -- 1.8.3.1