From: Jinshan Xiong Date: Tue, 4 Oct 2011 04:35:24 +0000 (-0700) Subject: ORNL-14: Configuration and Unit test cases X-Git-Tag: 2.1.51~11 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=a37738fcbd79e283c687a630d9ccb7d7a9fedb3e ORNL-14: Configuration and Unit test cases In this patch, some procfs and test cases are added for imperative recovery. Signed-off-by: Jinshan Xiong Change-Id: Ied49b622208157a74e8547e4609d61ee5041f624 Reviewed-on: http://review.whamcloud.com/1219 Tested-by: Hudson Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Oleg Drokin Reviewed-by: Yu Jian --- diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index 771f226..aeed486 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -541,6 +541,10 @@ extern int lprocfs_wr_ping(struct file *file, const char *buffer, unsigned long count, void *data); extern int lprocfs_wr_import(struct file *file, const char *buffer, unsigned long count, void *data); +extern int lprocfs_rd_pinger_recov(char *page, char **start, off_t off, + int count, int *eof, void *data); +extern int lprocfs_wr_pinger_recov(struct file *file, const char *buffer, + unsigned long count, void *data); /* Statfs helpers */ extern int lprocfs_rd_blksize(char *page, char **start, off_t off, @@ -845,6 +849,9 @@ static inline int lprocfs_rd_conn_uuid(char *page, char **start, off_t off, static inline int lprocfs_rd_import(char *page, char **start, off_t off, int count, int *eof, void *data) { return 0; } +static inline int lprocfs_rd_pinger_recov(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ return 0; } static inline int lprocfs_rd_state(char *page, char **start, off_t off, int count, int *eof, void *data) { return 0; } @@ -878,6 +885,9 @@ static inline int lprocfs_wr_ping(struct file *file, const char *buffer, static inline int lprocfs_wr_import(struct file *file, const char *buffer, unsigned long count, void *data) { return 0; } +static inline int lprocfs_wr_pinger_recov(struct file *file, const char *buffer, + unsigned long count, void *data) +{ return 0; } /* Statfs helpers */ static inline diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index 339a232..17c36dd 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -263,6 +263,7 @@ struct obd_import { imp_force_verify:1, /* force an immidiate ping */ imp_pingable:1, /* pingable */ imp_resend_replay:1, /* resend for replay */ + imp_no_pinger_recover:1,/* disable normal recovery, for test only. */ imp_force_reconnect:1; /* import must be reconnected instead of chouse new connection */ __u32 imp_connect_op; struct obd_connect_data imp_connect_data; diff --git a/lustre/mdc/lproc_mdc.c b/lustre/mdc/lproc_mdc.c index 1a84846..1834e82 100644 --- a/lustre/mdc/lproc_mdc.c +++ b/lustre/mdc/lproc_mdc.c @@ -166,6 +166,8 @@ static struct lprocfs_vars lprocfs_mdc_obd_vars[] = { { "import", lprocfs_rd_import, lprocfs_wr_import, 0 }, { "state", lprocfs_rd_state, 0, 0 }, { "hsm_nl", 0, mdc_wr_kuc, 0, 0, 0222 }, + { "pinger_recov", lprocfs_rd_pinger_recov, + lprocfs_wr_pinger_recov, 0, 0 }, { 0 } }; diff --git a/lustre/mgc/lproc_mgc.c b/lustre/mgc/lproc_mgc.c index e05b49b..b4305b0 100644 --- a/lustre/mgc/lproc_mgc.c +++ b/lustre/mgc/lproc_mgc.c @@ -39,6 +39,7 @@ #include #include #include +#include "mgc_internal.h" #ifdef LPROCFS @@ -50,6 +51,7 @@ static struct lprocfs_vars lprocfs_mgc_obd_vars[] = { { "mgs_conn_uuid", lprocfs_rd_conn_uuid, 0, 0 }, { "import", lprocfs_rd_import, 0, 0 }, { "state", lprocfs_rd_state, 0, 0 }, + { "ir_state", lprocfs_mgc_rd_ir_state, 0, 0 }, { 0 } }; diff --git a/lustre/mgc/mgc_internal.h b/lustre/mgc/mgc_internal.h index 0e5ad2c..358f982 100644 --- a/lustre/mgc/mgc_internal.h +++ b/lustre/mgc/mgc_internal.h @@ -46,11 +46,18 @@ #ifdef LPROCFS void lprocfs_mgc_init_vars(struct lprocfs_static_vars *lvars); +int lprocfs_mgc_rd_ir_state(char *page, char **start, off_t off, + int count, int *eof, void *data); #else static void lprocfs_mgc_init_vars(struct lprocfs_static_vars *lvars) { memset(lvars, 0, sizeof(*lvars)); } +static inline int lprocfs_mgc_rd_ir_state(char *page, char **start, + off_t off, int count, int *eof, void *data) +{ + return 0; +} #endif /* LPROCFS */ int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld); diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index 57f69e8..f4c7d13 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -413,6 +413,34 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) RETURN(rc); } +int lprocfs_mgc_rd_ir_state(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct obd_device *obd = data; + struct obd_import *imp = obd->u.cli.cl_import; + struct obd_connect_data *ocd = &imp->imp_connect_data; + struct config_llog_data *cld; + int rc = 0; + ENTRY; + + rc = snprintf(page, count, "imperative_recovery: %s\n", + OCD_HAS_FLAG(ocd, IMP_RECOV) ? "ON" : "OFF"); + rc += snprintf(page + rc, count - rc, "client_state:\n"); + + cfs_spin_lock(&config_list_lock); + cfs_list_for_each_entry(cld, &config_llog_list, cld_list_chain) { + if (cld->cld_recover == NULL) + continue; + rc += snprintf(page + rc, count - rc, + " - { client: %s, nidtbl_version: %u }\n", + cld->cld_logname, + cld->cld_recover->cld_cfg.cfg_last_idx); + } + cfs_spin_unlock(&config_list_lock); + + RETURN(rc); +} + /* reenqueue any lost locks */ #define RQ_RUNNING 0x1 #define RQ_NOW 0x2 diff --git a/lustre/mgs/lproc_mgs.c b/lustre/mgs/lproc_mgs.c index b424d2b..673ab77 100644 --- a/lustre/mgs/lproc_mgs.c +++ b/lustre/mgs/lproc_mgs.c @@ -217,8 +217,6 @@ static int mgs_live_seq_show(struct seq_file *seq, void *v) } seq_show_srpc_rules(seq, fsdb->fsdb_name, &fsdb->fsdb_srpc_gen); - seq_printf(seq, "\nImperative Recovery Status:\n"); - lprocfs_rd_ir_state(seq, fsdb); cfs_up(&fsdb->fsdb_sem); @@ -270,6 +268,7 @@ struct lprocfs_vars lprocfs_mgs_obd_vars[] = { { "num_exports", lprocfs_rd_num_exports, 0, 0 }, { "hash_stats", lprocfs_obd_rd_hash, 0, 0 }, { "evict_client", 0, lprocfs_wr_evict_client, 0 }, + { "ir_timeout", lprocfs_rd_ir_timeout, lprocfs_wr_ir_timeout, 0 }, { 0 } }; diff --git a/lustre/mgs/mgs_nids.c b/lustre/mgs/mgs_nids.c index 587a9d0..594256b 100644 --- a/lustre/mgs/mgs_nids.c +++ b/lustre/mgs/mgs_nids.c @@ -424,11 +424,11 @@ static int mgs_ir_notify(void *arg) /* do statistic */ fsdb->fsdb_notify_count++; - delta = (cfs_time_current() - curtime) / NSEC_PER_USEC; + delta = cfs_time_current() - curtime; fsdb->fsdb_notify_total += delta; if (delta > fsdb->fsdb_notify_max) fsdb->fsdb_notify_max = delta; - CDEBUG(D_MGS, "Revoke recover lock of %s spent %dus\n", + CDEBUG(D_MGS, "Revoke recover lock of %s %dT\n", fsdb->fsdb_name, delta); } else { CERROR("Fatal error %d for fs %s\n", @@ -789,18 +789,30 @@ int lprocfs_rd_ir_state(struct seq_file *seq, void *data) struct fs_db *fsdb = data; struct mgs_nidtbl *tbl = &fsdb->fsdb_nidtbl; const char *ir_strings[] = IR_STRINGS; + struct timeval tv_max; + struct timeval tv; /* mgs_live_seq_show() already holds fsdb_sem. */ ir_state_graduate(fsdb); + seq_printf(seq, "\nimperative_recovery_state:\n"); seq_printf(seq, - "\tstate: %s, nonir clients: %d\n" - "\tnidtbl version: %lld\n", + " state: %s\n" + " nonir_clients: %d\n" + " nidtbl_version: %lld\n", ir_strings[fsdb->fsdb_ir_state], fsdb->fsdb_nonir_clients, tbl->mn_version); - seq_printf(seq, "\tnotify total/max/count: %u/%u/%u\n", - fsdb->fsdb_notify_total, fsdb->fsdb_notify_max, + + cfs_duration_usec(fsdb->fsdb_notify_total, &tv); + cfs_duration_usec(fsdb->fsdb_notify_max, &tv_max); + + seq_printf(seq, " notify_duration_total: %lu.%06lu\n" + " notify_duation_max: %lu.%06lu\n" + " notify_count: %u\n", + tv.tv_sec, tv.tv_usec, + tv_max.tv_sec, tv_max.tv_usec, fsdb->fsdb_notify_count); + return 0; } diff --git a/lustre/osc/lproc_osc.c b/lustre/osc/lproc_osc.c index 7da9621..d0a41fc 100644 --- a/lustre/osc/lproc_osc.c +++ b/lustre/osc/lproc_osc.c @@ -610,6 +610,8 @@ static struct lprocfs_vars lprocfs_osc_obd_vars[] = { osc_wr_lockless_truncate, 0 }, { "import", lprocfs_rd_import, lprocfs_wr_import, 0 }, { "state", lprocfs_rd_state, 0, 0 }, + { "pinger_recov", lprocfs_rd_pinger_recov, + lprocfs_wr_pinger_recov, 0, 0 }, { 0 } }; diff --git a/lustre/ptlrpc/lproc_ptlrpc.c b/lustre/ptlrpc/lproc_ptlrpc.c index 94ec060..2b5cefb 100644 --- a/lustre/ptlrpc/lproc_ptlrpc.c +++ b/lustre/ptlrpc/lproc_ptlrpc.c @@ -866,4 +866,45 @@ out: } EXPORT_SYMBOL(lprocfs_wr_import); +int lprocfs_rd_pinger_recov(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct obd_device *obd = data; + struct obd_import *imp = obd->u.cli.cl_import; + int rc; + + LPROCFS_CLIMP_CHECK(obd); + rc = snprintf(page, count, "%d\n", !imp->imp_no_pinger_recover); + LPROCFS_CLIMP_EXIT(obd); + + return rc; +} +EXPORT_SYMBOL(lprocfs_rd_pinger_recov); + +int lprocfs_wr_pinger_recov(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct obd_device *obd = data; + struct client_obd *cli = &obd->u.cli; + struct obd_import *imp = cli->cl_import; + int rc, val; + + rc = lprocfs_write_helper(buffer, count, &val); + if (rc < 0) + return rc; + + if (val != 0 && val != 1) + return -ERANGE; + + LPROCFS_CLIMP_CHECK(obd); + cfs_spin_lock(&imp->imp_lock); + imp->imp_no_pinger_recover = !val; + cfs_spin_unlock(&imp->imp_lock); + LPROCFS_CLIMP_EXIT(obd); + + return count; + +} +EXPORT_SYMBOL(lprocfs_wr_pinger_recov); + #endif /* LPROCFS */ diff --git a/lustre/ptlrpc/pinger.c b/lustre/ptlrpc/pinger.c index 8251636..a9bc232 100644 --- a/lustre/ptlrpc/pinger.c +++ b/lustre/ptlrpc/pinger.c @@ -246,9 +246,10 @@ static void ptlrpc_pinger_process_import(struct obd_import *imp, return; if (level == LUSTRE_IMP_DISCON && !imp_is_deactive(imp)) { - /* wait at least a timeout before trying recovery again */ + /* wait for a while before trying recovery again */ imp->imp_next_ping = ptlrpc_next_reconnect(imp); - ptlrpc_initiate_recovery(imp); + if (!imp->imp_no_pinger_recover) + ptlrpc_initiate_recovery(imp); } else if (level != LUSTRE_IMP_FULL || imp->imp_obd->obd_no_recov || imp_is_deactive(imp)) { diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh index 36f8c82..f9bbf42 100755 --- a/lustre/tests/recovery-small.sh +++ b/lustre/tests/recovery-small.sh @@ -1100,6 +1100,291 @@ test_61() } run_test 61 "Verify to not reuse orphan objects - bug 17025" +check_cli_ir_state() +{ + local NODE=${1:-`hostname`} + local st + st=$(do_node $NODE "lctl get_param mgc.*.ir_state | + awk '/imperative_recovery:/ { print \\\$2}' ") + [ $st != ON -o $st != OFF ] || + error "Error state $st, must be ON or OFF" + echo -n $st +} + +check_target_ir_state() +{ + local target=${1} + local name=${target}_svc + local recovery_proc=obdfilter.${!name}.recovery_status + local st + + st=$(do_facet $target "lctl get_param -n $recovery_proc | + awk '/IR:/{ print \\\$2}'") + [ $st != ON -o $st != OFF ] || + error "Error state $st, must be ON or OFF" + echo -n $st +} + +set_ir_status() +{ + do_facet mgs lctl set_param -n mgs.MGS.live.$FSNAME="state=$1" +} + +get_ir_status() +{ + local state=$(do_facet mgs "lctl get_param -n mgs.MGS.live.$FSNAME | + awk '/state:/{ print \\\$2 }'") + echo -n ${state/,/} +} + +nidtbl_version_mgs() +{ + local ver=$(do_facet mgs "lctl get_param -n mgs.MGS.live.$FSNAME | + awk '/nidtbl_version:/{ print \\\$2 }'") + echo -n $ver +} + +# nidtbl_version_client [node] +nidtbl_version_client() +{ + local cli=$1 + local node=${2:-`hostname`} + + if [ X$cli = Xclient ]; then + cli=$FSNAME-client + else + local obdtype=${cli/%[0-9]*/} + [ $obdtype != mds ] && error "wrong parameters $cli" + + node=`facet_active_host $cli` + local t=${cli}_svc + cli=${!t} + fi + + local vers=$(do_node $node "lctl get_param -n mgc.*.ir_state" | + awk "/$cli/{print \$6}" |sort -u) + + # in case there are multiple mounts on the client node + local arr=($vers) + [ ${#arr[@]} -ne 1 ] && error "versions on client node mismatch" + echo -n $vers +} + +nidtbl_versions_match() +{ + [ `nidtbl_version_mgs` -eq `nidtbl_version_client ${1:-client}` ] +} + +target_instance_match() +{ + local srv=$1 + local obdtype + local cliname + + obdtype=${srv/%[0-9]*/} + case $obdtype in + mds) + obdname="mdt" + cliname="mdc" + ;; + ost) + obdname="obdfilter" + cliname="osc" + ;; + *) + error "invalid target type" $srv + return 1 + ;; + esac + + local target=${srv}_svc + local si=`do_facet $srv lctl get_param -n $obdname.${!target}.instance` + local ci=`lctl get_param -n $cliname.${!target}-${cliname}-*.import | \ + awk '/instance/{ print $2 }' |head -1` + + return `[ $si -eq $ci ]` +} + +test_100() +{ + # disable IR + set_ir_status disabled + + local saved_FAILURE_MODE=$FAILURE_MODE + [ `facet_host mgs` = `facet_host ost1` ] && FAILURE_MODE="SOFT" + fail ost1 + + # valid check + nidtbl_versions_match && + error "version must differ due to IR disabled" + target_instance_match ost1 || error "instance mismatch" + + # restore env + set_ir_status full + FAILURE_MODE=$saved_FAILURE_MODE +} +run_test 100 "IR: Make sure normal recovery still works w/o IR" + +test_101() +{ + set_ir_status full + + local OST1_IMP=$(get_osc_import_name client ost1) + + # disable pinger recovery + lctl set_param -n osc.$OST1_IMP.pinger_recov=0 + + fail ost1 + + target_instance_match ost1 || error "instance mismatch" + nidtbl_versions_match || error "version must match" + + lctl set_param -n osc.$OST1_IMP.pinger_recov=1 +} +run_test 101 "IR: Make sure IR works w/o normal recovery" + +test_102() +{ + local clients=${CLIENTS:-`hostname`} + local old_version + local new_version + local mgsdev=mgs + + set_ir_status full + + # let's have a new nidtbl version + fail ost1 + + # sleep for a while so that clients can see the failure of ost + # it must be MGC_TIMEOUT_MIN_SECONDS + MGC_TIMEOUT_RAND_CENTISEC. + # int mgc_request.c: + # define MGC_TIMEOUT_MIN_SECONDS 5 + # define MGC_TIMEOUT_RAND_CENTISEC 0x1ff /* ~500 * + local count=30 # 20 seconds at most + while [ $count -gt 0 ]; do + nidtbl_versions_match && break + sleep 1 + count=$((count-1)) + done + + nidtbl_versions_match || error "nidtbl mismatch" + + # get the version # + old_version=`nidtbl_version_client client` + + zconf_umount_clients $clients $MOUNT || error "Cannot umount client" + + # restart mgs + combined_mgs_mds && mgsdev=mds1 + remount_facet $mgsdev + fail ost1 + + zconf_mount_clients $clients $MOUNT || error "Cannot mount client" + + # check new version + new_version=`nidtbl_version_client client` + [ $new_version -lt $old_version ] && + error "nidtbl version wrong after mgs restarts" + return 0 +} +run_test 102 "IR: New client gets updated nidtbl after MGS restart" + +test_103() +{ + combined_mgs_mds && skip "mgs and mds on the same target" && return 0 + + # workaround solution to generate config log on the mds + remount_facet mds1 + + stop mgs + stop mds1 + + # We need this test because mds is like a client in IR context. + start mds1 $MDSDEV1 || error "MDS should start w/o mgs" + + # start mgs and remount mds w/ ir + start mgs $MGSDEV + clients_up + + # remount client so that fsdb will be created on the MGS + umount_client $MOUNT || error "umount failed" + mount_client $MOUNT || error "mount failed" + + # sleep 30 seconds so the MDS has a chance to detect MGS restarting + local count=30 + while [ $count -gt 0 ]; do + [ `nidtbl_version_client mds1` -ne 0 ] && break + sleep 1 + count=$((count-1)) + done + + # after a while, mds should be able to reconnect to mgs and fetch + # up-to-date nidtbl version + nidtbl_versions_match mds1 || error "mds nidtbl mismatch" + + # reset everything + set_ir_status full +} +run_test 103 "IR: MDS can start w/o MGS and get updated nidtbl later" + +test_104() +{ + set_ir_status full + + stop ost1 + start ost1 `ostdevname 1` "$OST_MOUNT_OPTS -onoir" || + error "OST1 cannot start" + clients_up + + local ir_state=$(check_target_ir_state ost1) + [ $ir_state = "OFF" ] || error "ir status on ost1 should be OFF" + ost1_opt= +} +run_test 104 "IR: ost can disable IR voluntarily" + +test_105() +{ + [ -z "$RCLIENTS" ] && skip "Needs multiple clients" && return 0 + + set_ir_status full + + # get one of the clients from client list + rcli=`echo $RCLIENTS |cut -d' ' -f 1` + + local old_MOUNTOPT=$MOUNTOPT + MOUNTOPT=${MOUNTOPT},noir + zconf_umount $rcli $MOUNT || error "umount failed" + zconf_mount $rcli $MOUNT || error "mount failed" + + # make sure lustre mount at $rcli disabling IR + local ir_state=$(check_cli_ir_state $rcli) + [ $ir_state = OFF ] || error "IR state must be OFF at $rcli" + + # make sure MGS's state is Partial + [ `get_ir_status` = "partial" ] || error "MGS IR state must be partial" + + fail ost1 + # make sure IR on ost1 is OFF + local ir_state=$(check_target_ir_state ost1) + [ $ir_state = "OFF" ] || error "IR status on ost1 should be OFF" + + # restore it + MOUNTOPT=$old_MOUNTOPT + zconf_umount $rcli $MOUNT || error "umount failed" + zconf_mount $rcli $MOUNT || error "mount failed" + + # make sure MGS's state is full + [ `get_ir_status` = "full" ] || error "MGS IR status must be full" + + fail ost1 + # make sure IR on ost1 is ON + local ir_state=$(check_target_ir_state ost1) + [ $ir_state = "ON" ] || error "IR status on ost1 should be OFF" + + return 0 +} +run_test 105 "IR: NON IR clients support" + complete $(basename $0) $SECONDS check_and_cleanup_lustre exit_status diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index cba9fd0..b26be11 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -3780,10 +3780,7 @@ convert_facet2label() { } get_clientosc_proc_path() { - local ost=$1 - - # exclude -osc-M* - echo "${1}-osc-[!M]*" + echo "${1}-osc-[^M]*" } get_lustre_version () {