In this patch, some procfs and test cases are added for imperative recovery.
Signed-off-by: Jinshan Xiong <jay@whamcloud.com>
Change-Id: Ied49b622208157a74e8547e4609d61ee5041f624
Reviewed-on: http://review.whamcloud.com/1219
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Yu Jian <yujian@whamcloud.com>
unsigned long count, void *data);
extern int lprocfs_wr_import(struct file *file, const char *buffer,
unsigned long count, void *data);
+extern int lprocfs_rd_pinger_recov(char *page, char **start, off_t off,
+ int count, int *eof, void *data);
+extern int lprocfs_wr_pinger_recov(struct file *file, const char *buffer,
+ unsigned long count, void *data);
/* Statfs helpers */
extern int lprocfs_rd_blksize(char *page, char **start, off_t off,
static inline int lprocfs_rd_import(char *page, char **start, off_t off,
int count, int *eof, void *data)
{ return 0; }
+static inline int lprocfs_rd_pinger_recov(char *page, char **start, off_t off,
+ int count, int *eof, void *data)
+{ return 0; }
static inline int lprocfs_rd_state(char *page, char **start, off_t off,
int count, int *eof, void *data)
{ return 0; }
static inline int lprocfs_wr_import(struct file *file, const char *buffer,
unsigned long count, void *data)
{ return 0; }
+static inline int lprocfs_wr_pinger_recov(struct file *file, const char *buffer,
+ unsigned long count, void *data)
+{ return 0; }
/* Statfs helpers */
static inline
imp_force_verify:1, /* force an immidiate ping */
imp_pingable:1, /* pingable */
imp_resend_replay:1, /* resend for replay */
+ imp_no_pinger_recover:1,/* disable normal recovery, for test only. */
imp_force_reconnect:1; /* import must be reconnected instead of chouse new connection */
__u32 imp_connect_op;
struct obd_connect_data imp_connect_data;
{ "import", lprocfs_rd_import, lprocfs_wr_import, 0 },
{ "state", lprocfs_rd_state, 0, 0 },
{ "hsm_nl", 0, mdc_wr_kuc, 0, 0, 0222 },
+ { "pinger_recov", lprocfs_rd_pinger_recov,
+ lprocfs_wr_pinger_recov, 0, 0 },
{ 0 }
};
#include <linux/vfs.h>
#include <obd_class.h>
#include <lprocfs_status.h>
+#include "mgc_internal.h"
#ifdef LPROCFS
{ "mgs_conn_uuid", lprocfs_rd_conn_uuid, 0, 0 },
{ "import", lprocfs_rd_import, 0, 0 },
{ "state", lprocfs_rd_state, 0, 0 },
+ { "ir_state", lprocfs_mgc_rd_ir_state, 0, 0 },
{ 0 }
};
#ifdef LPROCFS
void lprocfs_mgc_init_vars(struct lprocfs_static_vars *lvars);
+int lprocfs_mgc_rd_ir_state(char *page, char **start, off_t off,
+ int count, int *eof, void *data);
#else
static void lprocfs_mgc_init_vars(struct lprocfs_static_vars *lvars)
{
memset(lvars, 0, sizeof(*lvars));
}
+static inline int lprocfs_mgc_rd_ir_state(char *page, char **start,
+ off_t off, int count, int *eof, void *data)
+{
+ return 0;
+}
#endif /* LPROCFS */
int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld);
RETURN(rc);
}
+int lprocfs_mgc_rd_ir_state(char *page, char **start, off_t off,
+ int count, int *eof, void *data)
+{
+ struct obd_device *obd = data;
+ struct obd_import *imp = obd->u.cli.cl_import;
+ struct obd_connect_data *ocd = &imp->imp_connect_data;
+ struct config_llog_data *cld;
+ int rc = 0;
+ ENTRY;
+
+ rc = snprintf(page, count, "imperative_recovery: %s\n",
+ OCD_HAS_FLAG(ocd, IMP_RECOV) ? "ON" : "OFF");
+ rc += snprintf(page + rc, count - rc, "client_state:\n");
+
+ cfs_spin_lock(&config_list_lock);
+ cfs_list_for_each_entry(cld, &config_llog_list, cld_list_chain) {
+ if (cld->cld_recover == NULL)
+ continue;
+ rc += snprintf(page + rc, count - rc,
+ " - { client: %s, nidtbl_version: %u }\n",
+ cld->cld_logname,
+ cld->cld_recover->cld_cfg.cfg_last_idx);
+ }
+ cfs_spin_unlock(&config_list_lock);
+
+ RETURN(rc);
+}
+
/* reenqueue any lost locks */
#define RQ_RUNNING 0x1
#define RQ_NOW 0x2
}
seq_show_srpc_rules(seq, fsdb->fsdb_name, &fsdb->fsdb_srpc_gen);
- seq_printf(seq, "\nImperative Recovery Status:\n");
-
lprocfs_rd_ir_state(seq, fsdb);
cfs_up(&fsdb->fsdb_sem);
{ "num_exports", lprocfs_rd_num_exports, 0, 0 },
{ "hash_stats", lprocfs_obd_rd_hash, 0, 0 },
{ "evict_client", 0, lprocfs_wr_evict_client, 0 },
+ { "ir_timeout", lprocfs_rd_ir_timeout, lprocfs_wr_ir_timeout, 0 },
{ 0 }
};
/* do statistic */
fsdb->fsdb_notify_count++;
- delta = (cfs_time_current() - curtime) / NSEC_PER_USEC;
+ delta = cfs_time_current() - curtime;
fsdb->fsdb_notify_total += delta;
if (delta > fsdb->fsdb_notify_max)
fsdb->fsdb_notify_max = delta;
- CDEBUG(D_MGS, "Revoke recover lock of %s spent %dus\n",
+ CDEBUG(D_MGS, "Revoke recover lock of %s %dT\n",
fsdb->fsdb_name, delta);
} else {
CERROR("Fatal error %d for fs %s\n",
struct fs_db *fsdb = data;
struct mgs_nidtbl *tbl = &fsdb->fsdb_nidtbl;
const char *ir_strings[] = IR_STRINGS;
+ struct timeval tv_max;
+ struct timeval tv;
/* mgs_live_seq_show() already holds fsdb_sem. */
ir_state_graduate(fsdb);
+ seq_printf(seq, "\nimperative_recovery_state:\n");
seq_printf(seq,
- "\tstate: %s, nonir clients: %d\n"
- "\tnidtbl version: %lld\n",
+ " state: %s\n"
+ " nonir_clients: %d\n"
+ " nidtbl_version: %lld\n",
ir_strings[fsdb->fsdb_ir_state], fsdb->fsdb_nonir_clients,
tbl->mn_version);
- seq_printf(seq, "\tnotify total/max/count: %u/%u/%u\n",
- fsdb->fsdb_notify_total, fsdb->fsdb_notify_max,
+
+ cfs_duration_usec(fsdb->fsdb_notify_total, &tv);
+ cfs_duration_usec(fsdb->fsdb_notify_max, &tv_max);
+
+ seq_printf(seq, " notify_duration_total: %lu.%06lu\n"
+ " notify_duation_max: %lu.%06lu\n"
+ " notify_count: %u\n",
+ tv.tv_sec, tv.tv_usec,
+ tv_max.tv_sec, tv_max.tv_usec,
fsdb->fsdb_notify_count);
+
return 0;
}
osc_wr_lockless_truncate, 0 },
{ "import", lprocfs_rd_import, lprocfs_wr_import, 0 },
{ "state", lprocfs_rd_state, 0, 0 },
+ { "pinger_recov", lprocfs_rd_pinger_recov,
+ lprocfs_wr_pinger_recov, 0, 0 },
{ 0 }
};
}
EXPORT_SYMBOL(lprocfs_wr_import);
+int lprocfs_rd_pinger_recov(char *page, char **start, off_t off,
+ int count, int *eof, void *data)
+{
+ struct obd_device *obd = data;
+ struct obd_import *imp = obd->u.cli.cl_import;
+ int rc;
+
+ LPROCFS_CLIMP_CHECK(obd);
+ rc = snprintf(page, count, "%d\n", !imp->imp_no_pinger_recover);
+ LPROCFS_CLIMP_EXIT(obd);
+
+ return rc;
+}
+EXPORT_SYMBOL(lprocfs_rd_pinger_recov);
+
+int lprocfs_wr_pinger_recov(struct file *file, const char *buffer,
+ unsigned long count, void *data)
+{
+ struct obd_device *obd = data;
+ struct client_obd *cli = &obd->u.cli;
+ struct obd_import *imp = cli->cl_import;
+ int rc, val;
+
+ rc = lprocfs_write_helper(buffer, count, &val);
+ if (rc < 0)
+ return rc;
+
+ if (val != 0 && val != 1)
+ return -ERANGE;
+
+ LPROCFS_CLIMP_CHECK(obd);
+ cfs_spin_lock(&imp->imp_lock);
+ imp->imp_no_pinger_recover = !val;
+ cfs_spin_unlock(&imp->imp_lock);
+ LPROCFS_CLIMP_EXIT(obd);
+
+ return count;
+
+}
+EXPORT_SYMBOL(lprocfs_wr_pinger_recov);
+
#endif /* LPROCFS */
return;
if (level == LUSTRE_IMP_DISCON && !imp_is_deactive(imp)) {
- /* wait at least a timeout before trying recovery again */
+ /* wait for a while before trying recovery again */
imp->imp_next_ping = ptlrpc_next_reconnect(imp);
- ptlrpc_initiate_recovery(imp);
+ if (!imp->imp_no_pinger_recover)
+ ptlrpc_initiate_recovery(imp);
} else if (level != LUSTRE_IMP_FULL ||
imp->imp_obd->obd_no_recov ||
imp_is_deactive(imp)) {
}
run_test 61 "Verify to not reuse orphan objects - bug 17025"
+check_cli_ir_state()
+{
+ local NODE=${1:-`hostname`}
+ local st
+ st=$(do_node $NODE "lctl get_param mgc.*.ir_state |
+ awk '/imperative_recovery:/ { print \\\$2}' ")
+ [ $st != ON -o $st != OFF ] ||
+ error "Error state $st, must be ON or OFF"
+ echo -n $st
+}
+
+check_target_ir_state()
+{
+ local target=${1}
+ local name=${target}_svc
+ local recovery_proc=obdfilter.${!name}.recovery_status
+ local st
+
+ st=$(do_facet $target "lctl get_param -n $recovery_proc |
+ awk '/IR:/{ print \\\$2}'")
+ [ $st != ON -o $st != OFF ] ||
+ error "Error state $st, must be ON or OFF"
+ echo -n $st
+}
+
+set_ir_status()
+{
+ do_facet mgs lctl set_param -n mgs.MGS.live.$FSNAME="state=$1"
+}
+
+get_ir_status()
+{
+ local state=$(do_facet mgs "lctl get_param -n mgs.MGS.live.$FSNAME |
+ awk '/state:/{ print \\\$2 }'")
+ echo -n ${state/,/}
+}
+
+nidtbl_version_mgs()
+{
+ local ver=$(do_facet mgs "lctl get_param -n mgs.MGS.live.$FSNAME |
+ awk '/nidtbl_version:/{ print \\\$2 }'")
+ echo -n $ver
+}
+
+# nidtbl_version_client <mds1|client> [node]
+nidtbl_version_client()
+{
+ local cli=$1
+ local node=${2:-`hostname`}
+
+ if [ X$cli = Xclient ]; then
+ cli=$FSNAME-client
+ else
+ local obdtype=${cli/%[0-9]*/}
+ [ $obdtype != mds ] && error "wrong parameters $cli"
+
+ node=`facet_active_host $cli`
+ local t=${cli}_svc
+ cli=${!t}
+ fi
+
+ local vers=$(do_node $node "lctl get_param -n mgc.*.ir_state" |
+ awk "/$cli/{print \$6}" |sort -u)
+
+ # in case there are multiple mounts on the client node
+ local arr=($vers)
+ [ ${#arr[@]} -ne 1 ] && error "versions on client node mismatch"
+ echo -n $vers
+}
+
+nidtbl_versions_match()
+{
+ [ `nidtbl_version_mgs` -eq `nidtbl_version_client ${1:-client}` ]
+}
+
+target_instance_match()
+{
+ local srv=$1
+ local obdtype
+ local cliname
+
+ obdtype=${srv/%[0-9]*/}
+ case $obdtype in
+ mds)
+ obdname="mdt"
+ cliname="mdc"
+ ;;
+ ost)
+ obdname="obdfilter"
+ cliname="osc"
+ ;;
+ *)
+ error "invalid target type" $srv
+ return 1
+ ;;
+ esac
+
+ local target=${srv}_svc
+ local si=`do_facet $srv lctl get_param -n $obdname.${!target}.instance`
+ local ci=`lctl get_param -n $cliname.${!target}-${cliname}-*.import | \
+ awk '/instance/{ print $2 }' |head -1`
+
+ return `[ $si -eq $ci ]`
+}
+
+test_100()
+{
+ # disable IR
+ set_ir_status disabled
+
+ local saved_FAILURE_MODE=$FAILURE_MODE
+ [ `facet_host mgs` = `facet_host ost1` ] && FAILURE_MODE="SOFT"
+ fail ost1
+
+ # valid check
+ nidtbl_versions_match &&
+ error "version must differ due to IR disabled"
+ target_instance_match ost1 || error "instance mismatch"
+
+ # restore env
+ set_ir_status full
+ FAILURE_MODE=$saved_FAILURE_MODE
+}
+run_test 100 "IR: Make sure normal recovery still works w/o IR"
+
+test_101()
+{
+ set_ir_status full
+
+ local OST1_IMP=$(get_osc_import_name client ost1)
+
+ # disable pinger recovery
+ lctl set_param -n osc.$OST1_IMP.pinger_recov=0
+
+ fail ost1
+
+ target_instance_match ost1 || error "instance mismatch"
+ nidtbl_versions_match || error "version must match"
+
+ lctl set_param -n osc.$OST1_IMP.pinger_recov=1
+}
+run_test 101 "IR: Make sure IR works w/o normal recovery"
+
+test_102()
+{
+ local clients=${CLIENTS:-`hostname`}
+ local old_version
+ local new_version
+ local mgsdev=mgs
+
+ set_ir_status full
+
+ # let's have a new nidtbl version
+ fail ost1
+
+ # sleep for a while so that clients can see the failure of ost
+ # it must be MGC_TIMEOUT_MIN_SECONDS + MGC_TIMEOUT_RAND_CENTISEC.
+ # int mgc_request.c:
+ # define MGC_TIMEOUT_MIN_SECONDS 5
+ # define MGC_TIMEOUT_RAND_CENTISEC 0x1ff /* ~500 *
+ local count=30 # 20 seconds at most
+ while [ $count -gt 0 ]; do
+ nidtbl_versions_match && break
+ sleep 1
+ count=$((count-1))
+ done
+
+ nidtbl_versions_match || error "nidtbl mismatch"
+
+ # get the version #
+ old_version=`nidtbl_version_client client`
+
+ zconf_umount_clients $clients $MOUNT || error "Cannot umount client"
+
+ # restart mgs
+ combined_mgs_mds && mgsdev=mds1
+ remount_facet $mgsdev
+ fail ost1
+
+ zconf_mount_clients $clients $MOUNT || error "Cannot mount client"
+
+ # check new version
+ new_version=`nidtbl_version_client client`
+ [ $new_version -lt $old_version ] &&
+ error "nidtbl version wrong after mgs restarts"
+ return 0
+}
+run_test 102 "IR: New client gets updated nidtbl after MGS restart"
+
+test_103()
+{
+ combined_mgs_mds && skip "mgs and mds on the same target" && return 0
+
+ # workaround solution to generate config log on the mds
+ remount_facet mds1
+
+ stop mgs
+ stop mds1
+
+ # We need this test because mds is like a client in IR context.
+ start mds1 $MDSDEV1 || error "MDS should start w/o mgs"
+
+ # start mgs and remount mds w/ ir
+ start mgs $MGSDEV
+ clients_up
+
+ # remount client so that fsdb will be created on the MGS
+ umount_client $MOUNT || error "umount failed"
+ mount_client $MOUNT || error "mount failed"
+
+ # sleep 30 seconds so the MDS has a chance to detect MGS restarting
+ local count=30
+ while [ $count -gt 0 ]; do
+ [ `nidtbl_version_client mds1` -ne 0 ] && break
+ sleep 1
+ count=$((count-1))
+ done
+
+ # after a while, mds should be able to reconnect to mgs and fetch
+ # up-to-date nidtbl version
+ nidtbl_versions_match mds1 || error "mds nidtbl mismatch"
+
+ # reset everything
+ set_ir_status full
+}
+run_test 103 "IR: MDS can start w/o MGS and get updated nidtbl later"
+
+test_104()
+{
+ set_ir_status full
+
+ stop ost1
+ start ost1 `ostdevname 1` "$OST_MOUNT_OPTS -onoir" ||
+ error "OST1 cannot start"
+ clients_up
+
+ local ir_state=$(check_target_ir_state ost1)
+ [ $ir_state = "OFF" ] || error "ir status on ost1 should be OFF"
+ ost1_opt=
+}
+run_test 104 "IR: ost can disable IR voluntarily"
+
+test_105()
+{
+ [ -z "$RCLIENTS" ] && skip "Needs multiple clients" && return 0
+
+ set_ir_status full
+
+ # get one of the clients from client list
+ rcli=`echo $RCLIENTS |cut -d' ' -f 1`
+
+ local old_MOUNTOPT=$MOUNTOPT
+ MOUNTOPT=${MOUNTOPT},noir
+ zconf_umount $rcli $MOUNT || error "umount failed"
+ zconf_mount $rcli $MOUNT || error "mount failed"
+
+ # make sure lustre mount at $rcli disabling IR
+ local ir_state=$(check_cli_ir_state $rcli)
+ [ $ir_state = OFF ] || error "IR state must be OFF at $rcli"
+
+ # make sure MGS's state is Partial
+ [ `get_ir_status` = "partial" ] || error "MGS IR state must be partial"
+
+ fail ost1
+ # make sure IR on ost1 is OFF
+ local ir_state=$(check_target_ir_state ost1)
+ [ $ir_state = "OFF" ] || error "IR status on ost1 should be OFF"
+
+ # restore it
+ MOUNTOPT=$old_MOUNTOPT
+ zconf_umount $rcli $MOUNT || error "umount failed"
+ zconf_mount $rcli $MOUNT || error "mount failed"
+
+ # make sure MGS's state is full
+ [ `get_ir_status` = "full" ] || error "MGS IR status must be full"
+
+ fail ost1
+ # make sure IR on ost1 is ON
+ local ir_state=$(check_target_ir_state ost1)
+ [ $ir_state = "ON" ] || error "IR status on ost1 should be OFF"
+
+ return 0
+}
+run_test 105 "IR: NON IR clients support"
+
complete $(basename $0) $SECONDS
check_and_cleanup_lustre
exit_status
}
get_clientosc_proc_path() {
- local ost=$1
-
- # exclude -osc-M*
- echo "${1}-osc-[!M]*"
+ echo "${1}-osc-[^M]*"
}
get_lustre_version () {
git://git.whamcloud.com / fs / lustre-release.git / commitdiff