From dea72cdb5ce72eda8515da507ab604eb36c69550 Mon Sep 17 00:00:00 2001 From: shadow Date: Thu, 23 Apr 2009 18:14:02 +0000 Subject: [PATCH] port lazystatfs to HEAD. Branch HEAD b=17974 i=rread --- lustre/ChangeLog | 6 ++ lustre/llite/llite_internal.h | 1 + lustre/llite/llite_lib.c | 16 ++++ lustre/llite/lproc_llite.c | 30 +++++++ lustre/lov/lov_request.c | 5 +- lustre/osc/osc_request.c | 4 + lustre/tests/conf-sanity.sh | 199 ++++++++++++++++++++++++++++++++++++++++++ lustre/tests/multiop.c | 10 +++ 8 files changed, 269 insertions(+), 2 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index a9c1b8b..f5fb5a7 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -13,6 +13,12 @@ tbd Sun Microsystems, Inc. removed cwd "./" (refer to Bugzilla 14399). * File join has been disabled in this release, refer to Bugzilla 16929. +Severity : enhancement +Bugzilla : 17974 +Description: add lazystatfs mount option to allow statfs(2) to skip down OSTs +Details : allow skip disconnected ost for send statfs request and hide error + in this case. + Severity : major Frequency : rare Bugzilla : 18810 diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 2193873..2857837 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -297,6 +297,7 @@ enum stats_track_type { #define LL_SBI_OSS_CAPA 0x100 /* support oss capa */ #define LL_SBI_LOCALFLOCK 0x200 /* Local flocks support by kernel */ #define LL_SBI_LRU_RESIZE 0x400 /* lru resize support */ +#define LL_SBI_LAZYSTATFS 0x800 /* lazystatfs mount option */ /* default value for ll_sb_info->contention_time */ #define SBI_DEFAULT_CONTENTION_SECONDS 60 diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index e780458..bda1337 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -833,6 +833,16 @@ static int ll_options(char *options, int *flags) *flags &= ~tmp; goto next; } + tmp = ll_set_opt("lazystatfs", s1, LL_SBI_LAZYSTATFS); + if (tmp) { + *flags |= tmp; + goto next; + } + tmp = ll_set_opt("nolazystatfs", s1, LL_SBI_LAZYSTATFS); + if (tmp) { + *flags &= ~tmp; + goto next; + } LCONSOLE_ERROR_MSG(0x152, "Unknown option '%s', won't mount.\n", s1); @@ -1392,6 +1402,9 @@ int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs, CDEBUG(D_SUPER, "MDC blocks "LPU64"/"LPU64" objects "LPU64"/"LPU64"\n", osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files); + if (sbi->ll_flags & LL_SBI_LAZYSTATFS) + flags |= OBD_STATFS_NODELAY; + rc = obd_statfs_rqset(class_exp2obd(sbi->ll_dt_exp), &obd_osfs, max_age, flags); if (rc) { @@ -2153,5 +2166,8 @@ int ll_show_options(struct seq_file *seq, struct vfsmount *vfs) if (sbi->ll_flags & LL_SBI_ACL) seq_puts(seq, ",acl"); + if (sbi->ll_flags & LL_SBI_LAZYSTATFS) + seq_puts(seq, ",lazystatfs"); + RETURN(0); } diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c index cd077aa..98b4c96 100644 --- a/lustre/llite/lproc_llite.c +++ b/lustre/llite/lproc_llite.c @@ -533,6 +533,35 @@ static int ll_rd_statahead_stats(char *page, char **start, off_t off, sbi->ll_sa_miss); } +static int ll_rd_lazystatfs(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct super_block *sb = data; + struct ll_sb_info *sbi = ll_s2sbi(sb); + + return snprintf(page, count, "%u\n", + (sbi->ll_flags & LL_SBI_LAZYSTATFS) ? 1 : 0); +} + +static int ll_wr_lazystatfs(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct super_block *sb = data; + struct ll_sb_info *sbi = ll_s2sbi(sb); + int val, rc; + + rc = lprocfs_write_helper(buffer, count, &val); + if (rc) + return rc; + + if (val) + sbi->ll_flags |= LL_SBI_LAZYSTATFS; + else + sbi->ll_flags &= ~LL_SBI_LAZYSTATFS; + + return count; +} + static struct lprocfs_vars lprocfs_llite_obd_vars[] = { { "uuid", ll_rd_sb_uuid, 0, 0 }, //{ "mntpt_path", ll_rd_path, 0, 0 }, @@ -558,6 +587,7 @@ static struct lprocfs_vars lprocfs_llite_obd_vars[] = { { "stats_track_gid", ll_rd_track_gid, ll_wr_track_gid, 0 }, { "statahead_max", ll_rd_statahead_max, ll_wr_statahead_max, 0 }, { "statahead_stats", ll_rd_statahead_stats, 0, 0 }, + { "lazystatfs", ll_rd_lazystatfs, ll_wr_lazystatfs, 0 }, { 0 } }; diff --git a/lustre/lov/lov_request.c b/lustre/lov/lov_request.c index 706503f..d0580f3 100644 --- a/lustre/lov/lov_request.c +++ b/lustre/lov/lov_request.c @@ -1587,11 +1587,11 @@ static int cb_statfs_update(void *cookie, int rc) lov_sfs = oinfo->oi_osfs; success = lovreq->rq_rqset->set_success; - /* XXX: the same is done in lov_update_common_set, however lovset->set_exp is not initialized. */ lov_update_set(lovreq->rq_rqset, lovreq, rc); if (rc) { + /* XXX ignore error for disconnected ost ? */ if (rc && !(lov->lov_tgts[lovreq->rq_idx] && lov->lov_tgts[lovreq->rq_idx]->ltd_active)) rc = 0; @@ -1638,7 +1638,8 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo, for (i = 0; i < lov->desc.ld_tgt_count; i++) { struct lov_request *req; - if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active) { + if (!lov->lov_tgts[i] || (!lov->lov_tgts[i]->ltd_active + && (oinfo->oi_flags & OBD_STATFS_NODELAY))) { CDEBUG(D_HA, "lov idx %d inactive\n", i); continue; } diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index f44f44b..8c0a773 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3360,6 +3360,10 @@ static int osc_statfs_interpret(const struct lu_env *env, struct obd_statfs *msfs; ENTRY; + if ((rc == -ENOTCONN || rc == -EAGAIN) && + (aa->aa_oi->oi_flags & OBD_STATFS_NODELAY)) + GOTO(out, rc = 0); + if (rc != 0) GOTO(out, rc); diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 9a9b8fa..e6e48be 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -1753,6 +1753,205 @@ test_49() { # bug 17710 } run_test 49 "check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE" +lazystatfs() { + # Test both statfs and lfs df and fail if either one fails + multiop_bg_pause $1 f_ + RC1=$? + PID=$! + killall -USR1 multiop + [ $RC1 -ne 0 ] && log "lazystatfs multiop failed" + wait $PID || { RC1=$?; log "multiop return error "; } + + $LFS df & + PID=$! + sleep 5 + kill -s 0 $PID + RC2=$? + if [ $RC2 -eq 0 ]; then + kill -s 9 $PID + log "lazystatfs df failed" + fi + + RC=0 + [[ $RC1 -ne 0 || $RC2 -eq 0 ]] && RC=1 + return $RC +} + +test_50a() { + setup + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + lazystatfs $MOUNT || error "lazystatfs failed but no down servers" + + cleanup || return $? +} +run_test 50a "lazystatfs all servers available ==========================" + +test_50b() { + setup + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + # Wait for client to detect down OST + stop_ost || error "Unable to stop OST1" + CONN_PROC="osc.$FSNAME-OST0000-osc-[M]*.ost_server_uuid" + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ "${CONN_STATE}" = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + lazystatfs $MOUNT || error "lazystatfs should don't have returned EIO" + + umount_client $MOUNT || error "Unable to unmount client" + stop_mds || error "Unable to stop MDS" +} +run_test 50b "lazystatfs all servers down ==========================" + +test_50c() { + start_mds || error "Unable to start MDS" + start_ost || error "Unable to start OST1" + start_ost2 || error "Unable to start OST2" + mount_client $MOUNT || error "Unable to mount client" + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + # Wait for client to detect down OST + stop_ost || error "Unable to stop OST1" + CONN_PROC="osc.$FSNAME-OST0000-osc-[M]*.ost_server_uuid" + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + lazystatfs $MOUNT || error "lazystatfs failed with one down server" + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost2 || error "Unable to stop OST2" + stop_mds || error "Unable to stop MDS" +} +run_test 50c "lazystatfs one server down ==========================" + +test_50d() { + start_mds || error "Unable to start MDS" + start_ost || error "Unable to start OST1" + start_ost2 || error "Unable to start OST2" + mount_client $MOUNT || error "Unable to mount client" + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + # Issue the statfs during the window where the client still + # belives the OST to be available but it is in fact down. + # No failure just a statfs which hangs for a timeout interval. + stop_ost || error "Unable to stop OST1" + lazystatfs $MOUNT || error "lazystatfs failed with one down server" + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost2 || error "Unable to stop OST2" + stop_mds || error "Unable to stop MDS" +} +run_test 50d "lazystatfs client/server conn race ==========================" + +test_50e() { + local RC1 + local pid + CONN_PROC="osc.$FSNAME-OST0000-osc-[M]*.ost_server_uuid" + + reformat_and_config + start_mds || return 1 + #first client should see only one ost + start_ost || return 2 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ "${CONN_STATE}" != "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + lctl set_param llite.$FSNAME-*.lazystatfs=0 + + # Wait for client to detect down OST + stop_ost || error "Unable to stop OST1" + + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ "${CONN_STATE}" = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + mount_client $MOUNT || error "Unable to mount client" + + multiop_bg_pause $MOUNT _f + RC1=$? + pid=$! + + if [ $RC1 -ne 0 ]; then + log "multiop failed $RC1" + else + kill -USR1 $pid + sleep $(( $TIMEOUT+1 )) + kill -0 $pid + [ $? -ne 0 ] && error "process isn't sleep" + start_ost || error "Unable to start OST1" + wait $pid || error "statfs failed" + fi + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost || error "Unable to stop OST1" + stop_mds || error "Unable to stop MDS" +} +run_test 50e "normal statfs all servers down ==========================" + +test_50f() { + local RC1 + local pid + CONN_PROC="osc.$FSNAME-OST0001-osc-[M]*.ost_server_uuid" + + start_mds || error "Unable to start mds" + #first client should see only one ost + start_ost || error "Unable to start OST1" + start_ost2 || error "Unable to start OST2" + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ "${CONN_STATE}" != "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + lctl set_param llite.$FSNAME-*.lazystatfs=0 + + # Wait for client to detect down OST + stop_ost2 || error "Unable to stop OST2" + + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ "${CONN_STATE}" = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + mount_client $MOUNT || error "Unable to mount client" + + multiop_bg_pause $MOUNT _f + RC1=$? + pid=$! + + if [ $RC1 -ne 0 ]; then + log "lazystatfs multiop failed $RC1" + else + kill -USR1 $pid + sleep $(( $TIMEOUT+1 )) + kill -0 $pid + [ $? -ne 0 ] && error "process isn't sleep" + start_ost2 || error "Unable to start OST1" + wait $pid || error "statfs failed" + fi + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost || error "Unable to stop OST1" + stop_mds || error "Unable to stop MDS" + writeconf +} +run_test 50f "normal statfs one server in down ==========================" + + cleanup_gss equals_msg `basename $0`: test complete [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true diff --git a/lustre/tests/multiop.c b/lustre/tests/multiop.c index 86274f2..4923ac7 100755 --- a/lustre/tests/multiop.c +++ b/lustre/tests/multiop.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ char usage[] = " C[num] create with optional stripes\n" " d mkdir\n" " D open(O_DIRECTORY)\n" +" f statfs\n" " L link\n" " l symlink\n" " m mknod\n" @@ -186,6 +188,7 @@ int main(int argc, char **argv) char *fname, *commands; const char *newfile; struct stat st; + struct statfs stfs; size_t mmap_len = 0, i; unsigned char *mmap_ptr = NULL, junk = 0; int rc, len, fd = -1; @@ -248,6 +251,13 @@ int main(int argc, char **argv) exit(save_errno); } break; + case 'f': + if (statfs(fname, &stfs) == -1) { + save_errno = errno; + perror("statfs()"); + exit(save_errno); + } + break; case 'l': newfile = POP_ARG(); if (!newfile) -- 1.8.3.1