From bc34efd85de677eeb925945dbbf1c6c186fc24c6 Mon Sep 17 00:00:00 2001 From: shadow Date: Mon, 13 Apr 2009 19:02:48 +0000 Subject: [PATCH] add lazystatfs mount option to allow statfs(2) to skip down OSTs Branch b_release_1_8_0 b=17974 i=green i=vitaly --- lustre/ChangeLog | 6 ++ lustre/llite/llite_internal.h | 1 + lustre/llite/llite_lib.c | 17 ++++ lustre/llite/lproc_llite.c | 30 +++++++ lustre/lov/lov_request.c | 5 +- lustre/osc/osc_request.c | 4 + lustre/tests/conf-sanity.sh | 196 ++++++++++++++++++++++++++++++++++++++++++ lustre/tests/multiop.c | 10 +++ 8 files changed, 267 insertions(+), 2 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 8f94f36..339114c 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -43,6 +43,12 @@ Lustre filesystem with 4K stack may cause a stack overflow. For more information, please refer to bugzilla 17630. +Severity : enhancement +Bugzilla : 17974 +Description: add lazystatfs mount option to allow statfs(2) to skip down OSTs +Details : allow skip disconnected ost for send statfs request and hide error + in this case. + Severity : normal Frequency : rare, on llog test 6 Bugzilla : 16839 diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 522f9ce..c075598 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -295,6 +295,7 @@ enum stats_track_type { #define LL_SBI_LOCALFLOCK 0x40 /* Local flocks support by kernel */ #define LL_SBI_LRU_RESIZE 0x80 /* support lru resize */ #define LL_SBI_LLITE_CHECKSUM 0x100 /* checksum each page in memory */ +#define LL_SBI_LAZYSTATFS 0x200 /* lazystatfs mount option */ /* default value for ll_sb_info->contention_time */ #define SBI_DEFAULT_CONTENTION_SECONDS 60 diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index ff49304..e752ed2 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "llite_internal.h" cfs_mem_cache_t *ll_file_data_slab; @@ -869,6 +870,16 @@ static int ll_options(char *options, int *flags) *flags &= ~tmp; goto next; } + tmp = ll_set_opt("lazystatfs", s1, LL_SBI_LAZYSTATFS); + if (tmp) { + *flags |= tmp; + goto next; + } + tmp = ll_set_opt("nolazystatfs", s1, LL_SBI_LAZYSTATFS); + if (tmp) { + *flags &= ~tmp; + goto next; + } LCONSOLE_ERROR_MSG(0x152, "Unknown option '%s', won't mount.\n", s1); RETURN(-EINVAL); @@ -1779,6 +1790,9 @@ int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs, CDEBUG(D_SUPER, "MDC blocks "LPU64"/"LPU64" objects "LPU64"/"LPU64"\n", osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files); + if (sbi->ll_flags & LL_SBI_LAZYSTATFS) + flags |= OBD_STATFS_NODELAY; + rc = obd_statfs_rqset(class_exp2obd(sbi->ll_osc_exp), &obd_osfs, max_age, flags); if (rc) { @@ -2455,5 +2469,8 @@ int ll_show_options(struct seq_file *seq, struct vfsmount *vfs) if (sbi->ll_flags & LL_SBI_ACL) seq_puts(seq, ",acl"); + if (sbi->ll_flags & LL_SBI_LAZYSTATFS) + seq_puts(seq, ",lazystatfs"); + RETURN(0); } diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c index 6d3bb3d..af55527 100644 --- a/lustre/llite/lproc_llite.c +++ b/lustre/llite/lproc_llite.c @@ -582,6 +582,35 @@ static int ll_rd_statahead_stats(char *page, char **start, off_t off, sbi->ll_sa_miss); } +static int ll_rd_lazystatfs(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct super_block *sb = data; + struct ll_sb_info *sbi = ll_s2sbi(sb); + + return snprintf(page, count, "%u\n", + (sbi->ll_flags & LL_SBI_LAZYSTATFS) ? 1 : 0); +} + +static int ll_wr_lazystatfs(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct super_block *sb = data; + struct ll_sb_info *sbi = ll_s2sbi(sb); + int val, rc; + + rc = lprocfs_write_helper(buffer, count, &val); + if (rc) + return rc; + + if (val) + sbi->ll_flags |= LL_SBI_LAZYSTATFS; + else + sbi->ll_flags &= ~LL_SBI_LAZYSTATFS; + + return count; +} + static struct lprocfs_vars lprocfs_llite_obd_vars[] = { { "uuid", ll_rd_sb_uuid, 0, 0 }, //{ "mntpt_path", ll_rd_path, 0, 0 }, @@ -610,6 +639,7 @@ static struct lprocfs_vars lprocfs_llite_obd_vars[] = { ll_wr_lockless_truncate, 0}, { "statahead_max", ll_rd_statahead_max, ll_wr_statahead_max, 0 }, { "statahead_stats", ll_rd_statahead_stats, 0, 0 }, + { "lazystatfs", ll_rd_lazystatfs, ll_wr_lazystatfs, 0 }, { 0 } }; diff --git a/lustre/lov/lov_request.c b/lustre/lov/lov_request.c index 7e2eda5..9a94a3d 100644 --- a/lustre/lov/lov_request.c +++ b/lustre/lov/lov_request.c @@ -1594,11 +1594,11 @@ static int cb_statfs_update(struct obd_info *oinfo, int rc) lov_sfs = oinfo->oi_osfs; success = lovreq->rq_rqset->set_success; - /* XXX: the same is done in lov_update_common_set, however lovset->set_exp is not initialized. */ lov_update_set(lovreq->rq_rqset, lovreq, rc); if (rc) { + /* XXX ignore error for disconnected ost ? */ if (rc && !(lov->lov_tgts[lovreq->rq_idx] && lov->lov_tgts[lovreq->rq_idx]->ltd_active)) rc = 0; @@ -1637,7 +1637,8 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo, for (i = 0; i < lov->desc.ld_tgt_count; i++) { struct lov_request *req; - if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active) { + if (!lov->lov_tgts[i] || (!lov->lov_tgts[i]->ltd_active + && (oinfo->oi_flags & OBD_STATFS_NODELAY))) { CDEBUG(D_HA, "lov idx %d inactive\n", i); continue; } diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index e7c5b2c..4cfe870 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3316,6 +3316,10 @@ static int osc_statfs_interpret(struct ptlrpc_request *req, struct obd_statfs *msfs; ENTRY; + if ((rc == -ENOTCONN || rc == -EAGAIN) && + (aa->aa_oi->oi_flags & OBD_STATFS_NODELAY)) + GOTO(out, rc = 0); + if (rc != 0) GOTO(out, rc); diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 4dd2123..c6ecae5 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -1609,5 +1609,201 @@ test_48() { # bug 17636 run_test 48 "too many acls on file" +lazystatfs() { + # Test both statfs and lfs df and fail if either one fails + multiop_bg_pause $1 f_ + RC1=$? + PID=$! + killall -USR1 multiop + [ $RC1 -ne 0 ] && log "lazystatfs multiop failed" + wait $PID || { RC1=$?; log "multiop return error "; } + + $LFS df & + PID=$! + sleep 5 + kill -s 0 $PID + RC2=$? + if [ $RC2 -eq 0 ]; then + kill -s 9 $PID + log "lazystatfs df failed" + fi + + RC=0 + [[ $RC1 -ne 0 || $RC2 -eq 0 ]] && RC=1 + return $RC +} + +test_50a() { + setup + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + lazystatfs $MOUNT || error "lazystatfs failed but no down servers" + + cleanup || return $? +} +run_test 50a "lazystatfs all servers available ==========================" + +test_50b() { + setup + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + # Wait for client to detect down OST + stop_ost || error "Unable to stop OST1" + CONN_PROC="osc.$FSNAME-OST0000-osc.ost_server_uuid" + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + lazystatfs $MOUNT || error "lazystatfs should don't have returned EIO" + + umount_client $MOUNT || error "Unable to unmount client" + stop_mds || error "Unable to stop MDS" +} +run_test 50b "lazystatfs all servers down ==========================" + +test_50c() { + start_mds || error "Unable to start MDS" + start_ost || error "Unable to start OST1" + start_ost2 || error "Unable to start OST2" + mount_client $MOUNT || error "Unable to mount client" + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + # Wait for client to detect down OST + stop_ost || error "Unable to stop OST1" + CONN_PROC="osc.$FSNAME-OST0000-osc.ost_server_uuid" + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + lazystatfs $MOUNT || error "lazystatfs failed with one down server" + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost2 || error "Unable to stop OST2" + stop_mds || error "Unable to stop MDS" +} +run_test 50c "lazystatfs one server down ==========================" + +test_50d() { + start_mds || error "Unable to start MDS" + start_ost || error "Unable to start OST1" + start_ost2 || error "Unable to start OST2" + mount_client $MOUNT || error "Unable to mount client" + lctl set_param llite.$FSNAME-*.lazystatfs=1 + touch $DIR/$tfile + + # Issue the statfs during the window where the client still + # belives the OST to be available but it is in fact down. + # No failure just a statfs which hangs for a timeout interval. + stop_ost || error "Unable to stop OST1" + lazystatfs $MOUNT || error "lazystatfs failed with one down server" + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost2 || error "Unable to stop OST2" + stop_mds || error "Unable to stop MDS" +} +run_test 50d "lazystatfs client/server conn race ==========================" + +test_50e() { + local RC1 + local pid + CONN_PROC="osc.$FSNAME-OST0000-osc.ost_server_uuid" + + start_mds || return 1 + #first client should see only one ost + start_ost || return 2 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} != "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + lctl set_param llite.$FSNAME-*.lazystatfs=0 + + # Wait for client to detect down OST + stop_ost || error "Unable to stop OST1" + + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + mount_client $MOUNT || error "Unable to mount client" + + multiop_bg_pause $MOUNT _f + RC1=$? + pid=$! + + if [ $RC1 -ne 0 ]; then + log "lazystatfs multiop failed $RC1" + else + kill -USR1 $pid + sleep $(( $TIMEOUT+1 )) + kill -0 $pid + [ $? -ne 0 ] && error "process isn't sleep" + start_ost || error "Unable to start OST1" + wait $pid || error "statfs failed" + fi + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost || error "Unable to stop OST1" + stop_mds || error "Unable to stop MDS" +} +run_test 50e "normal statfs all servers down ==========================" + +test_50f() { + local RC1 + local pid + CONN_PROC="osc.$FSNAME-OST0001-osc.ost_server_uuid" + + start_mds || error "Unable to start mds" + #first client should see only one ost + start_ost || error "Unable to start OST1" + start_ost2 || error "Unable to start OST2" + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} != "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + lctl set_param llite.$FSNAME-*.lazystatfs=0 + + # Wait for client to detect down OST + stop_ost2 || error "Unable to stop OST2" + + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + while [ ${CONN_STATE} = "FULL" ]; do + sleep 1 + CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2` + done + + mount_client $MOUNT || error "Unable to mount client" + + multiop_bg_pause $MOUNT _f + RC1=$? + pid=$! + + if [ $RC1 -ne 0 ]; then + log "lazystatfs multiop failed $RC1" + else + kill -USR1 $pid + sleep $(( $TIMEOUT+1 )) + kill -0 $pid + [ $? -ne 0 ] && error "process isn't sleep" + start_ost2 || error "Unable to start OST1" + wait $pid || error "statfs failed" + fi + + umount_client $MOUNT || error "Unable to unmount client" + stop_ost || error "Unable to stop OST1" + stop_mds || error "Unable to stop MDS" +} +run_test 50f "normal statfs one server in down ==========================" + equals_msg `basename $0`: test complete [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true diff --git a/lustre/tests/multiop.c b/lustre/tests/multiop.c index 946fe05..9bcd34f 100755 --- a/lustre/tests/multiop.c +++ b/lustre/tests/multiop.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,7 @@ char usage[] = " C[num] create with optional stripes\n" " d mkdir\n" " D open(O_DIRECTORY)\n" +" f statfs\n" " L link\n" " l symlink\n" " m mknod\n" @@ -173,6 +175,7 @@ int main(int argc, char **argv) char *fname, *commands; const char *newfile; struct stat st; + struct statfs stfs; size_t mmap_len = 0, i; unsigned char *mmap_ptr = NULL, junk = 0; int rc, len, fd = -1; @@ -235,6 +238,13 @@ int main(int argc, char **argv) exit(save_errno); } break; + case 'f': + if (statfs(fname, &stfs) == -1) { + save_errno = errno; + perror("statfs()"); + exit(save_errno); + } + break; case 'l': newfile = POP_ARG(); if (!newfile) -- 1.8.3.1