LU-3041 lfsck: improve lfsck.sh on b1_8

The test directory of lfsck.sh contains some files referencing same
object, which could cause error when removing the directory on test
cleanup. Also, in lfsck.sh we shouldn't use debugfs to remove objects,
that'll cause quota usage inconsistence warning in e2fsck.

This patch is a backport of commit 622148 and 2df010, and it also
includes part of the patch of LU-3133 and LU-3180.

Test-Parameters: testlist=lfsck

Signed-off-by: Liu Ying <emoly.liu@intel.com>
Change-Id: I6fd5db0c921e744f12a92676eb906d730b9c3d10
Reviewed-on: http://review.whamcloud.com/5968
Reviewed-by: Niu Yawei <yawei.niu@intel.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Johann Lombardi <johann.lombardi@intel.com>

diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index 70e370b..49dc23f 100644 (file)
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -124,4 +124,4 @@ MACHINEFILE_OPTION=${MACHINEFILE_OPTION:-"-machinefile"}
 # This is used by a small number of tests to share state between the client
 # running the tests, or in some cases between the servers (e.g. lfsck.sh).
 # It needs to be a non-lustre filesystem that is available on all the nodes.
-SHARED_DIRECTORY=${SHARED_DIRECTORY:-""}    # bug 17839 comment 65
+SHARED_DIRECTORY=${SHARED_DIRECTORY:-$TMP}    # bug 17839 comment 65

diff --git a/lustre/tests/lfsck.sh b/lustre/tests/lfsck.sh
index 55c2b04..4627bda 100644 (file)
--- a/lustre/tests/lfsck.sh
+++ b/lustre/tests/lfsck.sh
@@ -16,9 +16,9 @@ NUMDIRS=${NUMDIRS:-4}
 OSTIDX=${OSTIDX:-0} # the OST index in LOV
 OBJGRP=${OBJGRP:-0} # the OST object group
 
-[ -d "$SHARED_DIRECTORY" ] || \
-    { skip "SHARED_DIRECTORY should be specified with a shared directory \
-which can be accessable on all of the nodes" && exit 0; }
+[ ! -d "$SHARED_DIRECTORY" ] &&
+    skip_env "SHARED_DIRECTORY should be accessible on all nodes" &&
+    exit 0
 
 which getfattr &>/dev/null || { skip_env "could not find getfattr" && exit 0; }
 which setfattr &>/dev/null || { skip_env "could not find setfattr" && exit 0; }
@@ -100,24 +100,6 @@ get_objects() {
     echo $ost_objids
 }
 
-# Get the OST nodet name (given the OST index).
-get_ost_node() {
-    local obdidx=$1
-    local ost_uuid
-    local ost_node
-    local node
-
-    ost_uuid=$(ostuuid_from_index $obdidx)
-
-    for node in $(osts_nodes); do
-        do_node $node "lctl get_param -n obdfilter.*.uuid" | grep -q $ost_uuid
-        [ ${PIPESTATUS[1]} -eq 0 ] && ost_node=$node && break
-    done
-    [ -z "$ost_node" ] && \
-        echo "failed to find the OST with index $obdidx" && return 1
-    echo $ost_node
-}
-
 # Get the OST target device (given the OST facet name and OST index).
 get_ost_dev() {
     local node=$1
@@ -170,30 +152,9 @@ get_files() {
     echo $files
 }
 
-# Remove objects associated with files.
+# Remove objects from OST.
 remove_objects() {
-    local node=$1
-    shift
-    local ostdev=$1
-    shift
-    local group=$1
-    shift
-    local objids="$@"
-    local tmp
-    local i
-    local rc
-
-    echo "removing objects from $ostdev on $facet: $objids"
-    tmp=$(mktemp $SHARED_DIRECTORY/debugfs.XXXXXXXXXX)
-    for i in $objids; do
-        echo "rm O/$group/d$((i % 32))/$i" >> $tmp
-    done
-
-    do_node $node "$DEBUGFS -w -f $tmp $ostdev"
-    rc=${PIPESTATUS[0]}
-    rm -f $tmp
-
-    return $rc
+    do_rpc_nodes $(facet_host $1) remove_ost_objects $@
 }
 
 # Remove files from MDS.
@@ -211,40 +172,43 @@ duplicate_files() {
 # get the server target devices
 get_svr_devs
 
+TESTDIR=$DIR/d0.$TESTSUITE
 if is_empty_fs $MOUNT; then
     # create test directory
-    TESTDIR=$DIR/d0.$TESTSUITE
     mkdir -p $TESTDIR || error "mkdir $TESTDIR failed"
 
     # create some dirs and files on the filesystem
     create_files $TESTDIR $NUMDIRS $NUMFILES
 
-    # get the objids for files in group $OBJGRP on the OST with index $OSTIDX
+    # get objids for files in group $OBJGRP on the OST with index $OSTIDX
+    echo "objects to be removed, leaving dangling references:"
     OST_REMOVE=$(get_objects $OSTIDX $OBJGRP \
                 $(seq -f $TESTDIR/testfile.%g $NUMFILES))
 
     # get the node name and target device for the OST with index $OSTIDX
-    OSTNODE=$(get_ost_node $OSTIDX) || error "get_ost_node by index $OSTIDX failed"
-    OSTDEV=$(get_ost_dev $OSTNODE $OSTIDX) || \
+    OSTNODE=$(facet_active_host ost$((OSTIDX + 1)))
+    OSTDEV=$(get_ost_dev $OSTNODE $OSTIDX) ||
        error "get_ost_dev $OSTNODE $OSTIDX failed"
 
     # get the file names to be duplicated on the MDS
+    echo "files to be duplicated, leaving double-referenced objects:"
     MDS_DUPE=$(get_files dup $TESTDIR $NUMFILES) || error "$MDS_DUPE"
     # get the file names to be removed from the MDS
+    echo "files to be removed, leaving orphan objects:"
     MDS_REMOVE=$(get_files remove $TESTDIR $NUMFILES) || error "$MDS_REMOVE"
 
     stopall -f || error "cleanupall failed"
 
     # remove objects associated with files in group $OBJGRP
     # on the OST with index $OSTIDX
-    remove_objects $OSTNODE $OSTDEV $OBJGRP $OST_REMOVE || \
+    remove_objects ost$((OSTIDX + 1)) $OSTDEV $OBJGRP $OST_REMOVE ||
         error "removing objects failed"
 
     # remove files from MDS
     remove_files mds $MDSDEV $MDS_REMOVE || error "removing files failed"
 
     # create EAs on files so objects are referenced from different files
-    duplicate_files mds $MDSDEV $MDS_DUPE || \
+    duplicate_files mds $MDSDEV $MDS_DUPE ||
         error "duplicating files failed"
     FSCK_MAX_ERR=1   # file system errors corrected
 else # is_empty_fs $MOUNT
@@ -257,8 +221,10 @@ fi
 generate_db
 
 # remount filesystem
+ORIG_REFORMAT=$REFORMAT
 REFORMAT=""
 check_and_setup_lustre
+REFORMAT=$ORIG_REFORMAT
 
 # run lfsck
 rc=0
@@ -280,5 +246,14 @@ else
 fi
 
 complete $(basename $0) $SECONDS
+# The test directory contains some files referencing to some object
+# which could cause error when removing the directory.
+RMCNT=0
+while [ -d $TESTDIR ]; do
+       RMCNT=$((RMCNT + 1))
+       rm -fr $TESTDIR || echo "$RMCNT round: rm $TESTDIR failed"
+       [ $RMCNT -ge 10 ] && error "cleanup $TESTDIR failed $RMCNT times"
+       remount_client $MOUNT
+done
 check_and_cleanup_lustre
 exit_status

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index f127797..5fabded 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2394,9 +2394,9 @@ is_empty_dir() {
 is_empty_fs() {
     [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \
        -print | wc -l) = 1 ] || return 1
-    [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found && return 0
-    [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre && return 0
-    return 1
+    [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found || return 1
+    [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1
+    return 0
 }
 
 check_and_setup_lustre() {
@@ -4210,6 +4210,29 @@ max_recovery_time () {
     echo $service_time
 }
 
+# Remove objects from OST
+remove_ost_objects() {
+    local facet=$1
+    local ostdev=$2
+    local group=$3
+    shift 3
+    local objids="$@"
+    local mntpt=$(facet_mntpt $facet)
+    local opts=$OST_MOUNT_OPTS
+    local i
+    local rc
+
+    echo "removing objects from $ostdev on $facet: $objids"
+    mount -t $FSTYPE $OST_MOUNT_OPTS $ostdev $mntpt || return $?
+    rc=0
+    for i in $objids; do
+        rm $mntpt/O/$group/d$((i % 32))/$i || { rc=$?; break; }
+    done
+    umount -f $mntpt || return $?
+    return $rc
+}
+
+#Remove files from MDT
 remove_mdt_files() {
     local facet=$1
     local mdtdev=$2
@@ -4219,7 +4242,7 @@ remove_mdt_files() {
 
     echo "removing files from $mdtdev on $facet: $files"
     mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
-    rc=0;
+    rc=0
     for f in $files; do
        rm $mntpt/ROOT/$f || { rc=$?; break; }
     done