LU-9186 tests: fix sanityn test_76 and related code

Fix sanityn.sh test_76 to skip already open files when checking
whether mdt.*.exports.*.open_files accurately lists open files.
Otherwise, if some previous test leaves a file descriptor open
it will cause this test to fail spuriously.

Limit the number of open files to 256*MDSCOUNT rather than 2048,
so the test runs faster (80s vs. 215s in my VM with 2 MDTs).

Properly close the open shell FDs using "FD<&-" rather than
redirecting them to /dev/null, otherwise they stay open forever.

Print a '.' only once per 32 files opened/closed.

List open files in *.open_files in the order they were opened,
rather than reverse order, since this makes it easier to see which
files have been open longer.

Print out the pathname of the open files being skipped.

Don't explicitly mount $MOUNT2 in sanityn.sh.  This is already
handled inside check_and_setup_lustre() at the start of sanityn.sh.

Improve free_fd() to accept a "last_fd" argument to avoid searching
all open FDs on each call if it is known the fds are not closing.

Test-Parameters: trivial
Signed-off-by: Andreas Dilger <andreas.dilger@intel.com>
Change-Id: I274b6db96f49a52c086f836c005558ef4b3ebbe5
Reviewed-on: https://review.whamcloud.com/25847
Tested-by: Jenkins
Reviewed-by: Jian Yu <jian.yu@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Ashish Purkar <ashish.purkar@seagate.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c
index 934c6b8..63151d8 100644 (file)
--- a/lustre/mdt/mdt_open.c
+++ b/lustre/mdt/mdt_open.c
@@ -479,7 +479,7 @@ static int mdt_mfd_open(struct mdt_thread_info *info, struct mdt_object *p,
                mdt_mfd_close(info, mfd);
        } else {
                spin_lock(&med->med_open_lock);
-               list_add(&mfd->mfd_list, &med->med_open_head);
+               list_add_tail(&mfd->mfd_list, &med->med_open_head);
                spin_unlock(&med->med_open_lock);
        }
 

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index dc5f7ca..9c944ec 100644 (file)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -61,9 +61,6 @@ check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
 
 build_test_filter
 
-mkdir -p $MOUNT2
-mount_client $MOUNT2
-
 test_1a() {
        touch $DIR1/f1
        [ -f $DIR2/f1 ] || error
@@ -2930,7 +2927,7 @@ test_76() { #LU-946
                skip "Need MDS version at least 2.5.53" && return
 
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
-       local fcount=2048
+       local fcount=$((MDSCOUNT * 256))
        declare -a fd_list
        declare -a fid_list
 
@@ -2946,20 +2943,28 @@ test_76() { #LU-946
        # drop all open locks and close any cached "open" files on the client
        cancel_lru_locks mdc
 
-       echo -n "open files "
-       ulimit -n 8096
+       local open_fids_cmd="$LCTL get_param -n mdt.*.exports.'$nid'.open_files"
+       local fid_list=($(do_nodes $(comma_list $(mdts_nodes)) $open_fids_cmd))
+       local already=${#fid_list[@]}
+       for (( i = 0; i < $already; i++ )) ; do
+               log "already open[$i]: $($LFS fid2path $DIR2 ${fid_list[i]})"
+       done
+
+       echo -n "opening files: "
+       ulimit -n $((fcount + 50))
        for ((i = 0; i < $fcount; i++)); do
                touch $DIR/$tdir/f_$i
-               local fd=$(free_fd)
-               local cmd="exec $fd<$DIR/$tdir/f_$i"
-               eval $cmd
+               local fd=$(free_fd ${fd_list[i]})
+               local open_cmd="exec $fd<$DIR/$tdir/f_$i"
+               eval $open_cmd
+
                fd_list[i]=$fd
-               echo -n "."
+
+               (( $i % 32 == 0 )) && echo -n "."
        done
        echo
 
-       local get_open_fids="$LCTL get_param -n mdt.*.exports.'$nid'.open_files"
-       local fid_list=($(do_nodes $(comma_list $(mdts_nodes)) $get_open_fids))
+       fid_list=($(do_nodes $(comma_list $(mdts_nodes)) $open_fids_cmd))
 
        # Possible errors in openfiles FID list.
        # 1. Missing FIDs. Check 1
@@ -2969,19 +2974,28 @@ test_76() { #LU-946
        # 5. Valid FID, points to some other file. Check 3
 
        # Check 1
-       [ ${#fid_list[@]} -ne $fcount ] &&
-               error "${#fid_list[@]} != $fcount open files"
-
-       for (( i = 0; i < $fcount; i++ )) ; do
-               cmd="exec ${fd_list[i]}</dev/null"
-               eval $cmd
-               filename=$($LFS fid2path $DIR2 ${fid_list[i]})
+       [ ${#fid_list[@]} -ne $((fcount + already)) ] &&
+               error "${#fid_list[@]} != $fcount (+$already old) open files"
+
+       echo -n "closing files: "
+       for (( fd = 0, fid = 0; fd < $fcount; fd++, fid++ )) ; do
+               local close_cmd="exec ${fd_list[fd]}<&-"
+               eval $close_cmd
+               filename=$($LFS fid2path $DIR2 ${fid_list[fid]})
+
+               while [[ ! "$filename" =~ "$DIR2/$tdir/f_" ]]; do
+                       echo "skip old open file $filename"
+                       ((fid++))
+                       filename=$($LFS fid2path $DIR2 ${fid_list[fid]})
+               done
 
                # Check 2
                rm --interactive=no $filename
                [ $? -ne 0 ] &&
-                       error "Nonexisting fid ${fid_list[i]} listed."
+                       error "Nonexisting fid ${fid_list[fid]} listed."
+               (( $fd % 32 == 0 )) && echo -n "."
        done
+       echo
 
        # Check 3
        ls_op=$(ls $DIR2/$tdir | wc -l)
@@ -2990,7 +3004,7 @@ test_76() { #LU-946
 
        rm -rf $DIR/$tdir
 }
-run_test 76 "Verify MDS open_files tracking for 2048 files"
+run_test 76 "Verify MDT open_files listing"
 
 nrs_write_read() {
        local n=16
@@ -3910,9 +3924,8 @@ log "cleanup: ======================================================"
 # kill and wait in each test only guarentee script finish, but command in script
 # like 'rm' 'chmod' may still be running, wait for all commands to finish
 # otherwise umount below will fail
-wait_update $HOSTNAME "fuser -m $MOUNT2" "" || true
-
-[ "$(mount | grep $MOUNT2)" ] && umount $MOUNT2
+[ "$(mount | grep $MOUNT2)" ] && wait_update $HOSTNAME "fuser -m $MOUNT2" "" ||
+       true
 
 complete $SECONDS
 rm -f $SAMPLE_FILE

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 732587e..8a33852 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1377,8 +1377,7 @@ mount_facet() {
        # commit the device label change to disk
        if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then
                echo "Commit the device label on ${!dev}"
-               do_facet $facet "sync; sync; sync"
-               sleep 5
+               do_facet $facet "sync; sleep 5; sync"
        fi
 
 
@@ -7288,16 +7287,22 @@ test_mkdir() {
        fi
 }
 
-# find the smallest and not in use file descriptor
+# free_fd: find the smallest and not in use file descriptor [above @last_fd]
+#
+# If called many times, passing @last_fd will avoid repeated searching
+# already-open FDs repeatedly if we know they are still in use.
+#
+# usage: free_fd [last_fd]
 free_fd()
 {
-        local max_fd=$(ulimit -n)
-        local fd=3
-        while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do
-                ((++fd))
-        done
-        [ $fd -lt $max_fd ] || error "finding free file descriptor failed"
-        echo $fd
+       local max_fd=$(ulimit -n)
+       local fd=$((${1:-2} + 1))
+
+       while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do
+               ((++fd))
+       done
+       [ $fd -lt $max_fd ] || error "finding free file descriptor failed"
+       echo $fd
 }
 
 check_mount_and_prep()