From ab5d05414bf2005fdfa2d3e0b18acec4cadf4342 Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Fri, 13 May 2022 08:44:34 -0500 Subject: [PATCH] EX-3838 test: sanity-pumount races In sanity-pumount, add functions wait_for_open(), wait_for_exec(), and wait_for_mmap() to wait for the forked PID to exec, open, or mmap the file on the Lustre FS. Use these to prevent fork exec check races. Test-Parameters: trivial testlist=sanity-pumount,sanity-pumount,sanity-pumount,sanity-pumount clientextra_install_params="--packages pumount" Signed-off-by: John L. Hammond Change-Id: I2a89989521cc4e61d35799c1d4fdb8ce37ae12d9 Reviewed-on: https://review.whamcloud.com/47344 Tested-by: jenkins Tested-by: Maloo --- lustre/tests/sanity-pumount.sh | 78 ++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/lustre/tests/sanity-pumount.sh b/lustre/tests/sanity-pumount.sh index 75a1003..54b98dd 100755 --- a/lustre/tests/sanity-pumount.sh +++ b/lustre/tests/sanity-pumount.sh @@ -55,6 +55,7 @@ declare -r SIGUSR2=12 declare -r SIGTERM=15 declare -r SIGCHLD=17 # Ignored by default. declare SLEEP_TIME=60 +declare WAIT_TIME=30 function sanity_pumount_init() { local mount @@ -120,6 +121,33 @@ function check_unmount_complete() { return 0 } +function wait_for_open() { + local pid="$1" + local path="$2" + local fd="${3:-0-1024}" + local deadline=$((SECONDS + WAIT_TIME)) + + [[ -e "$path" ]] || error "'$path' does not exist" + + while ((SECONDS < deadline)); do + if lsof -a -d "$fd" -p "$pid" -- "$path"; then + return 0 + fi + + sleep 1 + done + + error "pid '$pid' failed to open '$path'" +} + +function wait_for_exec() { + wait_for_open "$1" "$2" txt +} + +function wait_for_mmap() { + wait_for_open "$1" "$2" mem +} + function wait_signaled() { local pid=$1 local sig=$2 @@ -189,7 +217,7 @@ test_21a() { echo ZZZ > "$file" $MULTIOP "$file" o_c & pid=$! - sleep 5 + wait_for_open "$pid" "$file" pumount "$MOUNT1" || error "pumount failed" wait_signaled $pid $SIGKILL @@ -204,9 +232,9 @@ test_22a() { init_pumount_env dd if=/dev/zero of="$file" bs=4K count=1 - $MULTIOP "$file" OsM_c & + $MULTIOP "$file" OsMc_ & pid=$! - sleep 5 + wait_for_mmap "$pid" "$file" pumount "$MOUNT1" || error "pumount failed" wait_signaled $pid $SIGKILL @@ -221,9 +249,9 @@ test_22b() { init_pumount_env dd if=/dev/zero of="$file" bs=4K count=1 - $MULTIOP "$file" OsM_c & + $MULTIOP "$file" OsMc_ & pid=$! - sleep 5 + wait_for_mmap "$pid" "$file" rm "$file" pumount "$MOUNT1" || error "pumount failed" @@ -334,7 +362,7 @@ test_31d() { init_pumount_env - mkdir "$dir" + mkdir "$dir" exec {fd}<"$dir" sleep $SLEEP_TIME & pid=$! @@ -375,6 +403,7 @@ test_32a() { $sleep $SLEEP_TIME & pid=$! + wait_for_exec "$pid" "$sleep" pumount "$MOUNT1" || error "pumount failed" kill_wait_signaled $pid $SIGUSR2 $SIGKILL @@ -391,6 +420,7 @@ test_32b() { $sleep $SLEEP_TIME & pid=$! + wait_for_exec "$pid" "$sleep" rm $sleep pumount "$MOUNT1" || error "pumount failed" @@ -408,6 +438,7 @@ test_32c() { $sleep $SLEEP_TIME & pid=$! + wait_for_exec "$pid" "$sleep" mv $sleep $sleep-1 pumount "$MOUNT1" || error "pumount failed" @@ -424,6 +455,7 @@ test_33a() { pushd $DIR sleep $SLEEP_TIME & pid=$! + wait_for_open "$pid" "$DIR" cwd popd pumount "$MOUNT1" || error "pumount failed" @@ -527,10 +559,10 @@ test_55a() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep # pumount --print fields # COMM=1 PID=2 REF=3 HANDLE_TYPE=4 HANDLE=5 PATH=6 @@ -551,10 +583,10 @@ test_55b() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep pumount --print "$MOUNT1" | awk -v pid=$pid '$2 == pid' | grep . || error "pumount --print PID != $pid'" @@ -572,10 +604,10 @@ test_55c() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep pumount --print "$MOUNT1" | awk -v fd="$fd" '$3 == fd' | grep . || error "pumount --print FD != $fd" @@ -595,10 +627,10 @@ test_55d() { exec {fd}>"$file" fid=$($LFS path2fid "$file") - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep pumount --print "$MOUNT1" | awk -v fid="$fid" '$4 == "lustre" && $5 == fid' | grep . || error "pumount --print HANDLE_TYPE != 'lustre' or FID != '$fid'" @@ -616,10 +648,10 @@ test_55e() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep # XXX path changes after umount2(..., MNT_DETACH) from '/mnt/$FSNAME/$tfile' to '/$tfile' pumount --print "$MOUNT1" | awk -v path="/$tfile" '$6 == path' | grep . || @@ -641,10 +673,10 @@ test_55f() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep # COMM=1 PID=2 REF=3 HANDLE_TYPE=4 HANDLE=5 PATH=6 path=$(pumount --dry-run "$MOUNT1" | @@ -668,10 +700,10 @@ test_55g() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- - sleep 1 + wait_for_exec "$pid" /usr/bin/sleep # COMM=1 PID=2 REF=3 HANDLE_TYPE=4 HANDLE=5 PATH=6 path=$(pumount --dry-run "$MOUNT1" | @@ -693,9 +725,10 @@ test_56a() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- + wait_for_exec "$pid" /usr/bin/sleep pumount --signal=$sig "$MOUNT1" || error "pumount --signal failed" kill_wait_signaled $pid $SIGUSR2 $SIGTERM @@ -713,9 +746,10 @@ test_57a() { init_pumount_env exec {fd}>"$file" - sleep $SLEEP_TIME & + /usr/bin/sleep $SLEEP_TIME & pid=$! exec {fd}>&- + wait_for_exec "$pid" /usr/bin/sleep status=0 pumount --signal=$SIGCHLD "$MOUNT1" || status=$? -- 1.8.3.1