From 7dfad1b4a3220ec38259f02d8bf8b17af73356a9 Mon Sep 17 00:00:00 2001 From: Johann Lombardi Date: Mon, 9 Jan 2012 18:18:26 +0100 Subject: [PATCH] LU-1943 tests: unlink all files when mdsrate times out Port of ORI-453 to the master When mdsrate cannot create the requested number of files in the allocated timeframe (i.e. the --time option), filenames are not necessarily consecutive and can have an index higher than the number of files that was actually created. Here is an example: 0: iters 0 nfiles 100000 time 1 begin 0 end 99998 dirthreads 2. 1: iters 0 nfiles 100000 time 1 begin 1 end 99999 dirthreads 2. 0: client-4.lab.whamcloud.com starting at Mon Jan 9 08:26:07 2012 1: client-12.lab.whamcloud.com starting at Mon Jan 9 08:26:07 2012 0: created 1884 files, last file 'f3766'. Rate: 2470.42 eff 2470.59 aggr 1235.29 avg client creates/sec (total: 2 threads 2440 creates 1 dirs 2 threads/dir 0.99 secs) 0: client-4.lab.whamcloud.com finished at Mon Jan 9 08:26:08 2012 1: created 556 files, last file 'f1111'. 1: client-12.lab.whamcloud.com finished at Mon Jan 9 08:26:08 2012 2440 files have been created and the last file name was f1111 for rank 1 and f3766 for rank 0. Running mdsrate --unlink --nfiles 2440 in this case would discard all files with an index >= 2440. With this patch, each mdsrate task will complete properly and report the right number of executed operation as soon as one file is missing in the per-task sequence. This behavior can be changed with the --ignore and --recreate option which are already available. This way, mdsrate --unlink --nfiles 100000 will work fine in the example above since it will unlink all the existing files, without doing tons of useless lookups. Signed-off-by: Johann Lombardi Signed-off-by: Mikhail Pershin Change-Id: I96f690ad0f36c65353bbc5a401455479b4c3783b Reviewed-on: http://review.whamcloud.com/4195 Reviewed-by: Johann Lombardi Tested-by: Hudson Tested-by: Maloo Reviewed-by: Alex Zhuravlev --- lustre/tests/mdsrate-create-large.sh | 10 ---------- lustre/tests/mdsrate-create-small.sh | 10 ---------- lustre/tests/mpi/mdsrate.c | 14 +++++++++++--- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/lustre/tests/mdsrate-create-large.sh b/lustre/tests/mdsrate-create-large.sh index 07b04e5..398eb1a 100644 --- a/lustre/tests/mdsrate-create-large.sh +++ b/lustre/tests/mdsrate-create-large.sh @@ -67,11 +67,6 @@ else log "===== $0 ### 1 NODE UNLINK ###" - if [ -f "$LOG" ]; then - CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG) - [ $CREATED -gt 0 ] && NUM_FILES=$CREATED - fi - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" echo "+ ${COMMAND}" @@ -110,11 +105,6 @@ else log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###" - if [ -f "$LOG" ]; then - CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG) - [ $CREATED -gt 0 ] && NUM_FILES=$CREATED - fi - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" echo "+ ${COMMAND}" diff --git a/lustre/tests/mdsrate-create-small.sh b/lustre/tests/mdsrate-create-small.sh index 59b33a6..575166d 100644 --- a/lustre/tests/mdsrate-create-small.sh +++ b/lustre/tests/mdsrate-create-small.sh @@ -79,11 +79,6 @@ else else log "===== $0 ### 1 NODE UNLINK ###" - if [ -f "$LOG" ]; then - CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG) - [ $CREATED -gt 0 ] && NUM_FILES=$CREATED - fi - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" echo "+ ${COMMAND}" @@ -129,11 +124,6 @@ else else log "===== $0 ### $NUM_CLIENTS NODES UNLINK with $THREADS_PER_CLIENT threads per client ###" - if [ -f "$LOG" ]; then - CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG) - [ $CREATED -gt 0 ] && NUM_FILES=$CREATED - fi - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" echo "+ ${COMMAND}" diff --git a/lustre/tests/mpi/mdsrate.c b/lustre/tests/mpi/mdsrate.c index 4685387..f930ed6 100644 --- a/lustre/tests/mpi/mdsrate.c +++ b/lustre/tests/mpi/mdsrate.c @@ -573,7 +573,8 @@ main(int argc, char *argv[]) /* if we're not measuring creation rates then precreate * the files we're operating on. */ - if ((mode != CREATE) && (mode != MKNOD) && !ignore) { + if ((mode != CREATE) && (mode != MKNOD) && !ignore && + (mode != UNLINK || recreate)) { /* create the files in reverse order. When we encounter * a file that already exists, assume the remainder of * the files exist to save time. The timed performance @@ -737,8 +738,12 @@ main(int argc, char *argv[]) if (rc) { if (((rc = errno) == EINTR) && alarm_caught) break; - if (((rc = errno) == ENOENT) && ignore) - continue; + if ((rc = errno) == ENOENT) { + if (ignore) + continue; + /* no more files to unlink */ + break; + } fatal(myrank, "unlink(%s) error: %s\n", filename, strerror(rc)); } @@ -785,6 +790,9 @@ main(int argc, char *argv[]) "(total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n", effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops, cmd, ndirs, dirthreads, interval); + if (mode == UNLINK && !recreate && !ignore && ag_ops != nfiles) + printf("Warning: only unlinked %d files instead of %d" + "\n", ag_ops, nfiles); } if (recreate) { -- 1.8.3.1