LU-1943 tests: unlink all files when mdsrate times out

Port of ORI-453 to the master

When mdsrate cannot create the requested number of files in the
allocated timeframe (i.e. the --time option), filenames are not
necessarily consecutive and can have an index higher than the
number of files that was actually created. Here is an example:

0: iters 0 nfiles 100000 time 1 begin 0 end 99998 dirthreads 2.
1: iters 0 nfiles 100000 time 1 begin 1 end 99999 dirthreads 2.
0: client-4.lab.whamcloud.com starting at Mon Jan  9 08:26:07 2012
1: client-12.lab.whamcloud.com starting at Mon Jan  9 08:26:07 2012
0: created 1884 files, last file 'f3766'.
Rate: 2470.42 eff 2470.59 aggr 1235.29 avg client creates/sec
     (total: 2 threads 2440 creates 1 dirs 2 threads/dir 0.99 secs)
0: client-4.lab.whamcloud.com finished at Mon Jan  9 08:26:08 2012
1: created 556 files, last file 'f1111'.
1: client-12.lab.whamcloud.com finished at Mon Jan  9 08:26:08 2012

2440 files have been created and the last file name was f1111 for
rank 1 and f3766 for rank 0. Running mdsrate --unlink --nfiles 2440
in this case would discard all files with an index >= 2440.

With this patch, each mdsrate task will complete properly and report
the right number of executed operation as soon as one file is missing
in the per-task sequence. This behavior can be changed with the
--ignore and --recreate option which are already available.
This way, mdsrate --unlink --nfiles 100000 will work fine in the
example above since it will unlink all the existing files, without
doing tons of useless lookups.

Signed-off-by: Johann Lombardi <johann@whamcloud.com>
Signed-off-by: Mikhail Pershin <tappro@whamcloud.com>
Change-Id: I96f690ad0f36c65353bbc5a401455479b4c3783b
Reviewed-on: http://review.whamcloud.com/4195
Reviewed-by: Johann Lombardi <johann.lombardi@intel.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>

diff --git a/lustre/tests/mdsrate-create-large.sh b/lustre/tests/mdsrate-create-large.sh
index 07b04e5..398eb1a 100644 (file)
--- a/lustre/tests/mdsrate-create-large.sh
+++ b/lustre/tests/mdsrate-create-large.sh
@@ -67,11 +67,6 @@ else
     
     log "===== $0 ### 1 NODE UNLINK ###"
 
     
     log "===== $0 ### 1 NODE UNLINK ###"
 

-    if [ -f "$LOG" ]; then
-        CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG)
-        [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
-    fi
-

     COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                 --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
     echo "+ ${COMMAND}"
     COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                 --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
     echo "+ ${COMMAND}"


@@ -110,11 +105,6 @@ else
 
     log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###"
 
 
     log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###"
 

-    if [ -f "$LOG" ]; then
-        CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG)
-        [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
-    fi
-

     COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                 --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
     echo "+ ${COMMAND}"
     COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                 --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
     echo "+ ${COMMAND}"

diff --git a/lustre/tests/mdsrate-create-small.sh b/lustre/tests/mdsrate-create-small.sh
index 59b33a6..575166d 100644 (file)
--- a/lustre/tests/mdsrate-create-small.sh
+++ b/lustre/tests/mdsrate-create-small.sh
@@ -79,11 +79,6 @@ else
     else
         log "===== $0 ### 1 NODE UNLINK ###"
 
     else
         log "===== $0 ### 1 NODE UNLINK ###"
 

-        if [ -f "$LOG" ]; then
-            CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG)
-           [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
-        fi
-

         COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                      --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
         echo "+ ${COMMAND}"
         COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                      --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
         echo "+ ${COMMAND}"


@@ -129,11 +124,6 @@ else
     else
         log "===== $0 ### $NUM_CLIENTS NODES UNLINK with $THREADS_PER_CLIENT threads per client ###"
 
     else
         log "===== $0 ### $NUM_CLIENTS NODES UNLINK with $THREADS_PER_CLIENT threads per client ###"
 

-        if [ -f "$LOG" ]; then
-            CREATED=$(sed -n '/^Rate:/s/^.* \([0-9]*\) creates .*/\1/p' $LOG)
-            [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
-        fi
-

         COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                       --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
         echo "+ ${COMMAND}"
         COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                       --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
         echo "+ ${COMMAND}"

diff --git a/lustre/tests/mpi/mdsrate.c b/lustre/tests/mpi/mdsrate.c
index 4685387..f930ed6 100644 (file)
--- a/lustre/tests/mpi/mdsrate.c
+++ b/lustre/tests/mpi/mdsrate.c
@@ -573,7 +573,8 @@ main(int argc, char *argv[])
 
         /* if we're not measuring creation rates then precreate
          * the files we're operating on. */
 
         /* if we're not measuring creation rates then precreate
          * the files we're operating on. */

-        if ((mode != CREATE) && (mode != MKNOD) && !ignore) {
+        if ((mode != CREATE) && (mode != MKNOD) && !ignore &&
+            (mode != UNLINK || recreate)) {

                 /* create the files in reverse order. When we encounter
                  * a file that already exists, assume the remainder of 
                  * the files exist to save time. The timed performance
                 /* create the files in reverse order. When we encounter
                  * a file that already exists, assume the remainder of 
                  * the files exist to save time. The timed performance


@@ -737,8 +738,12 @@ main(int argc, char *argv[])
                         if (rc) {
                                 if (((rc = errno) == EINTR) && alarm_caught)
                                         break;
                         if (rc) {
                                 if (((rc = errno) == EINTR) && alarm_caught)
                                         break;

-                                if (((rc = errno) == ENOENT) && ignore)
-                                        continue;
+                                if ((rc = errno) == ENOENT) {
+                                        if (ignore)
+                                                continue;
+                                        /* no more files to unlink */
+                                        break;
+                                }

                                 fatal(myrank, "unlink(%s) error: %s\n",
                                       filename, strerror(rc));
                         }
                                 fatal(myrank, "unlink(%s) error: %s\n",
                                       filename, strerror(rc));
                         }


@@ -785,6 +790,9 @@ main(int argc, char *argv[])
                        "(total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n",
                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops,
                        cmd, ndirs, dirthreads, interval);
                        "(total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n",
                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops,
                        cmd, ndirs, dirthreads, interval);

+                if (mode == UNLINK && !recreate && !ignore && ag_ops != nfiles)
+                        printf("Warning: only unlinked %d files instead of %d"
+                               "\n", ag_ops, nfiles);

         }
 
         if (recreate) {
         }
 
         if (recreate) {