6 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
7 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
8 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
10 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
11 . $LUSTRE/tests/test-framework.sh
13 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
16 require_dsh_mds || exit 0
17 require_dsh_ost || exit 0
20 skip "skip lfsck performance test under non-SLOW mode" && exit 0
22 NTHREADS=${NTHREADS:-0}
24 MINCOUNT=${MINCOUNT:-4096}
25 MAXCOUNT=${MAXCOUNT:-8192}
26 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
27 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
28 BASE_COUNT=${BASE_COUNT:-8192}
30 INCFACTOR=${INCFACTOR:-25} #percent
31 MINSUBDIR=${MINSUBDIR:-1}
32 MAXSUBDIR=${MAXSUBDIR:-2}
33 TOTSUBDIR=${TOTSUBDIR:-2}
36 RCMD="do_facet ${SINGLEMDS}"
37 RLCTL="${RCMD} ${LCTL}"
38 MDT_DEV="${FSNAME}-MDT0000"
39 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
40 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
41 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV}"
42 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
43 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
44 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
46 LFSCKDIR="$MOUNT/lfsck/"
48 if [ ${NTHREADS} -eq 0 ]; then
49 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
50 NTHREADS=$((CPUCORE * 2))
54 ${ECHOCMD} "${LCTL} <<-EOF
55 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
61 ${ECHOCMD} "${LCTL} <<-EOF
69 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
72 ${ECHOCMD} "${LCTL} <<-EOF
77 for ((j=1; j<${threads}; j++)); do
78 ${ECHOCMD} "${LCTL} <<-EOF
80 test_mkdir ${tdir}${j}
84 ${ECHOCMD} "${LCTL} <<-EOF
86 --threads ${threads} 0 ${echodev} test_create \
87 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
92 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
96 lfsck_create_nfiles() {
101 local ldir="/test-${lbase}"
106 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
107 local usize=$((count / NTHREADS))
108 [ ${usize} -eq 0 ] && break
109 local tdir=${ldir}-${cycle}-
111 echo "[cycle: ${cycle}] [threads: ${threads}]"\
112 "[files: ${count}] [basedir: ${tdir}]"
113 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
114 error "Fail to start MDS!"
115 #define OBD_FAIL_FID_IGIF 0x1504
116 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
122 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
123 stop ${SINGLEMDS} || error "Fail to stop MDS!"
125 total=$((total - usize * NTHREADS))
126 [ ${total} -eq 0 ] && break
127 lbase=$((lbase + usize))
139 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
140 reformat_external_journal
141 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
142 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
143 $(mdsvdevname 1) >/dev/null || error "Fail to reformat the MDS!"
145 for ((i=$MINCOUNT; i<=$MAXCOUNT; i=$((i * FACTOR)))); do
146 local nfiles=$((i - BCOUNT))
148 echo "+++ start to create for ${i} files set at: $(date) +++"
149 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
150 error "Fail to create files!"
151 echo "+++ end to create for ${i} files set at: $(date) +++"
154 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
155 error "Fail to start MDS!"
157 echo "start lfsck_namespace for ${i} files set at: $(date)"
158 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
161 local STATUS=$($SHOW_NAMESPACE |
162 awk '/^status/ { print $2 }')
163 [ "$STATUS" == "completed" ] && break
164 sleep 3 # check status every 3 seconds
167 echo "end lfsck_namespace for ${i} files set at: $(date)"
168 SPEED=$($SHOW_NAMESPACE |
169 awk '/^average_speed_phase1/ { print $2 }')
170 echo "lfsck_namespace speed is ${SPEED}/sec"
171 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
174 run_test 0 "lfsck namespace performance (routine case) without load"
177 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
178 skip "not implemented for ZFS" && return
184 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
185 reformat_external_journal
186 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
187 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
188 $(mdsvdevname 1) > /dev/null || error "Fail to reformat the MDS"
190 for ((i=$MINCOUNT_REPAIR; i<=$MAXCOUNT_REPAIR; i=$((i * FACTOR)))); do
191 local nfiles=$((i - BCOUNT))
193 echo "+++ start to create for ${i} files set at: $(date) +++"
194 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
195 error "Fail to create files!"
196 echo "+++ end to create for ${i} files set at: $(date) +++"
199 local stime=$(date +%s)
200 echo "backup/restore ${i} files start at: $(date)"
201 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
202 echo "backup/restore ${i} files end at: $(date)"
203 local etime=$(date +%s)
204 local delta=$((etime - stime))
205 [ $delta -gt 0 ] || delta=1
206 echo "backup/restore ${i} files used ${delta} seconds"
207 echo "backup/restore speed is $((i / delta))/sec"
209 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
210 error "Fail to start MDS!"
212 echo "start lfsck_namespace for ${i} files set at: $(date)"
213 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
216 local STATUS=$($SHOW_NAMESPACE |
217 awk '/^status/ { print $2 }')
218 [ "$STATUS" == "completed" ] && break
219 sleep 3 # check status every 3 seconds
222 echo "end lfsck_namespace for ${i} files set at: $(date)"
223 local SPEED=$($SHOW_NAMESPACE |
224 awk '/^average_speed_phase1/ { print $2 }')
225 echo "lfsck_namespace speed is ${SPEED}/sec"
226 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
229 run_test 1 "lfsck namespace performance (backup/restore) without load"
234 for ((i=$MINCOUNT_REPAIR; i<=$MAXCOUNT_REPAIR; i=$((i * FACTOR)))); do
236 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
237 reformat_external_journal
238 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) \
239 --backfstype $(facet_fstype ${SINGLEMDS}) --reformat \
240 ${MDT_DEVNAME} $(mdsvdevname 1) > /dev/null ||
241 error "Fail to reformat the MDS!"
243 echo "+++ start to create for ${i} files set at: $(date) +++"
244 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
245 error "Fail to create files!"
246 echo "+++ end to create for ${i} files set at: $(date) +++"
248 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
249 error "Fail to start MDS!"
251 echo "start lfsck_namespace for ${i} files set at: $(date)"
252 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
255 local STATUS=$($SHOW_NAMESPACE |
256 awk '/^status/ { print $2 }')
257 [ "$STATUS" == "completed" ] && break
258 sleep 3 # check status every 3 seconds
261 echo "end lfsck_namespace for ${i} files set at: $(date)"
262 local SPEED=$($SHOW_NAMESPACE |
263 awk '/^average_speed_phase1/ { print $2 }')
264 echo "lfsck_namespace speed is ${SPEED}/sec"
265 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
268 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
271 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
273 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
278 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
279 reformat_external_journal
280 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
281 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
282 $(mdsvdevname 1) > /dev/null || error "Fail to reformat the MDS"
284 for ((i=$inc_count; i<=$BASE_COUNT; i=$((i + inc_count)))); do
285 local nfiles=$((i - BCOUNT))
287 echo "+++ start to create for ${i} files set at: $(date) +++"
288 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
289 error "Fail to create files!"
290 echo "+++ end to create for ${i} files set at: $(date) +++"
294 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
295 error "Fail to start MDS!"
297 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
298 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
301 local STATUS=$($SHOW_NAMESPACE |
302 awk '/^status/ { print $2 }')
303 [ "$STATUS" == "completed" ] && break
304 sleep 3 # check status every 3 seconds
307 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
308 local FULL_SPEED=$($SHOW_NAMESPACE |
309 awk '/^average_speed_phase1/ { print $2 }')
310 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
311 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
312 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
315 for ((j=$inc_speed; j<$FULL_SPEED; j=$((j + inc_speed)))); do
316 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
317 error "Fail to start MDS!"
319 $STOP_LFSCK > /dev/null 2>&1
320 echo "start lfsck_namespace with speed ${j} at: $(date)"
321 $START_NAMESPACE --reset -s ${j} ||
322 error "Fail to start lfsck_namespace with speed ${j}!"
323 # lfsck_namespace will be paused when MDS stop,
324 # and will be restarted automatically when mount up again.
325 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
327 local nfiles=$(((i - BCOUNT) / 2))
329 echo "+++ start to create for ${i} files set at: $(date) +++"
330 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
331 error "Fail to create files!"
332 echo "+++ end to create for ${i} files set at: $(date) +++"
337 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
338 error "Fail to start MDS!"
340 $STOP_LFSCK /dev/null 2>&1
341 echo "start lfsck_namespace with full speed at: $(date)"
342 $START_NAMESPACE --reset -s 0 ||
343 error "Fail to start lfsck_namespace with full speed!"
344 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
346 local nfiles=$(((i - BCOUNT) / 2))
348 echo "+++ start to create for ${i} files set at: $(date) +++"
349 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
350 error "Fail to create files!"
351 echo "+++ end to create for ${i} files set at: $(date) +++"
353 run_test 3 "lfsck namespace impact on create performance"
358 $RLCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout
363 echo "***** Start layout LFSCK on all devices at: $(date) *****"
364 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
366 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
367 mdd.${MDT_DEV}.lfsck_layout |
368 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
372 echo "***** End layout LFSCK on all devices at: $(date) *****"
374 for n in $(seq $MDSCOUNT); do
377 local SPEED=$(show_layout ${n} |
378 awk '/^average_speed_phase1/ { print $2 }')
380 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
389 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
390 local basedir="$mntpt/lfsck/$idx1/$idx2"
393 error_noexit "(11) Fail to mkdir $mntpt"
397 mount_client $mntpt || {
398 error_noexit "(12) Fail to mount $mntpt"
404 error_noexit "(13) Fail to mkdir $basedir"
408 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
409 createmany -o ${basedir}/f $UNIT || {
411 error_noexit "(14) Fail to gen $UNIT files under $basedir"
414 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
423 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
424 for ((k=0; k<$MDSCOUNT; k++)); do
425 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
427 for ((l=1; l<=$cnt; l++)); do
428 layout_gen_one ${k} ${l} &
433 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
438 local saved_mdscount=$MDSCOUNT
439 local saved_ostcount=$OSTCOUNT
442 stopall > /dev/null || error "(1) Fail to stopall"
445 for ((i=1; i<=$saved_ostcount; i=$((i * 2)))); do
448 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
451 for ((j=$MINSUBDIR; j<=$MAXSUBDIR; j=$((j * FACTOR)))); do
453 formatall > /dev/null ||
454 error "(2) Fail to formatall, subdirs=${j}"
457 setupall > /dev/null ||
458 error "(3) Fail to setupall, subdirs=${j}"
461 error "(4) Fail to mkdir $LFSCKDIR, subdirs=${j}"
463 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
464 error "(5) Fail to setstripe on $LFSCKDIR, subdirs=${j}"
467 layout_gen_set ${j} || RC=$?
469 error "(6) Fail to generate set $RC, subdirs=${j}"
472 layout_test_one || RC=$?
474 error "(7) LFSCK failed with $RC, subdirs=${j}"
478 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
481 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
484 MDSCOUNT=$saved_mdscount
485 OSTCOUNT=$saved_ostcount
488 formatall > /dev/null || error "(9) Fail to stopall"
494 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
497 echo "Inject failure stub to simulate dangling reference"
498 #define OBD_FAIL_LFSCK_DANGLING 0x1610
499 for i in $(seq $OSTCOUNT); do
500 do_facet ost${i} $LCTL set_param fail_loc=0x1610
505 for i in $(seq $OSTCOUNT); do
506 do_facet ost${i} $LCTL set_param fail_loc=0
509 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
513 local saved_mdscount=$MDSCOUNT
516 stopall > /dev/null || error "(1) Fail to stopall"
518 for ((i=1; i<=$saved_mdscount; i++)); do
521 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
524 for ((j=$MINSUBDIR; j<=$MAXSUBDIR; j=$((j * FACTOR)))); do
526 formatall > /dev/null ||
527 error "(2) Fail to formatall, subdirs=${j}"
530 setupall > /dev/null ||
531 error "(3) Fail to setupall, subdirs=${j}"
534 error "(4) Fail to mkdir $LFSCKDIR, subdirs=${j}"
536 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
537 error "(5) Fail to setstripe on $LFSCKDIR, subdirs=${j}"
540 layout_gen_set ${j} || RC=$?
542 error "(6) Fail to generate set $RC, subdirs=${j}"
545 layout_test_one || RC=$?
547 error "(7) LFSCK failed with $RC, subdirs=${j}"
551 stopall > /dev/null || error "(8) Fail to stopall"
554 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
557 MDSCOUNT=$saved_mdscount
560 formatall > /dev/null || error "(9) Fail to stopall"
566 run_test 5a "lfsck layout performance (routine case) without load for DNE"
569 echo "Inject failure stub to simulate dangling reference"
570 #define OBD_FAIL_LFSCK_DANGLING 0x1610
571 for i in $(seq $OSTCOUNT); do
572 do_facet ost${i} $LCTL set_param fail_loc=0x1610
577 for i in $(seq $OSTCOUNT); do
578 do_facet ost${i} $LCTL set_param fail_loc=0
581 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
583 layout_fast_create() {
587 local ldir="/test-${lbase}"
592 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
593 local usize=$((count / NTHREADS))
594 [ ${usize} -eq 0 ] && break
595 local tdir=${ldir}-${cycle}-
597 echo "[cycle: $cycle] [threads: $threads]"\
598 "[files: $count] [basedir: $tdir]"
602 total=$((total - usize * NTHREADS))
603 [ $total -eq 0 ] && break
604 lbase=$((lbase + usize))
616 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
619 stopall > /dev/null || error "(1) Fail to stopall"
621 local saved_mdscount=$MDSCOUNT
625 formatall > /dev/null || error "(2) Fail to formatall"
628 setupall > /dev/null || error "(3) Fail to setupall"
630 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
632 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
633 error "(5) Fail to setstripe on $LFSCKDIR"
636 layout_gen_set $TOTSUBDIR || RC=$?
638 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
641 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
642 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
643 error "(7) Fail to start layout LFSCK"
645 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
646 mdd.${MDT_DEV}.lfsck_layout |
647 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
649 error "(8) layout LFSCK cannot finished in time"
651 echo "***** End layout LFSCK on single MDS at: $(date) *****"
653 local SPEED=$(show_layout 1 |
654 awk '/^average_speed_phase1/ { print $2 }')
655 echo "lfsck_layout full_speed is $SPEED objs/sec"
657 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
658 local nfiles=$((inc_count / 2))
661 for ((m=0, n=$INCFACTOR; n<100;
662 m=$((m + inc_count)), n=$((n + INCFACTOR)))); do
663 local sl=$((SPEED * n / 100))
665 $STOP_LFSCK > /dev/null 2>&1
667 echo "start lfsck_layout with speed ${sl} at: $(date)"
668 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
670 "(9) Fail to start lfsck_layout with speed ${sl}"
672 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
673 layout_fast_create $nfiles ${m} $NTHREADS ||
674 lfsck_detach_error "(10) Fail to create files"
675 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
678 $STOP_LFSCK > /dev/null 2>&1
680 echo "start lfsck_layout with full speed at: $(date)"
681 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
683 "(11) Fail to start lfsck_layout with full speed"
685 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
686 layout_fast_create $nfiles ${m} $NTHREADS ||
687 lfsck_detach_error "(12) Fail to create files"
688 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
691 $STOP_LFSCK > /dev/null 2>&1
693 echo "create without lfsck_layout run back-ground"
694 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
695 layout_fast_create $nfiles ${m} $NTHREADS ||
696 lfsck_detach_error "(13) Fail to create files"
697 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
702 stopall > /dev/null || error "(14) Fail to stopall"
704 MDSCOUNT=$saved_mdscount
707 formatall > /dev/null || error "(15) Fail to stopall"
709 run_test 6 "lfsck layout impact on create performance"
711 # cleanup the system at last