6 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
7 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
8 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
10 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
11 . $LUSTRE/tests/test-framework.sh
13 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
16 require_dsh_mds || exit 0
17 require_dsh_ost || exit 0
20 skip "skip lfsck performance test under non-SLOW mode" && exit 0
22 NTHREADS=${NTHREADS:-0}
24 MINCOUNT=${MINCOUNT:-4096}
25 MAXCOUNT=${MAXCOUNT:-8192}
26 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
27 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
28 BASE_COUNT=${BASE_COUNT:-8192}
30 INCFACTOR=${INCFACTOR:-25} #percent
31 MINSUBDIR=${MINSUBDIR:-1}
32 MAXSUBDIR=${MAXSUBDIR:-2}
33 TOTSUBDIR=${TOTSUBDIR:-2}
36 RCMD="do_facet ${SINGLEMDS}"
37 RLCTL="${RCMD} ${LCTL}"
38 MDT_DEV="${FSNAME}-MDT0000"
39 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
40 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
41 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV}"
42 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
43 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
44 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
46 if [ ${NTHREADS} -eq 0 ]; then
47 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
48 NTHREADS=$((CPUCORE * 2))
52 ${ECHOCMD} "${LCTL} <<-EOF
53 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
59 ${ECHOCMD} "${LCTL} <<-EOF
67 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
70 ${ECHOCMD} "${LCTL} <<-EOF
75 for ((j = 1; j < ${threads}; j++)); do
76 ${ECHOCMD} "${LCTL} <<-EOF
78 test_mkdir ${tdir}${j}
82 ${ECHOCMD} "${LCTL} <<-EOF
84 --threads ${threads} 0 ${echodev} test_create \
85 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
90 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
94 lfsck_create_nfiles() {
99 local ldir="/test-${lbase}"
104 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
105 local usize=$((count / NTHREADS))
106 [ ${usize} -eq 0 ] && break
107 local tdir=${ldir}-${cycle}-
109 echo "[cycle: ${cycle}] [threads: ${threads}]"\
110 "[files: ${count}] [basedir: ${tdir}]"
111 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
112 error "Fail to start MDS!"
113 #define OBD_FAIL_FID_IGIF 0x1504
114 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
120 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
121 stop ${SINGLEMDS} || error "Fail to stop MDS!"
123 total=$((total - usize * NTHREADS))
124 [ ${total} -eq 0 ] && break
125 lbase=$((lbase + usize))
137 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
138 reformat_external_journal
139 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
140 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
141 $(mdsvdevname 1) >/dev/null || error "Fail to reformat the MDS!"
143 for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
144 local nfiles=$((i - BCOUNT))
146 echo "+++ start to create for ${i} files set at: $(date) +++"
147 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
148 error "Fail to create files!"
149 echo "+++ end to create for ${i} files set at: $(date) +++"
152 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
153 error "Fail to start MDS!"
155 echo "start lfsck_namespace for ${i} files set at: $(date)"
156 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
159 local STATUS=$($SHOW_NAMESPACE |
160 awk '/^status/ { print $2 }')
161 [ "$STATUS" == "completed" ] && break
162 sleep 3 # check status every 3 seconds
165 echo "end lfsck_namespace for ${i} files set at: $(date)"
166 SPEED=$($SHOW_NAMESPACE |
167 awk '/^average_speed_phase1/ { print $2 }')
168 echo "lfsck_namespace speed is ${SPEED}/sec"
169 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
172 run_test 0 "lfsck namespace performance (routine case) without load"
175 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
176 skip "not implemented for ZFS" && return
182 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
183 reformat_external_journal
184 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
185 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
186 $(mdsvdevname 1) > /dev/null || error "Fail to reformat the MDS"
188 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
189 i = $((i * FACTOR)))); do
190 local nfiles=$((i - BCOUNT))
192 echo "+++ start to create for ${i} files set at: $(date) +++"
193 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
194 error "Fail to create files!"
195 echo "+++ end to create for ${i} files set at: $(date) +++"
198 local stime=$(date +%s)
199 echo "backup/restore ${i} files start at: $(date)"
200 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
201 echo "backup/restore ${i} files end at: $(date)"
202 local etime=$(date +%s)
203 local delta=$((etime - stime))
204 [ $delta -gt 0 ] || delta=1
205 echo "backup/restore ${i} files used ${delta} seconds"
206 echo "backup/restore speed is $((i / delta))/sec"
208 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
209 error "Fail to start MDS!"
211 echo "start lfsck_namespace for ${i} files set at: $(date)"
212 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
215 local STATUS=$($SHOW_NAMESPACE |
216 awk '/^status/ { print $2 }')
217 [ "$STATUS" == "completed" ] && break
218 sleep 3 # check status every 3 seconds
221 echo "end lfsck_namespace for ${i} files set at: $(date)"
222 local SPEED=$($SHOW_NAMESPACE |
223 awk '/^average_speed_phase1/ { print $2 }')
224 echo "lfsck_namespace speed is ${SPEED}/sec"
225 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
228 run_test 1 "lfsck namespace performance (backup/restore) without load"
233 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
234 i = $((i * FACTOR)))); do
236 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
237 reformat_external_journal
238 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) \
239 --backfstype $(facet_fstype ${SINGLEMDS}) --reformat \
240 ${MDT_DEVNAME} $(mdsvdevname 1) > /dev/null ||
241 error "Fail to reformat the MDS!"
243 echo "+++ start to create for ${i} files set at: $(date) +++"
244 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
245 error "Fail to create files!"
246 echo "+++ end to create for ${i} files set at: $(date) +++"
248 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
249 error "Fail to start MDS!"
251 echo "start lfsck_namespace for ${i} files set at: $(date)"
252 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
255 local STATUS=$($SHOW_NAMESPACE |
256 awk '/^status/ { print $2 }')
257 [ "$STATUS" == "completed" ] && break
258 sleep 3 # check status every 3 seconds
261 echo "end lfsck_namespace for ${i} files set at: $(date)"
262 local SPEED=$($SHOW_NAMESPACE |
263 awk '/^average_speed_phase1/ { print $2 }')
264 echo "lfsck_namespace speed is ${SPEED}/sec"
265 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
268 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
271 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
273 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
278 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
279 reformat_external_journal
280 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
281 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
282 $(mdsvdevname 1) > /dev/null || error "Fail to reformat the MDS"
284 for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
285 local nfiles=$((i - BCOUNT))
287 echo "+++ start to create for ${i} files set at: $(date) +++"
288 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
289 error "Fail to create files!"
290 echo "+++ end to create for ${i} files set at: $(date) +++"
294 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
295 error "Fail to start MDS!"
297 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
298 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
301 local STATUS=$($SHOW_NAMESPACE |
302 awk '/^status/ { print $2 }')
303 [ "$STATUS" == "completed" ] && break
304 sleep 3 # check status every 3 seconds
307 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
308 local FULL_SPEED=$($SHOW_NAMESPACE |
309 awk '/^average_speed_phase1/ { print $2 }')
310 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
311 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
312 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
315 for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
316 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
317 error "Fail to start MDS!"
319 $STOP_LFSCK > /dev/null 2>&1
320 echo "start lfsck_namespace with speed ${j} at: $(date)"
321 $START_NAMESPACE --reset -s ${j} ||
322 error "Fail to start lfsck_namespace with speed ${j}!"
323 # lfsck_namespace will be paused when MDS stop,
324 # and will be restarted automatically when mount up again.
325 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
327 local nfiles=$(((i - BCOUNT) / 2))
329 echo "+++ start to create for ${i} files set at: $(date) +++"
330 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
331 error "Fail to create files!"
332 echo "+++ end to create for ${i} files set at: $(date) +++"
337 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
338 error "Fail to start MDS!"
340 $STOP_LFSCK /dev/null 2>&1
341 echo "start lfsck_namespace with full speed at: $(date)"
342 $START_NAMESPACE --reset -s 0 ||
343 error "Fail to start lfsck_namespace with full speed!"
344 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
346 local nfiles=$(((i - BCOUNT) / 2))
348 echo "+++ start to create for ${i} files set at: $(date) +++"
349 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
350 error "Fail to create files!"
351 echo "+++ end to create for ${i} files set at: $(date) +++"
353 run_test 3 "lfsck namespace impact on create performance"
358 $RLCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout
363 echo "***** Start layout LFSCK on all devices at: $(date) *****"
364 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
366 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
367 mdd.${MDT_DEV}.lfsck_layout |
368 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
372 echo "***** End layout LFSCK on all devices at: $(date) *****"
374 for n in $(seq $MDSCOUNT); do
377 local SPEED=$(show_layout ${n} |
378 awk '/^average_speed_phase1/ { print $2 }')
380 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
389 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
390 local basedir="$mntpt/$tdir/$idx1/$idx2"
393 error_noexit "(11) Fail to mkdir $mntpt"
397 mount_client $mntpt || {
398 error_noexit "(12) Fail to mount $mntpt"
404 error_noexit "(13) Fail to mkdir $basedir"
408 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
409 createmany -o ${basedir}/f $UNIT || {
411 error_noexit "(14) Fail to gen $UNIT files under $basedir"
414 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
423 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
424 for ((k = 0; k < $MDSCOUNT; k++)); do
425 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
427 for ((l = 1; l <= $cnt; l++)); do
428 layout_gen_one ${k} ${l} &
433 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
438 local saved_mdscount=$MDSCOUNT
439 local saved_ostcount=$OSTCOUNT
442 stopall > /dev/null || error "(1) Fail to stopall"
444 LFSCKDIR="$DIR/$tdir"
446 for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
449 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
452 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
454 formatall > /dev/null ||
455 error "(2) Fail to formatall, subdirs=${j}"
458 setupall > /dev/null ||
459 error "(3) Fail to setupall, subdirs=${j}"
462 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
464 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
465 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
468 layout_gen_set ${j} || RC=$?
470 error "(6) generate set $RC, subdirs=${j}"
473 layout_test_one || RC=$?
475 error "(7) LFSCK failed with $RC, subdirs=${j}"
479 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
482 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
485 MDSCOUNT=$saved_mdscount
486 OSTCOUNT=$saved_ostcount
492 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
495 echo "Inject failure stub to simulate dangling reference"
496 #define OBD_FAIL_LFSCK_DANGLING 0x1610
497 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
501 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
505 local saved_mdscount=$MDSCOUNT
508 stopall > /dev/null || error "(1) Fail to stopall"
510 LFSCKDIR="$DIR/$tdir"
511 for ((i = 1; i <= $saved_mdscount; i++)); do
514 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
517 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
519 formatall > /dev/null ||
520 error "(2) Fail to formatall, subdirs=${j}"
523 setupall > /dev/null ||
524 error "(3) Fail to setupall, subdirs=${j}"
527 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
529 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
530 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
533 layout_gen_set ${j} || RC=$?
535 error "(6) generate set $RC, subdirs=${j}"
538 layout_test_one || RC=$?
540 error "(7) LFSCK failed with $RC, subdirs=${j}"
544 stopall > /dev/null || error "(8) Fail to stopall"
547 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
550 MDSCOUNT=$saved_mdscount
556 run_test 5a "lfsck layout performance (routine case) without load for DNE"
559 echo "Inject failure stub to simulate dangling reference"
560 #define OBD_FAIL_LFSCK_DANGLING 0x1610
561 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
565 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
567 layout_fast_create() {
571 local ldir="/test-${lbase}"
576 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
577 local usize=$((count / NTHREADS))
578 [ ${usize} -eq 0 ] && break
579 local tdir=${ldir}-${cycle}-
581 echo "[cycle: $cycle] [threads: $threads]"\
582 "[files: $count] [basedir: $tdir]"
586 total=$((total - usize * NTHREADS))
587 [ $total -eq 0 ] && break
588 lbase=$((lbase + usize))
600 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
603 stopall > /dev/null || error "(1) Fail to stopall"
605 local saved_mdscount=$MDSCOUNT
607 LFSCKDIR="$DIR/$tdir"
610 formatall > /dev/null || error "(2) Fail to formatall"
613 setupall > /dev/null || error "(3) Fail to setupall"
615 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
617 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
618 error "(5) Fail to setstripe on $LFSCKDIR"
621 layout_gen_set $TOTSUBDIR || RC=$?
623 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
626 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
627 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
628 error "(7) Fail to start layout LFSCK"
630 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
631 mdd.${MDT_DEV}.lfsck_layout |
632 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
634 error "(8) layout LFSCK cannot finished in time"
636 echo "***** End layout LFSCK on single MDS at: $(date) *****"
638 local SPEED=$(show_layout 1 |
639 awk '/^average_speed_phase1/ { print $2 }')
640 echo "lfsck_layout full_speed is $SPEED objs/sec"
642 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
643 local nfiles=$((inc_count / 2))
646 for ((m = 0, n = $INCFACTOR; n < 100;
647 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
648 local sl=$((SPEED * n / 100))
650 $STOP_LFSCK > /dev/null 2>&1
652 echo "start lfsck_layout with speed ${sl} at: $(date)"
653 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
655 "(9) Fail to start lfsck_layout with speed ${sl}"
657 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
658 layout_fast_create $nfiles ${m} $NTHREADS ||
659 lfsck_detach_error "(10) Fail to create files"
660 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
663 $STOP_LFSCK > /dev/null 2>&1
665 echo "start lfsck_layout with full speed at: $(date)"
666 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
668 "(11) Fail to start lfsck_layout with full speed"
670 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
671 layout_fast_create $nfiles ${m} $NTHREADS ||
672 lfsck_detach_error "(12) Fail to create files"
673 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
676 $STOP_LFSCK > /dev/null 2>&1
678 echo "create without lfsck_layout run back-ground"
679 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
680 layout_fast_create $nfiles ${m} $NTHREADS ||
681 lfsck_detach_error "(13) Fail to create files"
682 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
687 stopall > /dev/null || error "(14) Fail to stopall"
689 MDSCOUNT=$saved_mdscount
691 run_test 6 "lfsck layout impact on create performance"
693 # cleanup the system at last