6 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
7 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
8 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
10 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
11 . $LUSTRE/tests/test-framework.sh
13 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
16 #remove it when zfs-based backend iteration is enabled
17 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
18 skip "lfsck performance only for ldiskfs" && exit 0
20 require_dsh_mds || exit 0
21 require_dsh_ost || exit 0
24 skip "skip lfsck performance test under non-SLOW mode" && exit 0
26 NTHREADS=${NTHREADS:-0}
28 MINCOUNT=${MINCOUNT:-4096}
29 MAXCOUNT=${MAXCOUNT:-8192}
30 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
31 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
32 BASE_COUNT=${BASE_COUNT:-8192}
34 INCFACTOR=${INCFACTOR:-25} #percent
35 MINSUBDIR=${MINSUBDIR:-1}
36 MAXSUBDIR=${MAXSUBDIR:-2}
37 TOTSUBDIR=${TOTSUBDIR:-2}
40 RCMD="do_facet ${SINGLEMDS}"
41 RLCTL="${RCMD} ${LCTL}"
42 MDT_DEV="${FSNAME}-MDT0000"
43 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
44 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
45 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV}"
46 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
47 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
48 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
50 LFSCKDIR="$MOUNT/lfsck/"
52 if [ ${NTHREADS} -eq 0 ]; then
53 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
54 NTHREADS=$((CPUCORE * 2))
58 ${ECHOCMD} "${LCTL} <<-EOF
59 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
65 ${ECHOCMD} "${LCTL} <<-EOF
73 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
76 ${ECHOCMD} "${LCTL} <<-EOF
81 for ((j=1; j<${threads}; j++)); do
82 ${ECHOCMD} "${LCTL} <<-EOF
84 test_mkdir ${tdir}${j}
88 ${ECHOCMD} "${LCTL} <<-EOF
90 --threads ${threads} 0 ${echodev} test_create \
91 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
96 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
100 lfsck_create_nfiles() {
105 local ldir="/test-${lbase}"
110 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
111 local usize=$((count / NTHREADS))
112 [ ${usize} -eq 0 ] && break
113 local tdir=${ldir}-${cycle}-
115 echo "[cycle: ${cycle}] [threads: ${threads}]"\
116 "[files: ${count}] [basedir: ${tdir}]"
117 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
118 error "Fail to start MDS!"
119 #define OBD_FAIL_FID_IGIF 0x1504
120 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
126 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
127 stop ${SINGLEMDS} || error "Fail to stop MDS!"
129 total=$((total - usize * NTHREADS))
130 [ ${total} -eq 0 ] && break
131 lbase=$((lbase + usize))
143 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
144 reformat_external_journal
145 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
146 ldiskfs --reformat ${MDT_DEVNAME} $(mdsvdevname 1) > /dev/null ||
147 error "Fail to reformat the MDS!"
149 for ((i=$MINCOUNT; i<=$MAXCOUNT; i=$((i * FACTOR)))); do
150 local nfiles=$((i - BCOUNT))
152 echo "+++ start to create for ${i} files set at: $(date) +++"
153 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
154 error "Fail to create files!"
155 echo "+++ end to create for ${i} files set at: $(date) +++"
158 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
159 error "Fail to start MDS!"
161 echo "start lfsck_namespace for ${i} files set at: $(date)"
162 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
165 local STATUS=$($SHOW_NAMESPACE |
166 awk '/^status/ { print $2 }')
167 [ "$STATUS" == "completed" ] && break
168 sleep 3 # check status every 3 seconds
171 echo "end lfsck_namespace for ${i} files set at: $(date)"
172 SPEED=$($SHOW_NAMESPACE |
173 awk '/^average_speed_phase1/ { print $2 }')
174 echo "lfsck_namespace speed is ${SPEED}/sec"
175 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
178 run_test 0 "lfsck namespace performance (routine case) without load"
185 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
186 reformat_external_journal
187 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
188 ldiskfs --reformat ${MDT_DEVNAME} $(mdsvdevname 1) > /dev/null ||
189 error "Fail to reformat the MDS!"
191 for ((i=$MINCOUNT_REPAIR; i<=$MAXCOUNT_REPAIR; i=$((i * FACTOR)))); do
192 local nfiles=$((i - BCOUNT))
194 echo "+++ start to create for ${i} files set at: $(date) +++"
195 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
196 error "Fail to create files!"
197 echo "+++ end to create for ${i} files set at: $(date) +++"
200 local stime=$(date +%s)
201 echo "backup/restore ${i} files start at: $(date)"
202 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
203 echo "backup/restore ${i} files end at: $(date)"
204 local etime=$(date +%s)
205 local delta=$((etime - stime))
206 [ $delta -gt 0 ] || delta=1
207 echo "backup/restore ${i} files used ${delta} seconds"
208 echo "backup/restore speed is $((i / delta))/sec"
210 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
211 error "Fail to start MDS!"
213 echo "start lfsck_namespace for ${i} files set at: $(date)"
214 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
217 local STATUS=$($SHOW_NAMESPACE |
218 awk '/^status/ { print $2 }')
219 [ "$STATUS" == "completed" ] && break
220 sleep 3 # check status every 3 seconds
223 echo "end lfsck_namespace for ${i} files set at: $(date)"
224 local SPEED=$($SHOW_NAMESPACE |
225 awk '/^average_speed_phase1/ { print $2 }')
226 echo "lfsck_namespace speed is ${SPEED}/sec"
227 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
230 run_test 1 "lfsck namespace performance (backup/restore) without load"
235 for ((i=$MINCOUNT_REPAIR; i<=$MAXCOUNT_REPAIR; i=$((i * FACTOR)))); do
237 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
238 reformat_external_journal
239 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) \
240 --backfstype ldiskfs --reformat ${MDT_DEVNAME} \
241 $(mdsvdevname 1) > /dev/null ||
242 error "Fail to reformat the MDS!"
244 echo "+++ start to create for ${i} files set at: $(date) +++"
245 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
246 error "Fail to create files!"
247 echo "+++ end to create for ${i} files set at: $(date) +++"
249 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
250 error "Fail to start MDS!"
252 echo "start lfsck_namespace for ${i} files set at: $(date)"
253 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
256 local STATUS=$($SHOW_NAMESPACE |
257 awk '/^status/ { print $2 }')
258 [ "$STATUS" == "completed" ] && break
259 sleep 3 # check status every 3 seconds
262 echo "end lfsck_namespace for ${i} files set at: $(date)"
263 local SPEED=$($SHOW_NAMESPACE |
264 awk '/^average_speed_phase1/ { print $2 }')
265 echo "lfsck_namespace speed is ${SPEED}/sec"
266 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
269 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
272 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
274 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
279 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
280 reformat_external_journal
281 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
282 ldiskfs --reformat ${MDT_DEVNAME} $(mdsvdevname 1) > /dev/null ||
283 error "Fail to reformat the MDS!"
285 for ((i=$inc_count; i<=$BASE_COUNT; i=$((i + inc_count)))); do
286 local nfiles=$((i - BCOUNT))
288 echo "+++ start to create for ${i} files set at: $(date) +++"
289 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
290 error "Fail to create files!"
291 echo "+++ end to create for ${i} files set at: $(date) +++"
295 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
296 error "Fail to start MDS!"
298 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
299 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
302 local STATUS=$($SHOW_NAMESPACE |
303 awk '/^status/ { print $2 }')
304 [ "$STATUS" == "completed" ] && break
305 sleep 3 # check status every 3 seconds
308 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
309 local FULL_SPEED=$($SHOW_NAMESPACE |
310 awk '/^average_speed_phase1/ { print $2 }')
311 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
312 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
313 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
316 for ((j=$inc_speed; j<$FULL_SPEED; j=$((j + inc_speed)))); do
317 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
318 error "Fail to start MDS!"
320 $STOP_LFSCK > /dev/null 2>&1
321 echo "start lfsck_namespace with speed ${j} at: $(date)"
322 $START_NAMESPACE --reset -s ${j} ||
323 error "Fail to start lfsck_namespace with speed ${j}!"
324 # lfsck_namespace will be paused when MDS stop,
325 # and will be restarted automatically when mount up again.
326 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
328 local nfiles=$(((i - BCOUNT) / 2))
330 echo "+++ start to create for ${i} files set at: $(date) +++"
331 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
332 error "Fail to create files!"
333 echo "+++ end to create for ${i} files set at: $(date) +++"
338 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
339 error "Fail to start MDS!"
341 $STOP_LFSCK /dev/null 2>&1
342 echo "start lfsck_namespace with full speed at: $(date)"
343 $START_NAMESPACE --reset -s 0 ||
344 error "Fail to start lfsck_namespace with full speed!"
345 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
347 local nfiles=$(((i - BCOUNT) / 2))
349 echo "+++ start to create for ${i} files set at: $(date) +++"
350 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
351 error "Fail to create files!"
352 echo "+++ end to create for ${i} files set at: $(date) +++"
354 run_test 3 "lfsck namespace impact on create performance"
359 $RLCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout
364 echo "***** Start layout LFSCK on all devices at: $(date) *****"
365 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
367 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
368 mdd.${MDT_DEV}.lfsck_layout |
369 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
373 echo "***** End layout LFSCK on all devices at: $(date) *****"
375 for n in $(seq $MDSCOUNT); do
378 local SPEED=$(show_layout ${n} |
379 awk '/^average_speed_phase1/ { print $2 }')
381 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
390 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
391 local basedir="$mntpt/lfsck/$idx1/$idx2"
394 error_noexit "(11) Fail to mkdir $mntpt"
398 mount_client $mntpt || {
399 error_noexit "(12) Fail to mount $mntpt"
405 error_noexit "(13) Fail to mkdir $basedir"
409 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
410 createmany -o ${basedir}/f $UNIT || {
412 error_noexit "(14) Fail to gen $UNIT files under $basedir"
415 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
424 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
425 for ((k=0; k<$MDSCOUNT; k++)); do
426 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
428 for ((l=1; l<=$cnt; l++)); do
429 layout_gen_one ${k} ${l} &
434 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
439 local saved_mdscount=$MDSCOUNT
440 local saved_ostcount=$OSTCOUNT
443 stopall > /dev/null || error "(1) Fail to stopall"
446 for ((i=1; i<=$saved_ostcount; i=$((i * 2)))); do
449 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
452 for ((j=$MINSUBDIR; j<=$MAXSUBDIR; j=$((j * FACTOR)))); do
454 formatall > /dev/null ||
455 error "(2) Fail to formatall, subdirs=${j}"
458 setupall > /dev/null ||
459 error "(3) Fail to setupall, subdirs=${j}"
462 error "(4) Fail to mkdir $LFSCKDIR, subdirs=${j}"
464 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
465 error "(5) Fail to setstripe on $LFSCKDIR, subdirs=${j}"
468 layout_gen_set ${j} || RC=$?
470 error "(6) Fail to generate set $RC, subdirs=${j}"
473 layout_test_one || RC=$?
475 error "(7) LFSCK failed with $RC, subdirs=${j}"
479 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
482 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
485 MDSCOUNT=$saved_mdscount
486 OSTCOUNT=$saved_ostcount
489 formatall > /dev/null || error "(9) Fail to stopall"
495 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
498 echo "Inject failure stub to simulate dangling reference"
499 #define OBD_FAIL_LFSCK_DANGLING 0x1610
500 for i in $(seq $OSTCOUNT); do
501 do_facet ost${i} $LCTL set_param fail_loc=0x1610
506 for i in $(seq $OSTCOUNT); do
507 do_facet ost${i} $LCTL set_param fail_loc=0
510 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
514 local saved_mdscount=$MDSCOUNT
517 stopall > /dev/null || error "(1) Fail to stopall"
519 for ((i=1; i<=$saved_mdscount; i++)); do
522 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
525 for ((j=$MINSUBDIR; j<=$MAXSUBDIR; j=$((j * FACTOR)))); do
527 formatall > /dev/null ||
528 error "(2) Fail to formatall, subdirs=${j}"
531 setupall > /dev/null ||
532 error "(3) Fail to setupall, subdirs=${j}"
535 error "(4) Fail to mkdir $LFSCKDIR, subdirs=${j}"
537 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
538 error "(5) Fail to setstripe on $LFSCKDIR, subdirs=${j}"
541 layout_gen_set ${j} || RC=$?
543 error "(6) Fail to generate set $RC, subdirs=${j}"
546 layout_test_one || RC=$?
548 error "(7) LFSCK failed with $RC, subdirs=${j}"
552 stopall > /dev/null || error "(8) Fail to stopall"
555 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
558 MDSCOUNT=$saved_mdscount
561 formatall > /dev/null || error "(9) Fail to stopall"
567 run_test 5a "lfsck layout performance (routine case) without load for DNE"
570 echo "Inject failure stub to simulate dangling reference"
571 #define OBD_FAIL_LFSCK_DANGLING 0x1610
572 for i in $(seq $OSTCOUNT); do
573 do_facet ost${i} $LCTL set_param fail_loc=0x1610
578 for i in $(seq $OSTCOUNT); do
579 do_facet ost${i} $LCTL set_param fail_loc=0
582 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
584 layout_fast_create() {
588 local ldir="/test-${lbase}"
593 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
594 local usize=$((count / NTHREADS))
595 [ ${usize} -eq 0 ] && break
596 local tdir=${ldir}-${cycle}-
598 echo "[cycle: $cycle] [threads: $threads]"\
599 "[files: $count] [basedir: $tdir]"
603 total=$((total - usize * NTHREADS))
604 [ $total -eq 0 ] && break
605 lbase=$((lbase + usize))
617 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
620 stopall > /dev/null || error "(1) Fail to stopall"
622 local saved_mdscount=$MDSCOUNT
626 formatall > /dev/null || error "(2) Fail to formatall"
629 setupall > /dev/null || error "(3) Fail to setupall"
631 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
633 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
634 error "(5) Fail to setstripe on $LFSCKDIR"
637 layout_gen_set $TOTSUBDIR || RC=$?
639 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
642 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
643 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
644 error "(7) Fail to start layout LFSCK"
646 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
647 mdd.${MDT_DEV}.lfsck_layout |
648 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
650 error "(8) layout LFSCK cannot finished in time"
652 echo "***** End layout LFSCK on single MDS at: $(date) *****"
654 local SPEED=$(show_layout 1 |
655 awk '/^average_speed_phase1/ { print $2 }')
656 echo "lfsck_layout full_speed is $SPEED objs/sec"
658 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
659 local nfiles=$((inc_count / 2))
662 for ((m=0, n=$INCFACTOR; n<100;
663 m=$((m + inc_count)), n=$((n + INCFACTOR)))); do
664 local sl=$((SPEED * n / 100))
666 $STOP_LFSCK > /dev/null 2>&1
668 echo "start lfsck_layout with speed ${sl} at: $(date)"
669 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
671 "(9) Fail to start lfsck_layout with speed ${sl}"
673 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
674 layout_fast_create $nfiles ${m} $NTHREADS ||
675 lfsck_detach_error "(10) Fail to create files"
676 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
679 $STOP_LFSCK > /dev/null 2>&1
681 echo "start lfsck_layout with full speed at: $(date)"
682 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
684 "(11) Fail to start lfsck_layout with full speed"
686 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
687 layout_fast_create $nfiles ${m} $NTHREADS ||
688 lfsck_detach_error "(12) Fail to create files"
689 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
692 $STOP_LFSCK > /dev/null 2>&1
694 echo "create without lfsck_layout run back-ground"
695 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
696 layout_fast_create $nfiles ${m} $NTHREADS ||
697 lfsck_detach_error "(13) Fail to create files"
698 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
703 stopall > /dev/null || error "(14) Fail to stopall"
705 MDSCOUNT=$saved_mdscount
708 formatall > /dev/null || error "(15) Fail to stopall"
710 run_test 6 "lfsck layout impact on create performance"
712 # cleanup the system at last