2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:autoindent:shiftwidth=4:tabstop=4:
5 # Run select tests by setting ONLY, or as arguments to the script.
6 # Skip specific tests by setting EXCEPT.
8 # Run test by setting NOSETUP=true when ltest has setup env for us
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
14 ALWAYS_EXCEPT="$OST_POOLS_EXCEPT"
15 # bug number for skipped test: -
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 [ "$ALWAYS_EXCEPT$EXCEPT" ] && \
19 echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`"
24 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
25 . $LUSTRE/tests/test-framework.sh
27 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
30 check_and_setup_lustre
37 FAIL_ON_ERROR=${FAIL_ON_ERROR:-true}
41 SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
42 GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
45 TSTUSR=${TSTUSR:-"quota_usr"}
47 RUNAS="runas -u $TSTID -g $TSTID"
53 NON_EXISTANT_POOL=nonexistantpool
54 NON_EXISTANT_FS=nonexistantfs
55 INVALID_POOL=some_invalid_pool_name
58 TGT_MAX=$((TGT_COUNT-1))
60 TGT_LIST=$(seq $TGT_FIRST $TGT_STEP $TGT_MAX)
61 TGT_LIST2=$(seq $TGT_FIRST 2 $TGT_MAX)
63 TGT_ALL="$FSNAME-OST[$TGT_FIRST-$TGT_MAX/1]"
64 TGT_HALF="$FSNAME-OST[$TGT_FIRST-$TGT_MAX/2]"
66 TGT_UUID=$(for i in $TGT_LIST; do printf "$FSNAME-OST%04x_UUID " $i; done)
67 TGT_UUID2=$(for i in $TGT_LIST2; do printf "$FSNAME-OST%04x_UUID " $i; done)
72 local count=${3:-"-1"}
77 $SETSTRIPE -c $count -p $pool $dir -o $idx
79 $SETSTRIPE -c $count -p $pool $dir
82 error "$SETSTRIPE -p $pool $dir failed."
88 local count=${3:-"-1"}
89 local index=${4:-"-1"}
91 $SETSTRIPE -o $index -c $count -p $pool $file
93 error "$SETSTRIPE -p $pool $file failed."
99 for i in $(do_facet $SINGLEMDS lctl pool_list $FSNAME.$pool | \
100 grep -v "^Pool:" | sed -e 's/_UUID$//;s/^.*-OST//'); do
101 res="$res $(printf "%d" 0x$i)"
106 check_dir_in_pool() {
109 local res=$($GETSTRIPE $dir | grep "^stripe_count:" \
110 | cut -d ':' -f 5 | tr -d "[:blank:]")
111 if [[ "$res" == "$pool" ]]; then
114 error found $res instead of $pool
119 check_file_in_pool() {
120 local osts=$(osts_in_pool $2)
121 check_file_in_osts $1 "$osts" $3
124 check_file_in_osts() {
126 local pool_list=${2:-$TGT_LIST}
128 local res=$($GETSTRIPE $file | grep 0x | cut -f2)
132 found=$(echo :$pool_list: | tr " " ":" | grep :$i:)
133 if [[ "$found" == "" ]]; then
134 echo "pool list: $pool_list"
135 echo "striping: $res"
137 error "$file not allocated from OSTs $pool_list."
142 local ost_count=$($GETSTRIPE $file | grep 0x | wc -l)
143 [[ -n "$count" ]] && [[ $ost_count -ne $count ]] && \
144 { error "Stripe count $count expected; got $ost_count" && return 1; }
150 $GETSTRIPE -v $1 | grep "^lmm_pool:" | tr -d "[:blank:]" | cut -f 2 -d ':'
153 check_file_not_in_pool() {
156 local res=$(file_pool $file)
158 if [[ "$res" == "$pool" ]]; then
159 error "File $file is in pool: $res"
166 check_dir_not_in_pool() {
169 local res=$($GETSTRIPE -v $dir | grep "^stripe_count" | head -1 | \
171 if [[ "$res" == "$pool" ]]; then
172 error "File $dir is in pool: $res"
181 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool" ""\
182 ||error "Failed to remove targets from pool: $pool"
188 local tgt="${3}$(lctl get_param -n lov.$FSNAME-*.pools.$pool | \
189 sort -u | tr '\n' ' ')"
191 do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool $osts
193 [[ $RC -ne 0 ]] && return $RC
195 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool | \
196 sort -u | tr '\n' ' ' " "$tgt" || RC=1
197 [[ $RC -ne 0 ]] && error "pool_add failed: $1; $2"
201 create_pool_nofail() {
202 create_pool $FSNAME.$1
205 error "Pool creation of $1 failed"
210 create_pool $FSNAME.$1
213 error "Pool creation of $1 succeeded; should have failed"
218 # Destroy pools from previous test runs
219 for p in $(do_facet $SINGLEMDS lctl pool_list $FSNAME | \
220 grep $FSNAME.pool[0-$OSTCOUNT]); do
223 rm -rf $DIR/d0.${TESTSUITE}
231 trap "cleanup_pools $FSNAME" EXIT
235 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
236 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
240 # Tests for new commands added
243 echo "Creating a pool with a 1 character pool name"
246 echo "Creating a pool with a 10 character pool name"
247 create_pool_nofail p123456789
248 destroy_pool p123456789
250 echo "Creating a pool with a 16 character pool name"
251 create_pool_nofail p123456789123456
252 destroy_pool p123456789123456
254 echo "Creating a pool with a 17 character pool name; should fail"
255 create_pool_fail p1234567891234567
257 echo "Creating a pool with a 1000 character pool name; should fail"
259 for i in $(seq 1 999); do pool_name=${pool_name}"o"; done
260 create_pool_fail $pool_name
262 echo "pool_new should fail if fs-name or poolname are missing."
263 do_facet $SINGLEMDS lctl pool_new .pool1 2>/dev/null
265 error "pool_new did not fail even though fs-name was missing."
266 do_facet $SINGLEMDS lctl pool_new pool1 2>/dev/null
268 error "pool_new did not fail even though fs-name was missing."
269 do_facet $SINGLEMDS lctl pool_new ${FSNAME}. 2>/dev/null
271 error "pool_new did not fail even though pool name was missing."
272 do_facet $SINGLEMDS lctl pool_new . 2>/dev/null
274 error "pool_new did not fail even though pool name and fs-name " \
276 do_facet $SINGLEMDS lctl pool_new ${FSNAME},pool1 2>/dev/null
278 error "pool_new did not fail even though pool name format was wrong"
279 do_facet $SINGLEMDS lctl pool_new ${FSNAME}/pool1 2>/dev/null
281 error "pool_new did not fail even though pool name format was wrong"
283 do_facet $SINGLEMDS lctl pool_new ${FSNAME}.p 2>/dev/null
285 error "pool_new did not fail even though pool1 existed"
289 run_test 1 "Test lctl pool_new ========================================="
295 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null
297 error " pool_add did not fail even though pool did " \
300 run_test 2a "pool_add: non-existant pool"
304 do_facet $SINGLEMDS lctl pool_add $FSNAME.p1234567891234567890 \
305 $FSNAME-OST0000 2>/dev/null
307 error "pool_add did not fail even though pool name was invalid."
309 run_test 2b "pool_add: Invalid pool name"
311 # Testing various combinations of OST name list
317 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
321 create_pool_nofail $POOL
324 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL OST0000
325 RC=$?; [[ $RC -eq 0 ]] || \
326 error "pool_add failed. $FSNAME $POOL OST0000: $RC"
327 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000
331 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000
332 RC=$?; [[ $RC -eq 0 ]] || \
333 error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000: $RC"
334 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000
337 # 3. $FSNAME-OST0000_UUID
338 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID
339 RC=$?; [[ $RC -eq 0 ]] || \
340 error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000_UUID: $RC"
341 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID
344 # 4. $FSNAME-OST[0,1,2,3,]
346 for i in $TGT_LIST; do TGT=${TGT}$(printf "$i," $i); done
348 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
350 error "pool_add failed. $FSNAME.$POOL $TGT. $RC"
351 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT
354 # 5. $FSNAME-OST[0-5/1]
355 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL
356 RC=$?; [[ $RC -eq 0 ]] || \
357 error "pool_add failed. $FSNAME $POOL" "$TGT_ALL $RC"
358 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | \
359 sort -u | tr '\n' ' ' " "$TGT_UUID" || error "Add to pool failed"
360 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT_ALL
365 run_test 2c "pool_add: OST index combinations ==========================="
372 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
376 create_pool_nofail $POOL
378 TGT=$(printf "$FSNAME-OST%04x_UUID " $OSTCOUNT)
379 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
380 RC=$?; [[ $RC -ne 0 ]] || \
381 error "pool_add succeeded for an OST ($TGT) that does not exist."
385 run_test 2d "pool_add: OSTs that don't exist should be rejected ========"
393 $LCTL get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
397 create_pool_nofail $POOL
399 TGT="$FSNAME-OST0000_UUID "
400 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
401 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | \
402 sort -u | tr '\n' ' ' " "$TGT" || error "Add to pool failed"
403 RESULT=$(do_facet $SINGLEMDS \
404 "LOCALE=C $LCTL pool_add $FSNAME.$POOL $TGT 2>&1")
409 error "pool_add succeeded for an OST that was already in the pool."
411 [[ $(grep "already in pool" <<< $RESULT) ]] || \
412 error "pool_add failed as expected but error message not as expected."
416 run_test 2e "pool_add: OST already in a pool should be rejected ========"
420 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
424 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null
426 error "pool_remove did not fail even though pool did not exist."
428 run_test 3a "pool_remove: non-existant pool"
432 do_facet $SINGLEMDS lctl pool_remove ${NON_EXISTANT_FS}.$POOL OST0000 2>/dev/null
434 error "pool_remove did not fail even though fsname did not exist."
436 run_test 3b "pool_remove: non-existant fsname"
440 do_facet $SINGLEMDS lctl pool_remove $FSNAME.p1234567891234567890 \
441 $FSNAME-OST0000 2>/dev/null
443 error "pool_remove did not fail even though pool name was invalid."
445 run_test 3c "pool_remove: Invalid pool name"
447 # Testing various combinations of OST name list
450 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
454 create_pool_nofail $POOL
455 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL OST0000
456 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000
458 error "pool_remove failed. $FSNAME $POOL OST0000"
461 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000
462 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000
464 error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000"
467 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID
468 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID
470 error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000_UUID"
473 add_pool $POOL $TGT_ALL "$TGT_UUID"
474 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT_ALL
476 error "pool_remove failed. $FSNAME $POOL" $TGT_ALL
481 run_test 3d "pool_remove: OST index combinations ==========================="
485 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
489 do_facet $SINGLEMDS lctl pool_destroy $FSNAME.$POOL 2>/dev/null
491 error "pool_destroy did not fail even though pool did not exist."
493 run_test 4a "pool_destroy: non-existant pool"
497 do_facet $SINGLEMDS lctl pool_destroy ${NON_EXISTANT_FS}.$POOL 2>/dev/null
499 error "pool_destroy did not fail even though the filesystem did not exist."
501 run_test 4b "pool_destroy: non-existant fs-name"
505 create_pool_nofail $POOL
506 add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID "
508 do_facet $SINGLEMDS lctl pool_destroy ${FSNAME}.$POOL
510 error "pool_destroy succeeded with a non-empty pool."
513 run_test 4c "pool_destroy: non-empty pool ==============================="
518 $LCMD pool_list 2>/dev/null
520 error "pool_list did not fail even though fsname was not mentioned."
522 destroy_pool $POOL 2>/dev/null
523 destroy_pool $POOL2 2>/dev/null
525 create_pool_nofail $POOL
526 create_pool_nofail $POOL2
527 $LCMD pool_list $FSNAME
529 error "pool_list $FSNAME failed."
531 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL
533 $LCMD pool_list $FSNAME.$POOL
535 error "pool_list $FSNAME.$POOL failed."
537 $LCMD pool_list ${NON_EXISTANT_FS} 2>/dev/null
539 error "pool_list did not fail for a non-existant fsname $NON_EXISTANT_FS"
541 $LCMD pool_list ${FSNAME}.$NON_EXISTANT_POOL 2>/dev/null
543 error "pool_list did not fail for a non-existant pool $NON_EXISTANT_POOL"
545 if [[ ! $(grep $SINGLEMDS <<< $LCMD) ]]; then
546 echo $LCMD pool_list $DIR
549 error "pool_list failed for $DIR"
552 $LCMD pool_list ${DIR}/d1
554 error "pool_list failed for ${DIR}/d1"
557 rm -rf ${DIR}nonexistant
558 $LCMD pool_list ${DIR}nonexistant 2>/dev/null
560 error "pool_list did not fail for invalid mountpoint ${DIR}nonexistant"
568 # Issue commands from client
572 # Issue commands from MDS
573 sub_test_5 "do_facet $SINGLEMDS lctl"
574 sub_test_5 "do_facet $SINGLEMDS lfs"
577 run_test 5 "lfs/lctl pool_list"
581 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
582 local POOL_DIR=$POOL_ROOT/dir_tst
583 local POOL_FILE=$POOL_ROOT/file_tst
585 create_pool_nofail $POOL
587 do_facet $SINGLEMDS lctl pool_list $FSNAME
589 error "pool_list $FSNAME failed."
591 add_pool $POOL $TGT_ALL "$TGT_UUID"
594 $SETSTRIPE -c -1 -p $POOL $POOL_DIR
596 error "$SETSTRIPE -p $POOL failed."
597 check_dir_in_pool $POOL_DIR $POOL
599 # If an invalid pool name is specified, the command should fail
600 $SETSTRIPE -c 2 -p $INVALID_POOL $POOL_DIR 2>/dev/null
602 error "setstripe to invalid pool did not fail."
604 # If the pool name does not exist, the command should fail
605 $SETSTRIPE -c 2 -p $NON_EXISTANT_POOL $POOL_DIR 2>/dev/null
607 error "setstripe to non-existant pool did not fail."
609 # lfs setstripe should work as before if a pool name is not specified.
610 $SETSTRIPE -c -1 $POOL_DIR
612 error "$SETSTRIPE -p $POOL_DIR failed."
613 $SETSTRIPE -c -1 $POOL_FILE
615 error "$SETSTRIPE -p $POOL_FILE failed."
617 # lfs setstripe should fail if a start index that is outside the
619 create_pool_nofail $POOL2
620 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID "
621 $SETSTRIPE -o 1 -p $POOL2 $ROOT_POOL/$tfile 2>/dev/null
623 error "$SETSTRIPE with start index outside the pool did not fail."
626 run_test 6 "getstripe/setstripe"
630 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
632 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
634 create_pool_nofail $POOL
635 create_pool_nofail $POOL2
637 local start=$((TGT_FIRST+1))
638 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \
639 $FSNAME-OST[$start-$TGT_MAX/2]
641 add_pool $POOL $TGT_HALF "$TGT_UUID2"
643 create_dir $POOL_ROOT/dir1 $POOL
644 create_dir $POOL_ROOT/dir2 $POOL2
645 check_dir_in_pool $POOL_ROOT/dir1 $POOL
646 check_dir_in_pool $POOL_ROOT/dir1 $POOL
649 createmany -o $POOL_ROOT/dir1/$tfile $numfiles || \
650 error "createmany $POOL_ROOT/dir1/$tfile failed!"
652 for file in $POOL_ROOT/dir1/*; do
653 check_file_in_pool $file $POOL
656 createmany -o $POOL_ROOT/dir2/$tfile $numfiles || \
657 error "createmany $POOL_ROOT/dir2/$tfile failed!"
658 for file in $POOL_ROOT/dir2/*; do
659 check_file_in_pool $file $POOL2
662 rm -rf $POOL_ROOT/dir?
666 run_test 11 "OSTs in overlapping/multiple pools"
670 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
672 [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
674 create_pool_nofail $POOL
675 create_pool_nofail $POOL2
677 local start=$((TGT_FIRST+1))
678 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \
679 $FSNAME-OST[$start-$TGT_MAX/2]
681 add_pool $POOL $TGT_HALF "$TGT_UUID2"
683 echo creating some files in $POOL and $POOL2
685 create_dir $POOL_ROOT/dir1 $POOL
686 create_dir $POOL_ROOT/dir2 $POOL2
687 create_file $POOL_ROOT/file1 $POOL
688 create_file $POOL_ROOT/file2 $POOL2
690 echo Checking the files created
691 check_dir_in_pool $POOL_ROOT/dir1 $POOL
692 check_dir_in_pool $POOL_ROOT/dir2 $POOL2
693 check_file_in_pool $POOL_ROOT/file1 $POOL
694 check_file_in_pool $POOL_ROOT/file2 $POOL2
696 echo Changing the pool membership
697 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST[$TGT_FIRST]
698 do_facet $SINGLEMDS lctl pool_list $FSNAME.$POOL
699 FIRST_UUID=$(echo $TGT_UUID | awk '{print $1}')
700 add_pool $POOL2 $FSNAME-OST[$TGT_FIRST] "$FIRST_UUID "
701 do_facet $SINGLEMDS lctl pool_list $FSNAME.$POOL2
703 echo Checking the files again
704 check_dir_in_pool $POOL_ROOT/dir1 $POOL
705 check_dir_in_pool $POOL_ROOT/dir2 $POOL2
706 check_file_in_osts $POOL_ROOT/file1 "$TGT_LIST2"
707 check_file_in_osts $POOL_ROOT/file2 "$(seq $start 2 $TGT_MAX)"
709 echo Creating some more files
710 create_dir $POOL_ROOT/dir3 $POOL
711 create_dir $POOL_ROOT/dir4 $POOL2
712 create_file $POOL_ROOT/file3 $POOL
713 create_file $POOL_ROOT/file4 $POOL2
715 echo Checking the new files
716 check_file_in_pool $POOL_ROOT/file3 $POOL
717 check_file_in_pool $POOL_ROOT/file4 $POOL2
721 run_test 12 "OST Pool Membership"
725 [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
727 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
731 create_pool_nofail $POOL
732 add_pool $POOL $TGT_ALL "$TGT_UUID"
734 create_dir $POOL_ROOT/dir1 $POOL -1
735 createmany -o $POOL_ROOT/dir1/$tfile $numfiles || \
736 error "createmany $POOL_ROOT/dir1/$tfile failed!"
737 for file in $POOL_ROOT/dir1/*; do
738 check_file_in_pool $file $POOL $OSTCOUNT
741 create_file $POOL_ROOT/dir1/file1 $POOL 1 $TGT_FIRST
742 create_file $POOL_ROOT/dir1/file2 $POOL 1 $((TGT_FIRST+1))
743 create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2))
744 check_file_in_pool $POOL_ROOT/dir1/file1 $POOL 1
745 check_file_in_pool $POOL_ROOT/dir1/file2 $POOL 1
746 create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2))
747 check_file_in_osts $POOL_ROOT/dir1/file1 "$TGT_FIRST"
748 check_file_in_osts $POOL_ROOT/dir1/file2 "$((TGT_FIRST+1))"
749 check_file_in_osts $POOL_ROOT/dir1/file3 "$((TGT_FIRST+2))"
751 create_dir $POOL_ROOT/dir2 $POOL $count
752 createmany -o $POOL_ROOT/dir2/$tfile $numfiles || \
753 error "createmany $POOL_ROOT/dir2/$tfile failed!"
754 for file in $POOL_ROOT/dir2/*; do
755 check_file_in_pool $file $POOL $count
758 create_dir $POOL_ROOT/dir3 $POOL $count $((TGT_FIRST+1))
759 createmany -o $POOL_ROOT/dir3/$tfile_ $numfiles || \
760 error "createmany $POOL_ROOT/dir3/$tfile_ failed!"
761 for file in $POOL_ROOT/dir3/*; do
762 check_file_in_pool $file $POOL $count
765 create_dir $POOL_ROOT/dir4 $POOL 1
766 createmany -o $POOL_ROOT/dir4/$tfile_ $numfiles || \
767 error "createmany $POOL_ROOT/dir4/$tfile_ failed!"
768 for file in $POOL_ROOT/dir4/*; do
769 check_file_in_pool $file $POOL 1
772 create_dir $POOL_ROOT/dir5 $POOL 1 $((TGT_FIRST+2))
773 createmany -o $POOL_ROOT/dir5/$tfile_ $numfiles || \
774 error "createmany $POOL_ROOT/dir5/$tfile_ failed!"
775 for file in $POOL_ROOT/dir5/*; do
776 check_file_in_pool $file $POOL 1
777 check_file_in_osts $file "$((TGT_FIRST+2))"
780 rm -rf create_dir $POOL_ROOT/dir?
784 run_test 13 "Striping characteristics in a pool"
788 [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
790 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
794 # Create a new filesystem that is guaranteed to be balanced.
798 create_pool_nofail $POOL
799 create_pool_nofail $POOL2
801 add_pool $POOL $TGT_HALF "$TGT_UUID2"
802 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID "
804 create_dir $POOL_ROOT/dir1 $POOL 1
805 create_file $POOL_ROOT/dir1/file $POOL 1
806 local OST=$($GETSTRIPE $POOL_ROOT/dir1/file | grep 0x | cut -f2)
808 while [[ $i -lt $numfiles ]];
811 [[ $OST -gt $TGT_MAX ]] && OST=$TGT_FIRST
813 # echo "Iteration: $i OST: $OST"
814 create_file $POOL_ROOT/dir1/file${i} $POOL 1
815 check_file_in_pool $POOL_ROOT/dir1/file${i} $POOL
819 # Fill up OST0 until it is nearly full.
820 # Create 9 files of size OST0_SIZE/10 each.
821 create_dir $POOL_ROOT/dir2 $POOL2 1
822 $LFS df $POOL_ROOT/dir2
823 echo "Filling up OST0"
824 OST0_SIZE=`$LFS df $POOL_ROOT/dir2 | awk '/\[OST:0\]/ {print $4}'`
825 FILE_SIZE=$((OST0_SIZE/1024/10))
827 while [[ $i -lt 10 ]];
829 dd if=/dev/zero of=$POOL_ROOT/dir2/f${i} bs=1M count=$FILE_SIZE
832 $LFS df $POOL_ROOT/dir2
834 # OST $TGT_FIRST is no longer favored; but it may still be used.
835 create_dir $POOL_ROOT/dir3 $POOL 1
836 create_file $POOL_ROOT/dir3/file $POOL 1
837 createmany -o $POOL_ROOT/dir3/$tfile_ $numfiles || \
838 error "createmany $POOL_ROOT/dir3/$tfile_ failed!"
839 for file in $POOL_ROOT/dir3/*; do
840 check_file_in_pool $file $POOL
846 run_test 14 "Round robin and QOS striping within a pool"
850 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
854 while [[ $i -lt $OSTCOUNT ]]
856 create_pool_nofail pool${i}
858 local tgt=$(printf "$FSNAME-OST%04x_UUID " $i)
859 add_pool pool${i} "$FSNAME-OST[$i]" "$tgt"
860 create_dir $POOL_ROOT/dir${i} pool${i}
861 createmany -o $POOL_ROOT/dir$i/$tfile $numfiles || \
862 error "createmany $POOL_ROOT/dir$i/$tfile failed!"
864 for file in $POOL_ROOT/dir$i/*; do
865 check_file_in_osts $file $i
873 run_test 15 "One directory per OST/pool"
877 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
881 create_pool_nofail $POOL
883 add_pool $POOL $TGT_HALF "$TGT_UUID2"
885 local dir=$POOL_ROOT/$tdir
886 create_dir $dir $POOL
888 for i in $(seq 1 10); do
893 createmany -o $dir/$tfile $numfiles || \
894 error "createmany $dir/$tfile failed!"
896 for file in $dir/*; do
897 check_file_in_pool $file $POOL
900 rm -rf $POOL_ROOT/$tdir
904 run_test 16 "Inheritance of pool properties"
908 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
912 create_pool_nofail $POOL
914 add_pool $POOL $TGT_ALL "$TGT_UUID"
916 local dir=$POOL_ROOT/dir
917 create_dir $dir $POOL
919 createmany -o $dir/${tfile}1_ $numfiles || \
920 error "createmany $dir/${tfile}1_ failed!"
922 for file in $dir/*; do
923 check_file_in_pool $file $POOL
928 createmany -o $dir/${tfile}2_ $numfiles || \
929 error "createmany $dir/${tfile}2_ failed!"
934 run_test 17 "Referencing an empty pool"
942 sync; sleep 5 # give pending IO a chance to go to disk
943 stat=$(createmany -o $cdir/${tfile} $numfiles)
946 time=$(echo $stat | cut -f 5 -d ' ')
947 echo $stat >> /dev/stderr
953 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
955 local plaindir=$POOL_ROOT/plaindir
956 local pooldir=$POOL_ROOT/pooldir
964 echo "Create performance, iteration $i, $numfiles files x 3"
966 time1=$(create_perf $plaindir $numfiles)
967 echo "iter $i: $numfiles creates without pool: $time1"
968 t1=$(echo "scale=2; $t1 + $time1" | bc)
970 create_pool_nofail $POOL > /dev/null
971 add_pool $POOL $TGT_ALL "$TGT_UUID" > /dev/null
972 create_dir $pooldir $POOL
973 time2=$(create_perf $pooldir $numfiles)
974 echo "iter $i: $numfiles creates with pool: $time2"
975 t2=$(echo "scale=2; $t2 + $time2" | bc)
977 destroy_pool $POOL > /dev/null
978 time3=$(create_perf $pooldir $numfiles)
979 echo "iter $i: $numfiles creates with missing pool: $time3"
980 t3=$(echo "scale=2; $t3 + $time3" | bc)
985 time1=$(echo "scale=2; $t1 / $i" | bc)
986 echo Avg time taken for $numfiles creates without pool: $time1
987 time2=$(echo "scale=2; $t2 / $i" | bc)
988 echo Avg time taken for $numfiles creates with pool: $time2
989 time3=$(echo "scale=2; $t3 / $i" | bc)
990 echo Avg time taken for $numfiles creates with missing pool: $time3
992 # Set this high until we establish a baseline for what the degradation
995 diff=$(echo "scale=2; ($time2 - $time1) * 100 / $time1" | bc)
996 echo "No pool to wide pool: $diff %."
997 deg=$(echo "scale=2; $diff > $max" | bc)
998 [ "$deg" == "1" ] && error_ignore 23408 "Degradation with wide pool is $diff % (> $max %)"
1001 diff=$(echo "scale=2; ($time3 - $time1) * 100 / $time1" | bc)
1002 echo "No pool to missing pool: $diff %."
1003 deg=$(echo "scale=2; $diff > $max" | bc)
1004 [ "$deg" == "1" ] && error_ignore 23408 "Degradation with missing pool is $diff % (> $max %)"
1008 run_test 18 "File create in a directory which references a deleted pool"
1013 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1015 local dir1=$POOL_ROOT/dir1
1016 local dir2=$POOL_ROOT/dir2
1019 create_pool_nofail $POOL
1021 add_pool $POOL $TGT_HALF "$TGT_UUID2"
1023 create_dir $dir1 $POOL
1024 createmany -o $dir1/${tfile} $numfiles || \
1025 error "createmany $dir1/${tfile} failed!"
1026 for file in $dir1/*; do
1027 check_file_in_pool $file $POOL
1031 createmany -o $dir2/${tfile} $numfiles || \
1032 error "createmany $dir2/${tfile} failed!"
1033 for file in $dir2/*; do
1034 check_file_not_in_pool $file $POOL
1041 run_test 19 "Pools should not come into play when not specified"
1045 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1047 local dir1=$POOL_ROOT/dir1
1048 local dir2=$dir1/dir2
1049 local dir3=$dir1/dir3
1053 create_pool_nofail $POOL
1054 create_pool_nofail $POOL2
1056 add_pool $POOL $TGT_HALF "$TGT_UUID2"
1058 local start=$((TGT_FIRST+1))
1059 TGT=$(for i in `seq $start 2 $TGT_MAX`; \
1060 do printf "$FSNAME-OST%04x_UUID " $i; done)
1061 add_pool $POOL2 "$FSNAME-OST[$start-$TGT_MAX/2]" "$TGT"
1063 create_dir $dir1 $POOL
1064 create_file $dir1/file1 $POOL2
1065 create_dir $dir2 $POOL2
1068 $SETSTRIPE -c 1 $dir3 # No pool assignment
1070 $SETSTRIPE -c 1 $dir2/file4 # No pool assignment
1072 check_file_in_pool $dir1/file1 $POOL2
1073 check_file_in_pool $dir2/file2 $POOL2
1075 check_dir_not_in_pool $dir3 $POOL
1076 check_dir_not_in_pool $dir3 $POOL2
1078 check_file_not_in_pool $dir3/file3 $POOL
1079 check_file_not_in_pool $dir3/file3 $POOL2
1081 check_file_not_in_pool $dir2/file4 $POOL
1082 check_file_not_in_pool $dir2/file4 $POOL2
1088 run_test 20 "Different pools in a directory hierarchy."
1092 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1093 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
1097 local dir=$POOL_ROOT/dir
1099 create_pool_nofail $POOL
1101 add_pool $POOL $TGT_HALF "$TGT_UUID2"
1103 create_dir $dir $POOL $OSTCOUNT
1104 create_file $dir/file1 $POOL $OSTCOUNT
1105 $GETSTRIPE -v $dir/file1
1106 check_file_in_pool $dir/file1 $POOL
1112 run_test 21 "OST pool with fewer OSTs than stripe count"
1120 for c in $(seq 1 10);
1122 echo "Pool $pool, iteration $c"
1123 do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool OST[$TGT_FIRST-$TGT_MAX/$step] 2>/dev/null
1124 local TGT_SECOND=$(($TGT_FIRST+$step))
1125 if [ "$TGT_SECOND" -le "$TGT_MAX" ]; then
1126 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$pool OST[$TGT_SECOND-$TGT_MAX/$step]
1129 echo loop for $pool complete
1134 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1135 [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return
1139 create_pool_nofail $POOL
1140 add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID "
1141 create_pool_nofail $POOL2
1142 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID "
1147 create_dir $POOL_ROOT $POOL
1148 createmany -o $POOL_ROOT/${tfile} $numfiles || \
1149 error "createmany $POOL_ROOT/${tfile} failed!"
1154 run_test 22 "Simultaneous manipulation of a pool"
1158 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1159 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
1162 check_runas_id $TSTID $TSTID $RUNAS || {
1163 skip_env "User $RUNAS_ID does not exist - skipping"
1170 local BUNIT_SZ=1024 # min block quota unit(kB)
1171 local LOVNAME=`lctl get_param -n llite.*.lov.common_name | tail -n 1`
1172 local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.numobd`
1174 local dir=$POOL_ROOT/dir
1175 local file="$dir/$tfile-quota"
1177 create_pool_nofail $POOL
1179 local TGT=$(for i in `seq $TGT_FIRST 3 $TGT_MAX`; \
1180 do printf "$FSNAME-OST%04x_UUID " $i; done)
1181 add_pool $POOL "$FSNAME-OST[$TGT_FIRST-$TGT_MAX/3]" "$TGT"
1182 create_dir $dir $POOL
1184 $LFS quotaoff -ug $MOUNT
1185 $LFS quotacheck -ug $MOUNT
1186 LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1)))
1187 $LFS setquota -u $TSTUSR -b $LIMIT -B $LIMIT $dir
1189 $LFS quota -v -u $TSTUSR $dir
1191 $LFS setstripe $file -c 1 -p $POOL
1192 chown $TSTUSR.$TSTUSR $file
1196 LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BLK_SZ count=$((BUNIT_SZ*2)) || true
1197 $LFS quota -v -u $TSTUSR $dir
1198 cancel_lru_locks osc
1199 stat=$(LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BLK_SZ count=$BUNIT_SZ seek=$((BUNIT_SZ*2)) 2>&1)
1202 [[ $RC -eq 0 ]] && error "dd did not fail with EDQUOT."
1203 echo $stat | grep "Disk quota exceeded" > /dev/null
1204 [[ $? -eq 1 ]] && error "dd did not fail with EDQUOT."
1205 $LFS quota -v -u $TSTUSR $dir
1208 $LFS quotaoff -ug $MOUNT
1209 # $LFS setquota -u $TSTUSR -b $LIMIT -B $LIMIT $dir
1210 chown $TSTUSR.$TSTUSR $dir
1213 while [ $RC -eq 0 ];
1216 stat=$(LOCALE=C $RUNAS2 dd if=/dev/zero of=${file}$i bs=1M \
1217 count=$((LIMIT*LIMIT)) 2>&1)
1219 if [ $RC -eq 1 ]; then
1221 echo $stat | grep "Disk quota exceeded"
1222 [[ $? -eq 0 ]] && error "dd failed with EDQUOT"
1224 echo $stat | grep "No space left on device"
1225 [[ $? -ne 0 ]] && error "dd did not fail with ENOSPC; " \
1236 run_test 23 "OST pools and quota"
1240 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1241 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
1250 create_pool_nofail $POOL
1252 add_pool $POOL $TGT_ALL "$TGT_UUID"
1254 create_dir $POOL_ROOT/dir1 $POOL $OSTCOUNT
1256 mkdir $POOL_ROOT/dir2
1257 $SETSTRIPE $POOL_ROOT/dir2 -p $POOL -s 65536 -i 0 -c 1 || \
1258 error "$SETSTRIPE $POOL_ROOT/dir2 failed"
1260 mkdir $POOL_ROOT/dir3
1261 $SETSTRIPE $POOL_ROOT/dir3 -s 65536 -i 0 -c 1 || \
1262 error "$SETSTRIPE $POOL_ROOT/dir3 failed"
1264 mkdir $POOL_ROOT/dir4
1266 for i in $(seq 1 4);
1268 dir=${POOL_ROOT}/dir${i}
1277 createmany -o $dir/${tfile} $numfiles || \
1278 error "createmany $dir/${tfile} failed!"
1279 res=$($GETSTRIPE -v $dir | grep "^stripe_count:")
1280 if [ $? -ne 0 ]; then
1281 res=$($GETSTRIPE -v $dir | grep "^(Default) ")
1282 pool=$(cut -f 9 -d ' ' <<< $res)
1283 index=$(cut -f 7 -d ' ' <<< $res)
1284 size=$(cut -f 5 -d ' ' <<< $res)
1285 count=$(cut -f 3 -d ' ' <<< $res)
1287 pool=$(cut -f 8 -d ' ' <<< $res)
1288 index=$(cut -f 6 -d ' ' <<< $res)
1289 size=$(cut -f 4 -d ' ' <<< $res)
1290 count=$(cut -f 2 -d ' ' <<< $res)
1293 for file in $dir/*; do
1294 if [ "$pool" != "" ]; then
1295 check_file_in_pool $file $pool
1297 pool1=$(file_pool $file)
1298 count1=$($GETSTRIPE -v $file | grep "^lmm_stripe_count:" |\
1299 tr -d '[:blank:]' | cut -f 2 -d ':')
1300 size1=$($GETSTRIPE -v $file | grep "^lmm_stripe_size:" |\
1301 tr -d '[:blank:]' | cut -f 2 -d ':')
1302 [[ "$pool" != "$pool1" ]] && \
1303 error "Pool name ($pool) not inherited in $file($pool1)"
1304 [[ "$count" != "$count1" ]] && \
1305 error "Stripe count ($count) not inherited in $file ($count1)"
1306 [[ "$size" != "$size1" ]] && [[ "$size" != "0" ]] && \
1307 error "Stripe size ($size) not inherited in $file ($size1)"
1315 run_test 24 "Independence of pool from other setstripe parameters"
1319 local dev=$(mdsdevname ${SINGLEMDS//mds/})
1320 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1324 for i in $(seq 10); do
1325 create_pool_nofail pool$i
1326 do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool$i OST0000; sync"
1327 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool$i | \
1328 sort -u | tr '\n' ' ' " "$FSNAME-OST0000_UUID " || \
1329 error "pool_add failed: $1; $2"
1331 stop $SINGLEMDS || return 1
1332 start $SINGLEMDS ${dev} $MDS_MOUNT_OPTS || \
1333 { error "Failed to start $SINGLEMDS after stopping" && break; }
1334 wait_osc_import_state mds ost FULL
1338 # Veriy that the pool got created and is usable
1339 df $POOL_ROOT > /dev/null
1341 # Make sure OST0 can be striped on
1342 $SETSTRIPE -o 0 -c 1 $POOL_ROOT/$tfile
1343 STR=$($GETSTRIPE $POOL_ROOT/$tfile | grep 0x | cut -f2 | tr -d " ")
1344 rm $POOL_ROOT/$tfile
1345 if [[ "$STR" == "0" ]]; then
1346 echo "Creating a file in pool$i"
1347 create_file $POOL_ROOT/file$i pool$i || break
1348 check_file_in_pool $POOL_ROOT/file$i pool$i || break
1350 echo "OST 0 seems to be unavailable. Try later."
1356 run_test 25 "Create new pool and restart MDS ======================="
1359 local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.*clilov*.numobd`
1360 [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return
1362 local dev=$(mdsdevname ${SINGLEMDS//mds/})
1363 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1367 create_pool_nofail $POOL
1369 do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0000; sync"
1370 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
1371 sort -u | grep $FSNAME-OST0000_UUID " "$FSNAME-OST0000_UUID" || \
1372 error "pool_add failed: $1; $2"
1374 do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0002; sync"
1375 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
1376 sort -u | grep $FSNAME-OST0002_UUID" "$FSNAME-OST0002_UUID" || \
1377 error "pool_add failed: $1; $2"
1379 # Veriy that the pool got created and is usable
1381 echo "Creating files in $POOL"
1383 for ((i=0;i<10;i++)); do
1384 #OBD_FAIL_MDS_OSC_CREATE_FAIL 0x147
1385 #Fail OST0000 to make sure the objects create on the other ost in the pool
1386 do_facet $SINGLEMDS lctl set_param fail_loc=0x147
1387 do_facet $SINGLEMDS lctl set_param fail_val=0
1388 create_file $POOL_ROOT/file$i $POOL 1 -1 || break
1389 do_facet $SINGLEMDS lctl set_param fail_loc=0
1390 check_file_in_pool $POOL_ROOT/file$i $POOL || break
1394 run_test 26 "Choose other OSTs in the pool first in the creation remedy"
1398 complete $(basename $0) $SECONDS
1399 cleanup_pools $FSNAME
1400 check_and_cleanup_lustre