2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:autoindent:shiftwidth=4:tabstop=4:
5 # Run select tests by setting ONLY, or as arguments to the script.
6 # Skip specific tests by setting EXCEPT.
8 # Run test by setting NOSETUP=true when ltest has setup env for us
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
14 ALWAYS_EXCEPT="$OST_POOLS_EXCEPT"
15 # bug number for skipped test: -
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 [ "$ALWAYS_EXCEPT$EXCEPT" ] && \
19 echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`"
24 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
25 . $LUSTRE/tests/test-framework.sh
27 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
30 check_and_setup_lustre
35 POOLSLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
36 [ "$POOLSLOG" ] && rm -f $POOLSLOG || true
39 FAIL_ON_ERROR=${FAIL_ON_ERROR:-true}
43 SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
44 GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
47 TSTUSR=${TSTUSR:-"quota_usr"}
49 RUNAS="runas -u $TSTID -g $TSTID"
55 NON_EXISTANT_POOL=nonexistantpool
56 NON_EXISTANT_FS=nonexistantfs
57 INVALID_POOL=some_invalid_pool_name
60 TGT_MAX=$((TGT_COUNT-1))
62 TGT_LIST=$(seq $TGT_FIRST $TGT_STEP $TGT_MAX)
63 TGT_LIST2=$(seq $TGT_FIRST 2 $TGT_MAX)
65 TGT_ALL="$FSNAME-OST[$TGT_FIRST-$TGT_MAX/1]"
66 TGT_HALF="$FSNAME-OST[$TGT_FIRST-$TGT_MAX/2]"
68 TGT_UUID=$(for i in $TGT_LIST; do printf "$FSNAME-OST%04x_UUID " $i; done)
69 TGT_UUID2=$(for i in $TGT_LIST2; do printf "$FSNAME-OST%04x_UUID " $i; done)
74 local count=${3:-"-1"}
79 $SETSTRIPE -c $count -p $pool $dir -o $idx
81 $SETSTRIPE -c $count -p $pool $dir
84 error "$SETSTRIPE -p $pool $dir failed."
90 local count=${3:-"-1"}
91 local index=${4:-"-1"}
93 $SETSTRIPE -o $index -c $count -p $pool $file
95 error "$SETSTRIPE -p $pool $file failed."
101 for i in $(do_facet $SINGLEMDS lctl pool_list $FSNAME.$pool | \
102 grep -v "^Pool:" | sed -e 's/_UUID$//;s/^.*-OST//'); do
103 res="$res $(printf "%d" 0x$i)"
108 check_dir_in_pool() {
111 local res=$($GETSTRIPE $dir | grep "^stripe_count:" \
112 | cut -d ':' -f 5 | tr -d "[:blank:]")
113 if [[ "$res" == "$pool" ]]; then
116 error found $res instead of $pool
121 check_file_in_pool() {
122 local osts=$(osts_in_pool $2)
123 check_file_in_osts $1 "$osts" $3
126 check_file_in_osts() {
128 local pool_list=${2:-$TGT_LIST}
130 local res=$($GETSTRIPE $file | grep 0x | cut -f2)
134 found=$(echo :$pool_list: | tr " " ":" | grep :$i:)
135 if [[ "$found" == "" ]]; then
136 echo "pool list: $pool_list"
137 echo "striping: $res"
139 error "$file not allocated from OSTs $pool_list."
144 local ost_count=$($GETSTRIPE $file | grep 0x | wc -l)
145 [[ -n "$count" ]] && [[ $ost_count -ne $count ]] && \
146 { error "Stripe count $count expected; got $ost_count" && return 1; }
152 $GETSTRIPE -v $1 | grep "^lmm_pool:" | tr -d "[:blank:]" | cut -f 2 -d ':'
155 check_file_not_in_pool() {
158 local res=$(file_pool $file)
160 if [[ "$res" == "$pool" ]]; then
161 error "File $file is in pool: $res"
168 check_dir_not_in_pool() {
171 local res=$($GETSTRIPE -v $dir | grep "^stripe_count" | head -1 | \
173 if [[ "$res" == "$pool" ]]; then
174 error "File $dir is in pool: $res"
183 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool" ""\
184 ||error "Failed to remove targets from pool: $pool"
190 local tgt="${3}$(lctl get_param -n lov.$FSNAME-*.pools.$pool | \
191 sort -u | tr '\n' ' ')"
193 do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool $osts
195 [[ $RC -ne 0 ]] && return $RC
197 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool | \
198 sort -u | tr '\n' ' ' " "$tgt" || RC=1
199 [[ $RC -ne 0 ]] && error "pool_add failed: $1; $2"
203 create_pool_nofail() {
204 create_pool $FSNAME.$1
207 error "Pool creation of $1 failed"
212 create_pool $FSNAME.$1
215 error "Pool creation of $1 succeeded; should have failed"
220 # Destroy pools from previous test runs
221 for p in $(do_facet $SINGLEMDS lctl pool_list $FSNAME | \
222 grep $FSNAME.pool[0-$OSTCOUNT]); do
225 rm -rf $DIR/d0.${TESTSUITE}
233 trap "cleanup_pools $FSNAME" EXIT
237 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
238 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
242 # Tests for new commands added
245 echo "Creating a pool with a 1 character pool name"
248 echo "Creating a pool with a 10 character pool name"
249 create_pool_nofail p123456789
250 destroy_pool p123456789
252 echo "Creating a pool with a 16 character pool name"
253 create_pool_nofail p123456789123456
254 destroy_pool p123456789123456
256 echo "Creating a pool with a 17 character pool name; should fail"
257 create_pool_fail p1234567891234567
259 echo "Creating a pool with a 1000 character pool name; should fail"
261 for i in `seq 1 999`; do NAME=${NAME}"o"; done
262 create_pool_fail $NAME
264 echo "pool_new should fail if fs-name or poolname are missing."
265 do_facet $SINGLEMDS lctl pool_new .pool1 2>/dev/null
267 error "pool_new did not fail even though fs-name was missing."
268 do_facet $SINGLEMDS lctl pool_new pool1 2>/dev/null
270 error "pool_new did not fail even though fs-name was missing."
271 do_facet $SINGLEMDS lctl pool_new ${FSNAME}. 2>/dev/null
273 error "pool_new did not fail even though pool name was missing."
274 do_facet $SINGLEMDS lctl pool_new . 2>/dev/null
276 error "pool_new did not fail even though pool name and fs-name " \
278 do_facet $SINGLEMDS lctl pool_new ${FSNAME},pool1 2>/dev/null
280 error "pool_new did not fail even though pool name format was wrong"
281 do_facet $SINGLEMDS lctl pool_new ${FSNAME}/pool1 2>/dev/null
283 error "pool_new did not fail even though pool name format was wrong"
285 do_facet $SINGLEMDS lctl pool_new ${FSNAME}.p 2>/dev/null
287 error "pool_new did not fail even though pool1 existed"
291 run_test 1 "Test lctl pool_new ========================================="
297 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null
299 error " pool_add did not fail even though pool did " \
302 run_test 2a "pool_add: non-existant pool"
306 do_facet $SINGLEMDS lctl pool_add $FSNAME.p1234567891234567890 \
307 $FSNAME-OST0000 2>/dev/null
309 error "pool_add did not fail even though pool name was invalid."
311 run_test 2b "pool_add: Invalid pool name"
313 # Testing various combinations of OST name list
319 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
323 create_pool_nofail $POOL
326 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL OST0000
327 RC=$?; [[ $RC -eq 0 ]] || \
328 error "pool_add failed. $FSNAME $POOL OST0000: $RC"
329 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000
333 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000
334 RC=$?; [[ $RC -eq 0 ]] || \
335 error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000: $RC"
336 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000
339 # 3. lustre-OST0000_UUID
340 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID
341 RC=$?; [[ $RC -eq 0 ]] || \
342 error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000_UUID: $RC"
343 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID
346 # 4. lustre-OST[0,1,2,3,]
348 for i in $TGT_LIST; do TGT=${TGT}$(printf "$i," $i); done
350 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
352 error "pool_add failed. $FSNAME.$POOL $TGT. $RC"
353 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT
356 # 5. lustre-OST[0-5/1]
357 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL
358 RC=$?; [[ $RC -eq 0 ]] || \
359 error "pool_add failed. $FSNAME $POOL" "$TGT_ALL $RC"
360 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | \
361 sort -u | tr '\n' ' ' " "$TGT_UUID" || error "Add to pool failed"
362 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT_ALL
367 run_test 2c "pool_add: OST index combinations ==========================="
374 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
378 create_pool_nofail $POOL
380 TGT=$(printf "$FSNAME-OST%04x_UUID " $OSTCOUNT)
381 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
382 RC=$?; [[ $RC -ne 0 ]] || \
383 error "pool_add succeeded for an OST ($TGT) that does not exist."
387 run_test 2d "pool_add: OSTs that don't exist should be rejected ========"
395 $LCTL get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
399 create_pool_nofail $POOL
401 TGT="$FSNAME-OST0000_UUID "
402 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
403 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | \
404 sort -u | tr '\n' ' ' " "$TGT" || error "Add to pool failed"
405 RESULT=$(do_facet $SINGLEMDS \
406 "LOCALE=C $LCTL pool_add $FSNAME.$POOL $TGT 2>&1")
411 error "pool_add succeeded for an OST that was already in the pool."
413 [[ $(grep "already in pool" <<< $RESULT) ]] || \
414 error "pool_add failed as expected but error message not as expected."
418 run_test 2e "pool_add: OST already in a pool should be rejected ========"
422 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
426 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null
428 error "pool_remove did not fail even though pool did not exist."
430 run_test 3a "pool_remove: non-existant pool"
434 do_facet $SINGLEMDS lctl pool_remove ${NON_EXISTANT_FS}.$POOL OST0000 2>/dev/null
436 error "pool_remove did not fail even though fsname did not exist."
438 run_test 3b "pool_remove: non-existant fsname"
442 do_facet $SINGLEMDS lctl pool_remove $FSNAME.p1234567891234567890 \
443 $FSNAME-OST0000 2>/dev/null
445 error "pool_remove did not fail even though pool name was invalid."
447 run_test 3c "pool_remove: Invalid pool name"
449 # Testing various combinations of OST name list
452 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
456 create_pool_nofail $POOL
457 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL OST0000
458 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000
460 error "pool_remove failed. $FSNAME $POOL OST0000"
463 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000
464 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000
466 error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000"
469 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID
470 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID
472 error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000_UUID"
475 add_pool $POOL $TGT_ALL "$TGT_UUID"
476 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT_ALL
478 error "pool_remove failed. $FSNAME $POOL" $TGT_ALL
483 run_test 3d "pool_remove: OST index combinations ==========================="
487 lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
491 do_facet $SINGLEMDS lctl pool_destroy $FSNAME.$POOL 2>/dev/null
493 error "pool_destroy did not fail even though pool did not exist."
495 run_test 4a "pool_destroy: non-existant pool"
499 do_facet $SINGLEMDS lctl pool_destroy ${NON_EXISTANT_FS}.$POOL 2>/dev/null
501 error "pool_destroy did not fail even though the filesystem did not exist."
503 run_test 4b "pool_destroy: non-existant fs-name"
507 create_pool_nofail $POOL
508 add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID "
510 do_facet $SINGLEMDS lctl pool_destroy ${FSNAME}.$POOL
512 error "pool_destroy succeeded with a non-empty pool."
515 run_test 4c "pool_destroy: non-empty pool ==============================="
520 $LCMD pool_list 2>/dev/null
522 error "pool_list did not fail even though fsname was not mentioned."
524 destroy_pool $POOL 2>/dev/null
525 destroy_pool $POOL2 2>/dev/null
527 create_pool_nofail $POOL
528 create_pool_nofail $POOL2
529 $LCMD pool_list $FSNAME
531 error "pool_list $FSNAME failed."
533 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL
535 $LCMD pool_list $FSNAME.$POOL
537 error "pool_list $FSNAME.$POOL failed."
539 $LCMD pool_list ${NON_EXISTANT_FS} 2>/dev/null
541 error "pool_list did not fail for a non-existant fsname $NON_EXISTANT_FS"
543 $LCMD pool_list ${FSNAME}.$NON_EXISTANT_POOL 2>/dev/null
545 error "pool_list did not fail for a non-existant pool $NON_EXISTANT_POOL"
547 if [[ ! $(grep $SINGLEMDS <<< $LCMD) ]]; then
548 echo $LCMD pool_list $DIR
551 error "pool_list failed for $DIR"
554 $LCMD pool_list ${DIR}/d1
556 error "pool_list failed for ${DIR}/d1"
559 rm -rf ${DIR}nonexistant
560 $LCMD pool_list ${DIR}nonexistant 2>/dev/null
562 error "pool_list did not fail for invalid mountpoint ${DIR}nonexistant"
570 # Issue commands from client
574 # Issue commands from MDS
575 sub_test_5 "do_facet $SINGLEMDS lctl"
576 sub_test_5 "do_facet $SINGLEMDS lfs"
579 run_test 5 "lfs/lctl pool_list"
583 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
584 local POOL_DIR=$POOL_ROOT/dir_tst
585 local POOL_FILE=$POOL_ROOT/file_tst
587 create_pool_nofail $POOL
589 do_facet $SINGLEMDS lctl pool_list $FSNAME
591 error "pool_list $FSNAME failed."
593 add_pool $POOL $TGT_ALL "$TGT_UUID"
596 $SETSTRIPE -c -1 -p $POOL $POOL_DIR
598 error "$SETSTRIPE -p $POOL failed."
599 check_dir_in_pool $POOL_DIR $POOL
601 # If an invalid pool name is specified, the command should fail
602 $SETSTRIPE -c 2 -p $INVALID_POOL $POOL_DIR 2>/dev/null
604 error "setstripe to invalid pool did not fail."
606 # If the pool name does not exist, the command should fail
607 $SETSTRIPE -c 2 -p $NON_EXISTANT_POOL $POOL_DIR 2>/dev/null
609 error "setstripe to non-existant pool did not fail."
611 # lfs setstripe should work as before if a pool name is not specified.
612 $SETSTRIPE -c -1 $POOL_DIR
614 error "$SETSTRIPE -p $POOL_DIR failed."
615 $SETSTRIPE -c -1 $POOL_FILE
617 error "$SETSTRIPE -p $POOL_FILE failed."
619 # lfs setstripe should fail if a start index that is outside the
621 create_pool_nofail $POOL2
622 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID "
623 $SETSTRIPE -o 1 -p $POOL2 $ROOT_POOL/$tfile 2>/dev/null
625 error "$SETSTRIPE with start index outside the pool did not fail."
628 run_test 6 "getstripe/setstripe"
632 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
634 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
636 create_pool_nofail $POOL
637 create_pool_nofail $POOL2
639 local start=$((TGT_FIRST+1))
640 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \
641 $FSNAME-OST[$start-$TGT_MAX/2]
643 add_pool $POOL $TGT_HALF "$TGT_UUID2"
645 create_dir $POOL_ROOT/dir1 $POOL
646 create_dir $POOL_ROOT/dir2 $POOL2
647 check_dir_in_pool $POOL_ROOT/dir1 $POOL
648 check_dir_in_pool $POOL_ROOT/dir1 $POOL
651 createmany -o $POOL_ROOT/dir1/$tfile $numfiles || \
652 error "createmany $POOL_ROOT/dir1/$tfile failed!"
654 for file in $POOL_ROOT/dir1/*; do
655 check_file_in_pool $file $POOL
658 createmany -o $POOL_ROOT/dir2/$tfile $numfiles || \
659 error "createmany $POOL_ROOT/dir2/$tfile failed!"
660 for file in $POOL_ROOT/dir2/*; do
661 check_file_in_pool $file $POOL2
664 rm -rf $POOL_ROOT/dir?
668 run_test 11 "OSTs in overlapping/multiple pools"
672 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
674 [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
676 create_pool_nofail $POOL
677 create_pool_nofail $POOL2
679 local start=$((TGT_FIRST+1))
680 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \
681 $FSNAME-OST[$start-$TGT_MAX/2]
683 add_pool $POOL $TGT_HALF "$TGT_UUID2"
685 echo creating some files in $POOL and $POOL2
687 create_dir $POOL_ROOT/dir1 $POOL
688 create_dir $POOL_ROOT/dir2 $POOL2
689 create_file $POOL_ROOT/file1 $POOL
690 create_file $POOL_ROOT/file2 $POOL2
692 echo Checking the files created
693 check_dir_in_pool $POOL_ROOT/dir1 $POOL
694 check_dir_in_pool $POOL_ROOT/dir2 $POOL2
695 check_file_in_pool $POOL_ROOT/file1 $POOL
696 check_file_in_pool $POOL_ROOT/file2 $POOL2
698 echo Changing the pool membership
699 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST[$TGT_FIRST]
700 do_facet $SINGLEMDS lctl pool_list $FSNAME.$POOL
701 FIRST_UUID=$(echo $TGT_UUID | awk '{print $1}')
702 add_pool $POOL2 $FSNAME-OST[$TGT_FIRST] "$FIRST_UUID "
703 do_facet $SINGLEMDS lctl pool_list $FSNAME.$POOL2
705 echo Checking the files again
706 check_dir_in_pool $POOL_ROOT/dir1 $POOL
707 check_dir_in_pool $POOL_ROOT/dir2 $POOL2
708 check_file_in_osts $POOL_ROOT/file1 "$TGT_LIST2"
709 check_file_in_osts $POOL_ROOT/file2 "$(seq $start 2 $TGT_MAX)"
711 echo Creating some more files
712 create_dir $POOL_ROOT/dir3 $POOL
713 create_dir $POOL_ROOT/dir4 $POOL2
714 create_file $POOL_ROOT/file3 $POOL
715 create_file $POOL_ROOT/file4 $POOL2
717 echo Checking the new files
718 check_file_in_pool $POOL_ROOT/file3 $POOL
719 check_file_in_pool $POOL_ROOT/file4 $POOL2
723 run_test 12 "OST Pool Membership"
727 [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
729 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
733 create_pool_nofail $POOL
734 add_pool $POOL $TGT_ALL "$TGT_UUID"
736 create_dir $POOL_ROOT/dir1 $POOL -1
737 createmany -o $POOL_ROOT/dir1/$tfile $numfiles || \
738 error "createmany $POOL_ROOT/dir1/$tfile failed!"
739 for file in $POOL_ROOT/dir1/*; do
740 check_file_in_pool $file $POOL $OSTCOUNT
743 create_file $POOL_ROOT/dir1/file1 $POOL 1 $TGT_FIRST
744 create_file $POOL_ROOT/dir1/file2 $POOL 1 $((TGT_FIRST+1))
745 create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2))
746 check_file_in_pool $POOL_ROOT/dir1/file1 $POOL 1
747 check_file_in_pool $POOL_ROOT/dir1/file2 $POOL 1
748 create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2))
749 check_file_in_osts $POOL_ROOT/dir1/file1 "$TGT_FIRST"
750 check_file_in_osts $POOL_ROOT/dir1/file2 "$((TGT_FIRST+1))"
751 check_file_in_osts $POOL_ROOT/dir1/file3 "$((TGT_FIRST+2))"
753 create_dir $POOL_ROOT/dir2 $POOL $count
754 createmany -o $POOL_ROOT/dir2/$tfile $numfiles || \
755 error "createmany $POOL_ROOT/dir2/$tfile failed!"
756 for file in $POOL_ROOT/dir2/*; do
757 check_file_in_pool $file $POOL $count
760 create_dir $POOL_ROOT/dir3 $POOL $count $((TGT_FIRST+1))
761 createmany -o $POOL_ROOT/dir3/$tfile_ $numfiles || \
762 error "createmany $POOL_ROOT/dir3/$tfile_ failed!"
763 for file in $POOL_ROOT/dir3/*; do
764 check_file_in_pool $file $POOL $count
767 create_dir $POOL_ROOT/dir4 $POOL 1
768 createmany -o $POOL_ROOT/dir4/$tfile_ $numfiles || \
769 error "createmany $POOL_ROOT/dir4/$tfile_ failed!"
770 for file in $POOL_ROOT/dir4/*; do
771 check_file_in_pool $file $POOL 1
774 create_dir $POOL_ROOT/dir5 $POOL 1 $((TGT_FIRST+2))
775 createmany -o $POOL_ROOT/dir5/$tfile_ $numfiles || \
776 error "createmany $POOL_ROOT/dir5/$tfile_ failed!"
777 for file in $POOL_ROOT/dir5/*; do
778 check_file_in_pool $file $POOL 1
779 check_file_in_osts $file "$((TGT_FIRST+2))"
782 rm -rf create_dir $POOL_ROOT/dir?
786 run_test 13 "Striping characteristics in a pool"
790 [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
792 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
796 # Create a new filesystem that is guaranteed to be balanced.
800 create_pool_nofail $POOL
801 create_pool_nofail $POOL2
803 add_pool $POOL $TGT_HALF "$TGT_UUID2"
804 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID "
806 create_dir $POOL_ROOT/dir1 $POOL 1
807 create_file $POOL_ROOT/dir1/file $POOL 1
808 local OST=$($GETSTRIPE $POOL_ROOT/dir1/file | grep 0x | cut -f2)
810 while [[ $i -lt $numfiles ]];
813 [[ $OST -gt $TGT_MAX ]] && OST=$TGT_FIRST
815 # echo "Iteration: $i OST: $OST"
816 create_file $POOL_ROOT/dir1/file${i} $POOL 1
817 check_file_in_pool $POOL_ROOT/dir1/file${i} $POOL
821 # Fill up OST0 until it is nearly full.
822 # Create 9 files of size OST0_SIZE/10 each.
823 create_dir $POOL_ROOT/dir2 $POOL2 1
824 $LFS df $POOL_ROOT/dir2
825 echo "Filling up OST0"
826 OST0_SIZE=`$LFS df $POOL_ROOT/dir2 | awk '/\[OST:0\]/ {print $4}'`
827 FILE_SIZE=$((OST0_SIZE/1024/10))
829 while [[ $i -lt 10 ]];
831 dd if=/dev/zero of=$POOL_ROOT/dir2/f${i} bs=1M count=$FILE_SIZE
834 $LFS df $POOL_ROOT/dir2
836 # OST $TGT_FIRST is no longer favored; but it may still be used.
837 create_dir $POOL_ROOT/dir3 $POOL 1
838 create_file $POOL_ROOT/dir3/file $POOL 1
839 createmany -o $POOL_ROOT/dir3/$tfile_ $numfiles || \
840 error "createmany $POOL_ROOT/dir3/$tfile_ failed!"
841 for file in $POOL_ROOT/dir3/*; do
842 check_file_in_pool $file $POOL
848 run_test 14 "Round robin and QOS striping within a pool"
852 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
856 while [[ $i -lt $OSTCOUNT ]]
858 create_pool_nofail pool${i}
860 local tgt=$(printf "$FSNAME-OST%04x_UUID " $i)
861 add_pool pool${i} "$FSNAME-OST[$i]" "$tgt"
862 create_dir $POOL_ROOT/dir${i} pool${i}
863 createmany -o $POOL_ROOT/dir$i/$tfile $numfiles || \
864 error "createmany $POOL_ROOT/dir$i/$tfile failed!"
866 for file in $POOL_ROOT/dir$i/*; do
867 check_file_in_osts $file $i
875 run_test 15 "One directory per OST/pool"
879 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
883 create_pool_nofail $POOL
885 add_pool $POOL $TGT_HALF "$TGT_UUID2"
887 local dir=$POOL_ROOT/$tdir
888 create_dir $dir $POOL
890 for i in $(seq 1 10); do
895 createmany -o $dir/$tfile $numfiles || \
896 error "createmany $dir/$tfile failed!"
898 for file in $dir/*; do
899 check_file_in_pool $file $POOL
902 rm -rf $POOL_ROOT/$tdir
906 run_test 16 "Inheritance of pool properties"
910 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
914 create_pool_nofail $POOL
916 add_pool $POOL $TGT_ALL "$TGT_UUID"
918 local dir=$POOL_ROOT/dir
919 create_dir $dir $POOL
921 createmany -o $dir/${tfile}1_ $numfiles || \
922 error "createmany $dir/${tfile}1_ failed!"
924 for file in $dir/*; do
925 check_file_in_pool $file $POOL
930 createmany -o $dir/${tfile}2_ $numfiles || \
931 error "createmany $dir/${tfile}2_ failed!"
936 run_test 17 "Referencing an empty pool"
944 sync; sleep 5 # give pending IO a chance to go to disk
945 stat=$(createmany -o $cdir/${tfile} $numfiles)
948 time=$(echo $stat | cut -f 5 -d ' ')
949 echo $stat >> /dev/stderr
955 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
957 local plaindir=$POOL_ROOT/plaindir
958 local pooldir=$POOL_ROOT/pooldir
966 echo "Create performance, iteration $i, $numfiles files x 3"
968 time1=$(create_perf $plaindir $numfiles)
969 echo "iter $i: $numfiles creates without pool: $time1"
970 t1=$(echo "scale=2; $t1 + $time1" | bc)
972 create_pool_nofail $POOL > /dev/null
973 add_pool $POOL $TGT_ALL "$TGT_UUID" > /dev/null
974 create_dir $pooldir $POOL
975 time2=$(create_perf $pooldir $numfiles)
976 echo "iter $i: $numfiles creates with pool: $time2"
977 t2=$(echo "scale=2; $t2 + $time2" | bc)
979 destroy_pool $POOL > /dev/null
980 time3=$(create_perf $pooldir $numfiles)
981 echo "iter $i: $numfiles creates with missing pool: $time3"
982 t3=$(echo "scale=2; $t3 + $time3" | bc)
987 time1=$(echo "scale=2; $t1 / $i" | bc)
988 echo Avg time taken for $numfiles creates without pool: $time1
989 time2=$(echo "scale=2; $t2 / $i" | bc)
990 echo Avg time taken for $numfiles creates with pool: $time2
991 time3=$(echo "scale=2; $t3 / $i" | bc)
992 echo Avg time taken for $numfiles creates with missing pool: $time3
994 # Set this high until we establish a baseline for what the degradation
997 diff=$(echo "scale=2; ($time2 - $time1) * 100 / $time1" | bc)
998 echo "No pool to wide pool: $diff %."
999 deg=$(echo "scale=2; $diff > $max" | bc)
1000 [ "$deg" == "1" ] && error_ignore 23408 "Degradation with wide pool is $diff % (> $max %)"
1003 diff=$(echo "scale=2; ($time3 - $time1) * 100 / $time1" | bc)
1004 echo "No pool to missing pool: $diff %."
1005 deg=$(echo "scale=2; $diff > $max" | bc)
1006 [ "$deg" == "1" ] && error_ignore 23408 "Degradation with missing pool is $diff % (> $max %)"
1010 run_test 18 "File create in a directory which references a deleted pool"
1015 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1017 local dir1=$POOL_ROOT/dir1
1018 local dir2=$POOL_ROOT/dir2
1021 create_pool_nofail $POOL
1023 add_pool $POOL $TGT_HALF "$TGT_UUID2"
1025 create_dir $dir1 $POOL
1026 createmany -o $dir1/${tfile} $numfiles || \
1027 error "createmany $dir1/${tfile} failed!"
1028 for file in $dir1/*; do
1029 check_file_in_pool $file $POOL
1033 createmany -o $dir2/${tfile} $numfiles || \
1034 error "createmany $dir2/${tfile} failed!"
1035 for file in $dir2/*; do
1036 check_file_not_in_pool $file $POOL
1043 run_test 19 "Pools should not come into play when not specified"
1047 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1049 local dir1=$POOL_ROOT/dir1
1050 local dir2=$dir1/dir2
1051 local dir3=$dir1/dir3
1055 create_pool_nofail $POOL
1056 create_pool_nofail $POOL2
1058 add_pool $POOL $TGT_HALF "$TGT_UUID2"
1060 local start=$((TGT_FIRST+1))
1061 TGT=$(for i in `seq $start 2 $TGT_MAX`; \
1062 do printf "$FSNAME-OST%04x_UUID " $i; done)
1063 add_pool $POOL2 "$FSNAME-OST[$start-$TGT_MAX/2]" "$TGT"
1065 create_dir $dir1 $POOL
1066 create_file $dir1/file1 $POOL2
1067 create_dir $dir2 $POOL2
1070 $SETSTRIPE -c 1 $dir3 # No pool assignment
1072 $SETSTRIPE -c 1 $dir2/file4 # No pool assignment
1074 check_file_in_pool $dir1/file1 $POOL2
1075 check_file_in_pool $dir2/file2 $POOL2
1077 check_dir_not_in_pool $dir3 $POOL
1078 check_dir_not_in_pool $dir3 $POOL2
1080 check_file_not_in_pool $dir3/file3 $POOL
1081 check_file_not_in_pool $dir3/file3 $POOL2
1083 check_file_not_in_pool $dir2/file4 $POOL
1084 check_file_not_in_pool $dir2/file4 $POOL2
1090 run_test 20 "Different pools in a directory hierarchy."
1094 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1095 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
1099 local dir=$POOL_ROOT/dir
1101 create_pool_nofail $POOL
1103 add_pool $POOL $TGT_HALF "$TGT_UUID2"
1105 create_dir $dir $POOL $OSTCOUNT
1106 create_file $dir/file1 $POOL $OSTCOUNT
1107 $GETSTRIPE -v $dir/file1
1108 check_file_in_pool $dir/file1 $POOL
1114 run_test 21 "OST pool with fewer OSTs than stripe count"
1122 for c in $(seq 1 10);
1124 echo "Pool $pool, iteration $c"
1125 do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool OST[$TGT_FIRST-$TGT_MAX/$step] 2>/dev/null
1126 local TGT_SECOND=$(($TGT_FIRST+$step))
1127 if [ "$TGT_SECOND" -le "$TGT_MAX" ]; then
1128 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$pool OST[$TGT_SECOND-$TGT_MAX/$step]
1131 echo loop for $pool complete
1136 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1137 [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return
1141 create_pool_nofail $POOL
1142 add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID "
1143 create_pool_nofail $POOL2
1144 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID "
1149 create_dir $POOL_ROOT $POOL
1150 createmany -o $POOL_ROOT/${tfile} $numfiles || \
1151 error "createmany $POOL_ROOT/${tfile} failed!"
1156 run_test 22 "Simultaneous manipulation of a pool"
1160 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1161 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
1164 check_runas_id $TSTID $TSTID $RUNAS || {
1165 skip_env "User $RUNAS_ID does not exist - skipping"
1172 local BUNIT_SZ=1024 # min block quota unit(kB)
1173 local LOVNAME=`lctl get_param -n llite.*.lov.common_name | tail -n 1`
1174 local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.numobd`
1176 local dir=$POOL_ROOT/dir
1177 local file="$dir/$tfile-quota"
1179 create_pool_nofail $POOL
1181 local TGT=$(for i in `seq $TGT_FIRST 3 $TGT_MAX`; \
1182 do printf "$FSNAME-OST%04x_UUID " $i; done)
1183 add_pool $POOL "$FSNAME-OST[$TGT_FIRST-$TGT_MAX/3]" "$TGT"
1184 create_dir $dir $POOL
1186 $LFS quotaoff -ug $MOUNT
1187 $LFS quotacheck -ug $MOUNT
1188 LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1)))
1189 $LFS setquota -u $TSTUSR -b $LIMIT -B $LIMIT $dir
1191 $LFS quota -v -u $TSTUSR $dir
1193 $LFS setstripe $file -c 1 -p $POOL
1194 chown $TSTUSR.$TSTUSR $file
1198 LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BLK_SZ count=$((BUNIT_SZ*2)) || true
1199 $LFS quota -v -u $TSTUSR $dir
1200 cancel_lru_locks osc
1201 stat=$(LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BLK_SZ count=$BUNIT_SZ seek=$((BUNIT_SZ*2)) 2>&1)
1204 [[ $RC -eq 0 ]] && error "dd did not fail with EDQUOT."
1205 echo $stat | grep "Disk quota exceeded" > /dev/null
1206 [[ $? -eq 1 ]] && error "dd did not fail with EDQUOT."
1207 $LFS quota -v -u $TSTUSR $dir
1210 $LFS quotaoff -ug $MOUNT
1211 # $LFS setquota -u $TSTUSR -b $LIMIT -B $LIMIT $dir
1212 chown $TSTUSR.$TSTUSR $dir
1215 while [ $RC -eq 0 ];
1218 stat=$(LOCALE=C $RUNAS2 dd if=/dev/zero of=${file}$i bs=1M \
1219 count=$((LIMIT*LIMIT)) 2>&1)
1221 if [ $RC -eq 1 ]; then
1223 echo $stat | grep "Disk quota exceeded"
1224 [[ $? -eq 0 ]] && error "dd failed with EDQUOT"
1226 echo $stat | grep "No space left on device"
1227 [[ $? -ne 0 ]] && error "dd did not fail with ENOSPC; " \
1238 run_test 23 "OST pools and quota"
1242 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1243 [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
1252 create_pool_nofail $POOL
1254 add_pool $POOL $TGT_ALL "$TGT_UUID"
1256 create_dir $POOL_ROOT/dir1 $POOL $OSTCOUNT
1258 mkdir $POOL_ROOT/dir2
1259 $SETSTRIPE $POOL_ROOT/dir2 -p $POOL -s 65536 -i 0 -c 1 || \
1260 error "$SETSTRIPE $POOL_ROOT/dir2 failed"
1262 mkdir $POOL_ROOT/dir3
1263 $SETSTRIPE $POOL_ROOT/dir3 -s 65536 -i 0 -c 1 || \
1264 error "$SETSTRIPE $POOL_ROOT/dir3 failed"
1266 mkdir $POOL_ROOT/dir4
1268 for i in $(seq 1 4);
1270 dir=${POOL_ROOT}/dir${i}
1279 createmany -o $dir/${tfile} $numfiles || \
1280 error "createmany $dir/${tfile} failed!"
1281 res=$($GETSTRIPE -v $dir | grep "^stripe_count:")
1282 if [ $? -ne 0 ]; then
1283 res=$($GETSTRIPE -v $dir | grep "^(Default) ")
1284 pool=$(cut -f 9 -d ' ' <<< $res)
1285 index=$(cut -f 7 -d ' ' <<< $res)
1286 size=$(cut -f 5 -d ' ' <<< $res)
1287 count=$(cut -f 3 -d ' ' <<< $res)
1289 pool=$(cut -f 8 -d ' ' <<< $res)
1290 index=$(cut -f 6 -d ' ' <<< $res)
1291 size=$(cut -f 4 -d ' ' <<< $res)
1292 count=$(cut -f 2 -d ' ' <<< $res)
1295 for file in $dir/*; do
1296 if [ "$pool" != "" ]; then
1297 check_file_in_pool $file $pool
1299 pool1=$(file_pool $file)
1300 count1=$($GETSTRIPE -v $file | grep "^lmm_stripe_count:" |\
1301 tr -d '[:blank:]' | cut -f 2 -d ':')
1302 size1=$($GETSTRIPE -v $file | grep "^lmm_stripe_size:" |\
1303 tr -d '[:blank:]' | cut -f 2 -d ':')
1304 [[ "$pool" != "$pool1" ]] && \
1305 error "Pool name ($pool) not inherited in $file($pool1)"
1306 [[ "$count" != "$count1" ]] && \
1307 error "Stripe count ($count) not inherited in $file ($count1)"
1308 [[ "$size" != "$size1" ]] && [[ "$size" != "0" ]] && \
1309 error "Stripe size ($size) not inherited in $file ($size1)"
1317 run_test 24 "Independence of pool from other setstripe parameters"
1321 local dev=$(mdsdevname ${SINGLEMDS//mds/})
1322 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1326 for i in $(seq 10); do
1327 create_pool_nofail pool$i
1328 do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool$i OST0000; sync"
1329 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool$i | \
1330 sort -u | tr '\n' ' ' " "$FSNAME-OST0000_UUID " || \
1331 error "pool_add failed: $1; $2"
1333 stop $SINGLEMDS || return 1
1334 start $SINGLEMDS ${dev} $MDS_MOUNT_OPTS || \
1335 { error "Failed to start $SINGLEMDS after stopping" && break; }
1336 wait_osc_import_state mds ost FULL
1340 # Veriy that the pool got created and is usable
1341 df $POOL_ROOT > /dev/null
1343 # Make sure OST0 can be striped on
1344 $SETSTRIPE -o 0 -c 1 $POOL_ROOT/$tfile
1345 STR=$($GETSTRIPE $POOL_ROOT/$tfile | grep 0x | cut -f2 | tr -d " ")
1346 rm $POOL_ROOT/$tfile
1347 if [[ "$STR" == "0" ]]; then
1348 echo "Creating a file in pool$i"
1349 create_file $POOL_ROOT/file$i pool$i || break
1350 check_file_in_pool $POOL_ROOT/file$i pool$i || break
1352 echo "OST 0 seems to be unavailable. Try later."
1358 run_test 25 "Create new pool and restart MDS ======================="
1361 local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.*clilov*.numobd`
1362 [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return
1364 local dev=$(mdsdevname ${SINGLEMDS//mds/})
1365 local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
1369 create_pool_nofail $POOL
1371 do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0000; sync"
1372 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
1373 sort -u | grep $FSNAME-OST0000_UUID " "$FSNAME-OST0000_UUID" || \
1374 error "pool_add failed: $1; $2"
1376 do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0002; sync"
1377 wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
1378 sort -u | grep $FSNAME-OST0002_UUID" "$FSNAME-OST0002_UUID" || \
1379 error "pool_add failed: $1; $2"
1381 # Veriy that the pool got created and is usable
1383 echo "Creating files in $POOL"
1385 for ((i=0;i<10;i++)); do
1386 #OBD_FAIL_MDS_OSC_CREATE_FAIL 0x147
1387 #Fail OST0000 to make sure the objects create on the other ost in the pool
1388 do_facet $SINGLEMDS lctl set_param fail_loc=0x147
1389 do_facet $SINGLEMDS lctl set_param fail_val=0
1390 create_file $POOL_ROOT/file$i $POOL 1 -1 || break
1391 do_facet $SINGLEMDS lctl set_param fail_loc=0
1392 check_file_in_pool $POOL_ROOT/file$i $POOL || break
1396 run_test 26 "Choose other OSTs in the pool first in the creation remedy"
1398 log "cleanup: ======================================================"
1400 cleanup_pools $FSNAME
1401 check_and_cleanup_lustre
1402 echo '=========================== finished ==============================='
1403 [ -f "$POOLSLOG" ] && cat $POOLSLOG && grep -q FAIL $POOLSLOG && exit 1 || true
1404 echo "$0: completed"