+### Data-on-MDT sanity tests ###
+test_270a() {
+
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ # create DoM file
+ local dom=$DIR/$tdir/dom_file
+ local tmp=$DIR/$tdir/tmp_file
+
+ mkdir -p $DIR/$tdir
+
+ # basic checks for DoM component creation
+ $LFS setstripe -E 1024K -E 1024K -L mdt $dom 2>/dev/null &&
+ error "Can set MDT layout to non-first entry"
+
+ $LFS setstripe -E 1024K -L mdt -E 1024K -L mdt $dom 2>/dev/null &&
+ error "Can define multiple entries as MDT layout"
+
+ $LFS setstripe -E 1M -L mdt $dom ||
+ error "Can't create DoM layout"
+
+ [ $($LFS getstripe -L $dom) == "mdt" ] || error "bad pattern"
+ [ $($LFS getstripe -c $dom) == 0 ] || error "bad stripe count"
+ [ $($LFS getstripe -S $dom) == 1048576 ] || error "bad stripe size"
+
+ local mdtidx=$($GETSTRIPE -M $dom)
+ local mdtname=MDT$(printf %04x $mdtidx)
+ local facet=mds$((mdtidx + 1))
+ local space_check=1
+
+ # Skip free space checks with ZFS
+ if [ "$(facet_fstype $facet)" == "zfs" ]; then
+ space_check=0
+ fi
+
+ # write
+ sync
+ local mdtfree1=$(do_facet $facet \
+ lctl get_param -n osd*.*$mdtname.kbytesfree)
+ dd if=/dev/urandom of=$tmp bs=1024 count=100
+ # check also direct IO along write
+ dd if=$tmp of=$dom bs=102400 count=1 oflag=direct
+ sync
+ cmp $tmp $dom || error "file data is different"
+ [ $(stat -c%s $dom) == 102400 ] || error "bad size after write"
+ if [ $space_check == 1 ]; then
+ local mdtfree2=$(do_facet $facet \
+ lctl get_param -n osd*.*$mdtname.kbytesfree)
+ [ $(($mdtfree1 - $mdtfree2)) -ge 102 ] ||
+ error "MDT free space is wrong after write"
+ fi
+
+ # truncate
+ $TRUNCATE $dom 10000
+ [ $(stat -c%s $dom) == 10000 ] || error "bad size after truncate"
+ if [ $space_check == 1 ]; then
+ mdtfree1=$(do_facet $facet \
+ lctl get_param -n osd*.*$mdtname.kbytesfree)
+ [ $(($mdtfree1 - $mdtfree2)) -ge 92 ] ||
+ error "MDT free space is wrong after truncate"
+ fi
+
+ # append
+ cat $tmp >> $dom
+ sync
+ [ $(stat -c%s $dom) == 112400 ] || error "bad size after append"
+ if [ $space_check == 1 ]; then
+ mdtfree2=$(do_facet $facet \
+ lctl get_param -n osd*.*$mdtname.kbytesfree)
+ [ $(($mdtfree1 - $mdtfree2)) -ge 102 ] ||
+ error "MDT free space is wrong after append"
+ fi
+
+ # delete
+ rm $dom
+ if [ $space_check == 1 ]; then
+ mdtfree1=$(do_facet $facet \
+ lctl get_param -n osd*.*$mdtname.kbytesfree)
+ [ $(($mdtfree1 - $mdtfree2)) -ge 112 ] ||
+ error "MDT free space is wrong after removal"
+ fi
+
+ # combined striping
+ $LFS setstripe -E 1024K -L mdt -E EOF $dom ||
+ error "Can't create DoM + OST striping"
+
+ dd if=/dev/urandom of=$tmp bs=1024 count=2000
+ # check also direct IO along write
+ dd if=$tmp of=$dom bs=102400 count=20 oflag=direct
+ sync
+ cmp $tmp $dom || error "file data is different"
+ [ $(stat -c%s $dom) == 2048000 ] || error "bad size after write"
+ rm $dom
+ rm $tmp
+
+ return 0
+}
+run_test 270a "DoM: basic functionality tests"
+
+test_270b() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ local dom=$DIR/$tdir/dom_file
+ local max_size=1048576
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -E $max_size -L mdt $dom
+
+ # truncate over the limit
+ $TRUNCATE $dom $(($max_size + 1)) &&
+ error "successful truncate over the maximum size"
+ # write over the limit
+ dd if=/dev/zero of=$dom bs=$max_size seek=1 count=1 &&
+ error "successful write over the maximum size"
+ # append over the limit
+ dd if=/dev/zero of=$dom bs=$(($max_size - 3)) count=1
+ echo "12345" >> $dom && error "successful append over the maximum size"
+ rm $dom
+
+ return 0
+}
+run_test 270b "DoM: maximum size overflow checks for DoM-only file"
+
+test_270c() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -E 1024K -L mdt $DIR/$tdir
+
+ # check files inherit DoM EA
+ touch $DIR/$tdir/first
+ [ $($GETSTRIPE -L $DIR/$tdir/first) == "mdt" ] ||
+ error "bad pattern"
+ [ $($LFS getstripe -c $DIR/$tdir/first) == 0 ] ||
+ error "bad stripe count"
+ [ $($LFS getstripe -S $DIR/$tdir/first) == 1048576 ] ||
+ error "bad stripe size"
+
+ # check directory inherits DoM EA and uses it as default
+ mkdir $DIR/$tdir/subdir
+ touch $DIR/$tdir/subdir/second
+ [ $($LFS getstripe -L $DIR/$tdir/subdir/second) == "mdt" ] ||
+ error "bad pattern in sub-directory"
+ [ $($LFS getstripe -c $DIR/$tdir/subdir/second) == 0 ] ||
+ error "bad stripe count in sub-directory"
+ [ $($LFS getstripe -S $DIR/$tdir/subdir/second) == 1048576 ] ||
+ error "bad stripe size in sub-directory"
+ return 0
+}
+run_test 270c "DoM: DoM EA inheritance tests"
+
+test_270d() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -E 1024K -L mdt $DIR/$tdir
+
+ # inherit default DoM striping
+ mkdir $DIR/$tdir/subdir
+ touch $DIR/$tdir/subdir/f1
+
+ # change default directory striping
+ $LFS setstripe -c 1 $DIR/$tdir/subdir
+ touch $DIR/$tdir/subdir/f2
+ [ $($LFS getstripe -c $DIR/$tdir/subdir/f2) == 1 ] ||
+ error "wrong default striping in file 2"
+ [ $($LFS getstripe -L $DIR/$tdir/subdir/f2) == "raid0" ] ||
+ error "bad pattern in file 2"
+ return 0
+}
+run_test 270d "DoM: change striping from DoM to RAID0"
+
+test_270e() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ mkdir -p $DIR/$tdir/dom
+ mkdir -p $DIR/$tdir/norm
+ DOMFILES=20
+ NORMFILES=10
+ $LFS setstripe -E 1M -L mdt $DIR/$tdir/dom
+ $LFS setstripe -i 0 -S 2M $DIR/$tdir/norm
+
+ createmany -o $DIR/$tdir/dom/dom- $DOMFILES
+ createmany -o $DIR/$tdir/norm/norm- $NORMFILES
+
+ # find DoM files by layout
+ NUM=$($LFS find -L mdt -type f $DIR/$tdir 2>/dev/null | wc -l)
+ [ $NUM -eq $DOMFILES ] ||
+ error "lfs find -L: found $NUM, expected $DOMFILES"
+ echo "Test 1: lfs find 20 DOM files by layout: OK"
+
+ # there should be 1 dir with default DOM striping
+ NUM=$($LFS find -L mdt -type d $DIR/$tdir 2>/dev/null | wc -l)
+ [ $NUM -eq 1 ] ||
+ error "lfs find -L: found $NUM, expected 1 dir"
+ echo "Test 2: lfs find 1 DOM dir by layout: OK"
+
+ # find DoM files by stripe size
+ NUM=$($LFS find -S -1200K -type f $DIR/$tdir 2>/dev/null | wc -l)
+ [ $NUM -eq $DOMFILES ] ||
+ error "lfs find -S: found $NUM, expected $DOMFILES"
+ echo "Test 4: lfs find 20 DOM files by stripe size: OK"
+
+ # find files by stripe offset except DoM files
+ NUM=$($LFS find -i 0 -type f $DIR/$tdir 2>/dev/null | wc -l)
+ [ $NUM -eq $NORMFILES ] ||
+ error "lfs find -i: found $NUM, expected $NORMFILES"
+ echo "Test 5: lfs find no DOM files by stripe index: OK"
+ return 0
+}
+run_test 270e "DoM: lfs find with DoM files test"
+
+test_270f() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ local mdtname=${FSNAME}-MDT0000-mdtlov
+ local dom=$DIR/$tdir/dom_file
+ local dom_limit_saved=$(do_facet mds1 $LCTL get_param -n \
+ lod.$mdtname.dom_stripesize)
+ local dom_limit=131072
+
+ do_facet mds1 $LCTL set_param -n lod.$mdtname.dom_stripesize=$dom_limit
+ local dom_current=$(do_facet mds1 $LCTL get_param -n \
+ lod.$mdtname.dom_stripesize)
+ [ ${dom_limit} -eq ${dom_current} ] ||
+ error "Cannot change per-MDT DoM stripe limit to $dom_limit"
+
+ $LFS mkdir -i 0 -c 1 $DIR/$tdir
+ $LFS setstripe -d $DIR/$tdir
+ $LFS setstripe -E $dom_limit -L mdt $DIR/$tdir ||
+ error "Can't set directory default striping"
+
+ # exceed maximum stripe size
+ $LFS setstripe -E $(($dom_limit * 2)) -L mdt $dom &&
+ error "Able to create DoM component size more than LOD limit"
+
+ do_facet mds1 $LCTL set_param -n lod.$mdtname.dom_stripesize=0
+ dom_current=$(do_facet mds1 $LCTL get_param -n \
+ lod.$mdtname.dom_stripesize)
+ [ 0 -eq ${dom_current} ] ||
+ error "Can't set zero DoM stripe limit"
+
+ # too low values to be aligned with smallest stripe size 64K
+ do_facet mds1 $LCTL set_param -n lod.$mdtname.dom_stripesize=30000
+ dom_current=$(do_facet mds1 $LCTL get_param -n \
+ lod.$mdtname.dom_stripesize)
+ [ 30000 -eq ${dom_current} ] &&
+ error "Can set too small DoM stripe limit"
+
+ do_facet mds1 $LCTL set_param -n lod.$mdtname.dom_stripesize=2147483648
+ dom_current=$(do_facet mds1 $LCTL get_param -n \
+ lod.$mdtname.dom_stripesize)
+ echo $dom_current
+ [ 2147483648 -eq ${dom_current} ] &&
+ error "Can set too large DoM stripe limit"
+
+ do_facet mds1 $LCTL set_param -n \
+ lod.$mdtname.dom_stripesize=$((dom_limit * 2))
+ $LFS setstripe -E $((dom_limit * 2)) -L mdt $dom ||
+ error "Can't create DoM component size after limit change"
+ do_facet mds1 $LCTL set_param -n \
+ lod.$mdtname.dom_stripesize=$((dom_limit / 2))
+ $LFS setstripe -E $dom_limit -L mdt ${dom}_big &&
+ error "Can create big DoM component after limit decrease"
+ touch ${dom}_def ||
+ error "Can't create file with old default layout"
+
+ do_facet mds1 $LCTL set_param -n lod.*.dom_stripesize=$dom_limit_saved
+ return 0
+}
+run_test 270f "DoM: maximum DoM stripe size checks"
+
+test_271a() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ local dom=$DIR/$tdir/dom
+
+ mkdir -p $DIR/$tdir
+
+ $LFS setstripe -E 1024K -L mdt $dom
+
+ lctl set_param -n mdc.*.stats=clear
+ dd if=/dev/zero of=$dom bs=4096 count=1 || return 1
+ cat $dom > /dev/null
+ local reads=$(lctl get_param -n mdc.*.stats |
+ awk '/ost_read/ {print $2}')
+ [ -z $reads ] || error "Unexpected $reads READ RPCs"
+ ls $dom
+ rm -f $dom
+}
+run_test 271a "DoM: data is cached for read after write"
+
+test_271b() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ local dom=$DIR/$tdir/dom
+
+ mkdir -p $DIR/$tdir
+
+ $LFS setstripe -E 1024K -L mdt -E EOF $dom
+
+ lctl set_param -n mdc.*.stats=clear
+ dd if=/dev/zero of=$dom bs=4096 count=1 || return 1
+ cancel_lru_locks mdc
+ $CHECKSTAT -t file -s 4096 $dom || error "stat #1 fails"
+ # second stat to check size is cached on client
+ $CHECKSTAT -t file -s 4096 $dom || error "stat #2 fails"
+ local gls=$(lctl get_param -n mdc.*.stats |
+ awk '/ldlm_glimpse/ {print $2}')
+ [ -z $gls ] || error "Unexpected $gls glimpse RPCs"
+ rm -f $dom
+}
+run_test 271b "DoM: no glimpse RPC for stat (DoM only file)"
+
+test_271ba() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ local dom=$DIR/$tdir/dom
+
+ mkdir -p $DIR/$tdir
+
+ $LFS setstripe -E 1024K -L mdt -E EOF $dom
+
+ lctl set_param -n mdc.*.stats=clear
+ lctl set_param -n osc.*.stats=clear
+ dd if=/dev/zero of=$dom bs=2048K count=1 || return 1
+ cancel_lru_locks mdc
+ $CHECKSTAT -t file -s 2097152 $dom || error "stat"
+ # second stat to check size is cached on client
+ $CHECKSTAT -t file -s 2097152 $dom || error "stat"
+ local gls=$(lctl get_param -n mdc.*.stats |
+ awk '/ldlm_glimpse/ {print $2}')
+ [ -z $gls ] || error "Unexpected $gls glimpse RPCs"
+ local gls=$(lctl get_param -n osc.*.stats |
+ awk '/ldlm_glimpse/ {print $2}')
+ [ -z $gls ] || error "Unexpected $gls OSC glimpse RPCs"
+ rm -f $dom
+}
+run_test 271ba "DoM: no glimpse RPC for stat (combined file)"
+
+test_271c() {
+ # test to be enabled with lock_convert
+ skip "skipped until lock convert will be implemented" && return
+
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] &&
+ skip "Need MDS version at least 2.10.55" && return
+
+ local dom=$DIR/$tdir/dom
+
+ mkdir -p $DIR/$tdir
+
+ $LFS setstripe -E 1024K -L mdt $DIR/$tdir
+
+ local mdtidx=$($LFS getstripe -M $DIR/$tdir)
+ local facet=mds$((mdtidx + 1))
+
+ cancel_lru_locks mdc
+ do_facet $facet lctl set_param -n mdt.*.dom_lock=0
+ createmany -o $dom 1000
+ lctl set_param -n mdc.*.stats=clear
+ smalliomany -w $dom 1000 200
+ lctl get_param -n mdc.*.stats
+ local enq=$(lctl get_param -n mdc.*.stats |
+ awk '/ldlm_ibits_enqueue/ {print $2}')
+ # Each file has 1 open, 1 IO enqueues, total 2000
+ # but now we have also +1 getxattr for security.capability, total 3000
+ [ $enq -ge 2000 ] || error "Too few enqueues $enq, expected > 2000"
+ unlinkmany $dom 1000
+
+ cancel_lru_locks mdc
+ do_facet $facet lctl set_param -n mdt.*.dom_lock=1
+ createmany -o $dom 1000
+ lctl set_param -n mdc.*.stats=clear
+ smalliomany -w $dom 1000 200
+ lctl get_param -n mdc.*.stats
+ local enq_2=$(lctl get_param -n mdc.*.stats |
+ awk '/ldlm_ibits_enqueue/ {print $2}')
+ # Expect to see reduced amount of RPCs by 1000 due to single enqueue
+ # for OPEN and IO lock.
+ [ $((enq - enq_2)) -ge 1000 ] ||
+ error "Too many enqueues $enq_2, expected about $((enq - 1000))"
+ unlinkmany $dom 1000
+ return 0
+}
+run_test 271c "DoM: IO lock at open saves enqueue RPCs"
+
+test_276() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return
+ local pid
+
+ do_facet ost1 "(while true; do \
+ $LCTL get_param obdfilter.*.filesfree > /dev/null 2>&1; \
+ done) & pid=\\\$!; echo \\\$pid > $TMP/sanity_276_pid" &
+ pid=$!
+
+ for LOOP in $(seq 20); do
+ stop ost1
+ start ost1 $(ostdevname 1) $OST_MOUNT_OPTS
+ done
+ kill -9 $pid
+ do_facet ost1 "pid=\\\$(cat $TMP/sanity_276_pid); kill -9 \\\$pid; \
+ rm $TMP/sanity_276_pid"
+}
+run_test 276 "Race between mount and obd_statfs"
+