+# Reset async IO behavior after error case
+reset_async() {
+ FILE=$DIR/reset_async
+
+ # Ensure all OSCs are cleared
+ $LSTRIPE $FILE 0 -1 -1
+ dd if=/dev/zero of=$FILE bs=64k count=$OSTCOUNT
+ sync
+ rm $FILE
+}
+
+test_118a() #bug 11710
+{
+ reset_async
+
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ return 1;
+ fi
+}
+run_test 118a "verify O_SYNC works =========="
+
+test_118b()
+{
+ remote_ost_nodsh && skip "remote OST" && return
+
+ reset_async
+
+ #define OBD_FAIL_OST_ENOENT 0x217
+ do_facet ost sysctl -w lustre.fail_loc=0x217
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+ RC=$?
+ do_facet ost sysctl -w lustre.fail_loc=0
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+
+ if [[ $RC -eq 0 ]]; then
+ error "Must return error due to dropped pages, rc=$RC"
+ return 1;
+ fi
+
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ return 1;
+ fi
+
+ echo "Dirty pages not leaked on ENOENT"
+
+ # Due to the above error the OSC will issue all RPCs syncronously
+ # until a subsequent RPC completes successfully without error.
+ multiop $DIR/$tfile Ow4096yc
+ rm -f $DIR/$tfile
+
+ return 0
+}
+run_test 118b "Reclaim dirty pages on fatal error =========="
+
+test_118c()
+{
+ remote_ost_nodsh && skip "remote OST" && return
+
+ reset_async
+
+ #define OBD_FAIL_OST_EROFS 0x216
+ do_facet ost sysctl -w lustre.fail_loc=0x216
+
+ # multiop should block due to fsync until pages are written
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
+ MULTIPID=$!
+ sleep 1
+
+ if [[ `ps h -o comm -p $MULTIPID` != "multiop" ]]; then
+ error "Multiop failed to block on fsync, pid=$MULTIPID"
+ fi
+
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $WRITEBACK -eq 0 ]]; then
+ error "No page in writeback, writeback=$WRITEBACK"
+ fi
+
+ do_facet ost sysctl -w lustre.fail_loc=0
+ wait $MULTIPID
+ RC=$?
+ if [[ $RC -ne 0 ]]; then
+ error "Multiop fsync failed, rc=$RC"
+ fi
+
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "Dirty pages flushed via fsync on EROFS"
+ return 0
+}
+run_test 118c "Fsync blocks on EROFS until dirty pages are flushed =========="
+
+test_118d()
+{
+ remote_ost_nodsh && skip "remote OST" && return
+
+ reset_async
+
+ #define OBD_FAIL_OST_BRW_PAUSE_BULK
+ do_facet ost sysctl -w lustre.fail_loc=0x214
+ # multiop should block due to fsync until pages are written
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
+ MULTIPID=$!
+ sleep 1
+
+ if [[ `ps h -o comm -p $MULTIPID` != "multiop" ]]; then
+ error "Multiop failed to block on fsync, pid=$MULTIPID"
+ fi
+
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $WRITEBACK -eq 0 ]]; then
+ error "No page in writeback, writeback=$WRITEBACK"
+ fi
+
+ wait $MULTIPID || error "Multiop fsync failed, rc=$?"
+ do_facet ost sysctl -w lustre.fail_loc=0
+
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "Dirty pages gaurenteed flushed via fsync"
+ return 0
+}
+run_test 118d "Fsync validation inject a delay of the bulk =========="
+
+test_118f() {
+ reset_async
+
+ #define OBD_FAIL_OSC_BRW_PREP_REQ2 0x40a
+ sysctl -w lustre.fail_loc=0x8000040a
+
+ # Should simulate EINVAL error which is fatal
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+ RC=$?
+ if [[ $RC -eq 0 ]]; then
+ error "Must return error due to dropped pages, rc=$RC"
+ fi
+
+ sysctl -w lustre.fail_loc=0x0
+
+ LOCKED=$(grep -c locked $LPROC/llite/*/dump_page_cache)
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $LOCKED -ne 0 ]]; then
+ error "Locked pages remain in cache, locked=$LOCKED"
+ fi
+
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "No pages locked after fsync"
+
+ reset_async
+ return 0
+}
+run_test 118f "Simulate unrecoverable OSC side error =========="
+
+test_118g() {
+ reset_async
+
+ #define OBD_FAIL_OSC_BRW_PREP_REQ 0x406
+ sysctl -w lustre.fail_loc=0x406
+
+ # simulate local -ENOMEM
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+ RC=$?
+
+ sysctl -w lustre.fail_loc=0
+ if [[ $RC -eq 0 ]]; then
+ error "Must return error due to dropped pages, rc=$RC"
+ fi
+
+ LOCKED=$(grep -c locked $LPROC/llite/*/dump_page_cache)
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $LOCKED -ne 0 ]]; then
+ error "Locked pages remain in cache, locked=$LOCKED"
+ fi
+
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "No pages locked after fsync"
+
+ reset_async
+ return 0
+}
+run_test 118g "Don't stay in wait if we got local -ENOMEM =========="
+
+test_118h() {
+ remote_ost_nodsh && skip "remote OST" && return
+
+ reset_async
+
+ #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e
+ do_facet ost sysctl -w lustre.fail_loc=0x20e
+ # Should simulate ENOMEM error which is recoverable and should be handled by timeout
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+ RC=$?
+
+ do_facet ost sysctl -w lustre.fail_loc=0
+ if [[ $RC -eq 0 ]]; then
+ error "Must return error due to dropped pages, rc=$RC"
+ fi
+
+ LOCKED=$(grep -c locked $LPROC/llite/*/dump_page_cache)
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $LOCKED -ne 0 ]]; then
+ error "Locked pages remain in cache, locked=$LOCKED"
+ fi
+
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "No pages locked after fsync"
+
+ return 0
+}
+run_test 118h "Verify timeout in handling recoverables errors =========="
+
+test_118i() {
+ remote_ost_nodsh && skip "remote OST" && return
+
+ reset_async
+
+ #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e
+ do_facet ost sysctl -w lustre.fail_loc=0x20e
+
+ # Should simulate ENOMEM error which is recoverable and should be handled by timeout
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
+ PID=$!
+ sleep 5
+ do_facet ost sysctl -w lustre.fail_loc=0
+
+ wait $PID
+ RC=$?
+ if [[ $RC -ne 0 ]]; then
+ error "got error, but should be not, rc=$RC"
+ fi
+
+ LOCKED=$(grep -c locked $LPROC/llite/*/dump_page_cache)
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $LOCKED -ne 0 ]]; then
+ error "Locked pages remain in cache, locked=$LOCKED"
+ fi
+
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "No pages locked after fsync"
+
+ return 0
+}
+run_test 118i "Fix error before timeout in recoverable error =========="
+
+test_118j() {
+ remote_ost_nodsh && skip "remote OST" && return
+
+ reset_async
+
+ #define OBD_FAIL_OST_BRW_WRITE_BULK2 0x220
+ do_facet ost sysctl -w lustre.fail_loc=0x220
+
+ # return -EIO from OST
+ multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+ RC=$?
+ do_facet ost sysctl -w lustre.fail_loc=0x0
+ if [[ $RC -eq 0 ]]; then
+ error "Must return error due to dropped pages, rc=$RC"
+ fi
+
+ LOCKED=$(grep -c locked $LPROC/llite/*/dump_page_cache)
+ DIRTY=$(grep -c dirty $LPROC/llite/*/dump_page_cache)
+ WRITEBACK=$(grep -c writeback $LPROC/llite/*/dump_page_cache)
+ if [[ $LOCKED -ne 0 ]]; then
+ error "Locked pages remain in cache, locked=$LOCKED"
+ fi
+
+ # in recoverable error on OST we want resend and stay until it finished
+ if [[ $DIRTY -ne 0 || $WRITEBACK -ne 0 ]]; then
+ error "Dirty pages not flushed to disk, dirty=$DIRTY, writeback=$WRITEBACK"
+ fi
+
+ rm -f $DIR/$tfile
+ echo "No pages locked after fsync"
+
+ return 0
+}
+run_test 118j "Simulate unrecoverable OST side error =========="
+
+test_118k()
+{
+ #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e
+ do_facet ost sysctl -w lustre.fail_loc=0x20e
+ mkdir -p $DIR/$tdir
+
+ for ((i=0;i<10;i++)); do
+ (dd if=/dev/zero of=$DIR/$tdir/$tfile-$i bs=1M count=10 || \
+ error "dd to $DIR/$tdir/$tfile-$i failed" )&
+ SLEEPPID=$!
+ sleep 0.500s
+ kill $SLEEPPID
+ wait $SLEEPPID
+ done
+
+ sysctl -w lustre.fail_loc=0
+}
+run_test 118k "bio alloc -ENOMEM and IO TERM handling ========="
+
+test_119a() # bug 11737
+{
+ BSIZE=$((512 * 1024))
+ directio write $DIR/$tfile 0 1 $BSIZE
+ # We ask to read two blocks, which is more than a file size.
+ # directio will indicate an error when requested and actual
+ # sizes aren't equeal (a normal situation in this case) and
+ # print actual read amount.
+ NOB=`directio read $DIR/$tfile 0 2 $BSIZE | awk '/error/ {print $6}'`
+ if [ "$NOB" != "$BSIZE" ]; then
+ error "read $NOB bytes instead of $BSIZE"
+ fi
+ rm -f $DIR/$tfile
+}
+run_test 119a "Short directIO read must return actual read amount"
+
+test_119b() # bug 11737
+{
+ [ "$OSTCOUNT" -lt "2" ] && skip "skipping 2-stripe test" && return
+
+ $SETSTRIPE $DIR/$tfile -c 2
+ dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 seek=1 || error "dd failed"
+ sync
+ multiop $DIR/$tfile oO_RDONLY:O_DIRECT:r$((2048 * 1024)) || \
+ error "direct read failed"
+ rm -f $DIR/$tfile
+}
+run_test 119b "Sparse directIO read must return actual read amount"
+
+test_119c() # bug 13099
+{
+ BSIZE=1048576
+ directio write $DIR/$tfile 3 1 $BSIZE || error "direct write failed"
+ directio readhole $DIR/$tfile 0 2 $BSIZE || error "reading hole failed"
+ rm -f $DIR/$tfile
+}
+run_test 119c "Testing for direct read hitting hole"
+
+test_120a() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ cancel_lru_locks mdc
+ stat $DIR/$tdir > /dev/null
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ mkdir $DIR/$tdir/d1
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ [ $can1 -eq $can2 ] || error $((can2-can1)) "cancel RPC occured."
+ [ $blk1 -eq $blk2 ] || error $((blk2-blk1)) "blocking RPC occured."
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120a "Early Lock Cancel: mkdir test"
+
+test_120b() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ cancel_lru_locks mdc
+ stat $DIR/$tdir > /dev/null
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ touch $DIR/$tdir/f1
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ [ $can1 -eq $can2 ] || error $((can2-can1)) "cancel RPC occured."
+ [ $blk1 -eq $blk2 ] || error $((blk2-blk1)) "blocking RPC occured."
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120b "Early Lock Cancel: create test"
+
+test_120c() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ mkdir -p $DIR/$tdir/d1 $DIR/$tdir/d2
+ touch $DIR/$tdir/d1/f1
+ cancel_lru_locks mdc
+ stat $DIR/$tdir/d1 $DIR/$tdir/d2 $DIR/$tdir/d1/f1 > /dev/null
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ ln $DIR/$tdir/d1/f1 $DIR/$tdir/d2/f2
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ [ $can1 -eq $can2 ] || error $((can2-can1)) "cancel RPC occured."
+ [ $blk1 -eq $blk2 ] || error $((blk2-blk1)) "blocking RPC occured."
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120c "Early Lock Cancel: link test"
+
+test_120d() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ touch $DIR/$tdir
+ cancel_lru_locks mdc
+ stat $DIR/$tdir > /dev/null
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ chmod a+x $DIR/$tdir
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ [ $can1 -eq $can2 ] || error $((can2-can1)) "cancel RPC occured."
+ [ $blk1 -eq $blk2 ] || error $((blk2-blk1)) "blocking RPC occured."
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120d "Early Lock Cancel: setattr test"
+
+test_120e() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ dd if=/dev/zero of=$DIR/$tdir/f1 count=1
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+ dd if=$DIR/$tdir/f1 of=/dev/null
+ stat $DIR/$tdir $DIR/$tdir/f1 > /dev/null
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ unlink $DIR/$tdir/f1
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ [ $can1 -eq $can2 ] || error $((can2-can1)) "cancel RPC occured."
+ [ $blk1 -eq $blk2 ] || error $((blk2-blk1)) "blocking RPC occured."
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120e "Early Lock Cancel: unlink test"
+
+test_120f() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ mkdir -p $DIR/$tdir/d1 $DIR/$tdir/d2
+ dd if=/dev/zero of=$DIR/$tdir/d1/f1 count=1
+ dd if=/dev/zero of=$DIR/$tdir/d2/f2 count=1
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+ dd if=$DIR/$tdir/d1/f1 of=/dev/null
+ dd if=$DIR/$tdir/d2/f2 of=/dev/null
+ stat $DIR/$tdir/d1 $DIR/$tdir/d2 $DIR/$tdir/d1/f1 $DIR/$tdir/d2/f2 > /dev/null
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ mv $DIR/$tdir/d1/f1 $DIR/$tdir/d2/f2
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ [ $can1 -eq $can2 ] || error $((can2-can1)) "cancel RPC occured."
+ [ $blk1 -eq $blk2 ] || error $((blk2-blk1)) "blocking RPC occured."
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120f "Early Lock Cancel: rename test"
+
+test_120g() {
+ [ -z "`grep early_lock_cancel $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no early lock cancel on server" && return 0
+ lru_resize_disable mdc
+ lru_resize_disable osc
+ count=10000
+ echo create $count files
+ mkdir -p $DIR/$tdir
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+ t0=`date +%s`
+
+ can0=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk0=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ createmany -o $DIR/$tdir/f $count
+ sync
+ can1=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk1=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ t1=`date +%s`
+ echo total: $((can1-can0)) cancels, $((blk1-blk0)) blockings
+ echo rm $count files
+ rm -r $DIR/$tdir
+ sync
+ can2=`awk '/ldlm_cancel/ {print $2}' $LPROC/ldlm/services/ldlm_canceld/stats`
+ blk2=`awk '/ldlm_bl_callback/ {print $2}' $LPROC/ldlm/services/ldlm_cbd/stats`
+ t2=`date +%s`
+ echo total: $count removes in $((t2-t1))
+ echo total: $((can2-can1)) cancels, $((blk2-blk1)) blockings
+ sleep 2
+ # wait for commitment of removal
+ lru_resize_enable mdc
+ lru_resize_enable osc
+}
+run_test 120g "Early Lock Cancel: performance test"
+
+test_121() { #bug #10589
+ rm -rf $DIR/$tfile
+ writes=$(LANG=C dd if=/dev/zero of=$DIR/$tfile count=1 2>&1 | awk -F '+' '/out/ {print $1}')
+#define OBD_FAIL_LDLM_CANCEL_RACE 0x310
+ sysctl -w lustre.fail_loc=0x310
+ cancel_lru_locks osc > /dev/null
+ reads=$(LANG=C dd if=$DIR/$tfile of=/dev/null 2>&1 | awk -F '+' '/in/ {print $1}')
+ sysctl -w lustre.fail_loc=0
+ [ "$reads" -eq "$writes" ] || error "read" $reads "blocks, must be" $writes
+}
+run_test 121 "read cancel race ========="
+
+test_124a() {
+ [ -z "`grep lru_resize $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no lru resize on server" && return 0
+ NR=2000
+ mkdir -p $DIR/$tdir || error "failed to create $DIR/$tdir"
+
+ # use touch to produce $NR new locks
+ log "create $NR files at $DIR/$tdir"
+ createmany -o $DIR/$tdir/f $NR ||
+ error "failed to create $NR files in $DIR/$tdir"
+
+ cancel_lru_locks mdc
+ ls -l $DIR/$tdir > /dev/null
+
+ NSDIR=""
+ LRU_SIZE=0
+ for F in $LPROC/ldlm/namespaces/*mdc-*/lru_size; do
+ LRU_SIZE=$(cat $F)
+ if [ $LRU_SIZE -gt $(default_lru_size) ]; then
+ NSDIR=$(dirname $F)
+ log "using $(basename $NSDIR) namespace"
+ break
+ fi
+ done
+
+ if [ -z "$NSDIR" -o $LRU_SIZE -lt $(default_lru_size) ]; then
+ skip "Not enough cached locks created!"
+ return 0
+ fi
+ log "created $LRU_SIZE lock(s)"
+
+ # we want to sleep 30s to not make test too long
+ SLEEP=30
+ SLEEP_ADD=2
+
+ # we know that lru resize allows one client to hold $LIMIT locks for 10h
+ MAX_HRS=10
+
+ # get the pool limit
+ LIMIT=`cat $NSDIR/pool/limit`
+
+ # calculate lock volume factor taking into account data set size and the
+ # rule that number of locks will be getting smaller durring sleep interval
+ # and we need to additionally enforce LVF to take this into account.
+ # Use $LRU_SIZE_B here to take into account real number of locks created
+ # in the case of CMD, LRU_SIZE_B != $NR in most of cases
+ LVF=$(($MAX_HRS * 60 * 60 * $LIMIT / $SLEEP))
+ LRU_SIZE_B=$LRU_SIZE
+ log "make client drop locks $LVF times faster so that ${SLEEP}s is enough to cancel $LRU_SIZE lock(s)"
+ OLD_LVF=`cat $NSDIR/pool/lock_volume_factor`
+ echo "$LVF" > $NSDIR/pool/lock_volume_factor
+ log "sleep for $((SLEEP+SLEEP_ADD))s"
+ sleep $((SLEEP+SLEEP_ADD))
+ echo "$OLD_LVF" > $NSDIR/pool/lock_volume_factor
+ LRU_SIZE_A=`cat $NSDIR/lru_size`
+
+ [ $LRU_SIZE_B -gt $LRU_SIZE_A ] || {
+ error "No locks dropped in "$((SLEEP+SLEEP_ADD))"s. LRU size: $LRU_SIZE_A"
+ unlinkmany $DIR/$tdir/f $NR
+ return
+ }
+
+ log "Dropped "$((LRU_SIZE_B-LRU_SIZE_A))" locks in "$((SLEEP+SLEEP_ADD))"s"
+ log "unlink $NR files at $DIR/$tdir"
+ unlinkmany $DIR/$tdir/f $NR
+}
+run_test 124a "lru resize ======================================="
+
+test_124b() {
+ [ -z "`grep lru_resize $LPROC/mdc/*/connect_flags`" ] && \
+ skip "no lru resize on server" && return 0
+
+ # even for cmd no matter what metadata namespace to use for getting
+ # the limit, we use appropriate.
+ NSDIR=`find $LPROC/ldlm/namespaces | grep mdc | head -1`
+ LIMIT=`cat $NSDIR/pool/limit`
+
+ NR=$(($(default_lru_size)*20))
+ if [ $NR -gt $LIMIT ]; then
+ NR=$LIMIT
+ fi
+ lru_resize_disable mdc
+ mkdir -p $DIR/$tdir/disable_lru_resize ||
+ error "failed to create $DIR/$tdir/disable_lru_resize"
+
+ createmany -o $DIR/$tdir/disable_lru_resize/f $NR
+ log "doing ls -la $DIR/$tdir/disable_lru_resize 3 times"
+ cancel_lru_locks mdc
+ stime=`date +%s`
+ PID=""
+ ls -la $DIR/$tdir/disable_lru_resize > /dev/null &
+ PID="$PID $!"
+ sleep 2
+ ls -la $DIR/$tdir/disable_lru_resize > /dev/null &
+ PID="$PID $!"
+ sleep 2
+ ls -la $DIR/$tdir/disable_lru_resize > /dev/null &
+ PID="$PID $!"
+ wait $PID
+ etime=`date +%s`
+ nolruresize_delta=$((etime-stime))
+ log "ls -la time: $nolruresize_delta seconds"
+ log "lru_size = $(cat $NSDIR/lru_size)"
+ unlinkmany $DIR/$tdir/disable_lru_resize/f $NR
+
+ lru_resize_enable mdc
+ mkdir -p $DIR/$tdir/enable_lru_resize ||
+ error "failed to create $DIR/$tdir/enable_lru_resize"
+
+ createmany -o $DIR/$tdir/enable_lru_resize/f $NR
+ log "doing ls -la $DIR/$tdir/enable_lru_resize 3 times"
+ cancel_lru_locks mdc
+ stime=`date +%s`
+ PID=""
+ ls -la $DIR/$tdir/enable_lru_resize > /dev/null &
+ PID="$PID $!"
+ sleep 2
+ ls -la $DIR/$tdir/enable_lru_resize > /dev/null &
+ PID="$PID $!"
+ sleep 2
+ ls -la $DIR/$tdir/enable_lru_resize > /dev/null &
+ PID="$PID $!"
+ wait $PID
+ etime=`date +%s`
+ lruresize_delta=$((etime-stime))
+ log "ls -la time: $lruresize_delta seconds"
+ log "lru_size = $(cat $NSDIR/lru_size)"
+
+ if [ $lruresize_delta -gt $nolruresize_delta ]; then
+ log "ls -la is $(((lruresize_delta - $nolruresize_delta) * 100 / $nolruresize_delta))% slower with lru resize enabled"
+ elif [ $nolruresize_delta -gt $lruresize_delta ]; then
+ log "ls -la is $(((nolruresize_delta - $lruresize_delta) * 100 / $nolruresize_delta))% faster with lru resize enabled"
+ else
+ log "lru resize performs the same with no lru resize"
+ fi
+ unlinkmany $DIR/$tdir/enable_lru_resize/f $NR
+}
+run_test 124b "lru resize (performance test) ======================="
+
+test_125() { # 13358
+ [ -z "$(grep acl $LPROC/mdc/*-mdc-*/connect_flags)" ] && skip "must have acl enabled" && return
+ mkdir -p $DIR/d125 || error "mkdir failed"
+ $SETSTRIPE $DIR/d125 -s 65536 -c -1 || error "setstripe failed"
+ setfacl -R -m u:bin:rwx $DIR/d125 || error "setfacl $DIR/d125 failed"
+ ls -ld $DIR/d125 || error "cannot access $DIR/d125"
+}
+run_test 125 "don't return EPROTO when a dir has a non-default striping and ACLs"
+
+test_126() { # bug 12829/13455
+ [ "$UID" != 0 ] && echo "skipping $TESTNAME (must run as root)" && return
+ $RUNAS -u 0 -g 1 touch $DIR/$tfile || error "touch failed"
+ gid=`ls -n $DIR/$tfile | awk '{print $4}'`
+ rm -f $DIR/$tfile
+ [ $gid -eq "1" ] || error "gid is set to" $gid "instead of 1"
+}
+run_test 126 "check that the fsgid provided by the client is taken into account"
+