3 # This script is used to test large size LUN support in Lustre.
5 ################################################################################
8 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
9 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
14 if [ "$REFORMAT" != "yes" ]; then
15 skip_env "$0 reformats all devices,\
16 please set REFORMAT to run this test"
20 # Variable to run mdsrate
21 THREADS_PER_CLIENT=${THREADS_PER_CLIENT:-5} # thread(s) per client node
22 MACHINEFILE=${MACHINEFILE:-$TMP/$TESTSUITE.machines}
23 NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
24 NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
27 ALWAYS_EXCEPT="$LARGE_LUN_EXCEPT"
30 LARGE_LUN_RESTORE_MOUNT=false
31 if is_mounted $MOUNT || is_mounted $MOUNT2; then
32 LARGE_LUN_RESTORE_MOUNT=true
34 # Unmount and cleanup the Lustre filesystem
38 FULL_MODE=${FULL_MODE:-false}
39 RUN_FSCK=${RUN_FSCK:-true}
40 # if SLOW=yes, enable the FULL_MODE
41 [[ $SLOW = yes ]] && FULL_MODE=true
42 #########################################################################
43 # Dump the super block information for the filesystem present on device.
49 log "dump the super block information on $facet device $dev"
50 local fstype=$(facet_fstype $facet)
54 cmd="$DUMPE2FS -h $dev" ;;
56 cmd="$ZDB -l $(facet_vdevice $facet)" ;;
58 error "unknown fstype!" ;;
61 do_facet $facet "$cmd"
64 # Report Lustre filesystem disk space usage and inodes usage of each MDT/OST.
73 cmd="lfs df -h $mnt_pnt"
77 cmd="lfs df -i $mnt_pnt"
82 # Cleanup the directories and files created by llverfs utility.
85 local mnt=${2:-$MOUNT}
86 local cmd="rm -rf $mnt/{*.filecount,dir*}"
87 do_facet $target "$cmd"
92 generate_machine_file $NODES_TO_USE $MACHINEFILE ||
93 error "can not generate machinefile"
95 # set the default stripe count for files in this test to one
96 local testdir=$MOUNT/mdsrate
99 $LFS setstripe $testdir -i 0 -c 1
102 local num_dirs=$THREADS_PER_CLIENT
103 [[ $num_dirs -eq 0 ]] && num_dirs=1
104 local free_inodes=$(lfs df -i $MOUNT | grep "OST:0" | awk '{print $4}')
106 num_files=$((free_inodes / num_dirs))
108 local command="$MDSRATE $MDSRATE_DEBUG --create --verbose \
109 --ndirs $num_dirs --dirfmt '$testdir/dir%d' \
110 --nfiles $num_files --filefmt 'file%%d'"
113 mpi_run -machinefile $MACHINEFILE \
114 -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) $command
116 if [ ${PIPESTATUS[0]} != 0 ]; then
117 error "mdsrate create failed"
123 local fstype=$(facet_fstype $facet)
127 run_e2fsck $(facet_active_host $facet) $(facet_device $facet) \
128 "-y" || error "run e2fsck error"
131 # Could call fsck.zfs, but currently it does nothing,
132 # Could also call zpool scrub, but that could take a LONG time
133 # do_facet $facet "fsck.zfs $(facet_device $facet)"
138 # Run e2fsck on MDS and OST
142 check_fsfacet $SINGLEMDS
144 for num in $(seq $OSTCOUNT); do
145 check_fsfacet ost${num}
148 ################################## Main Flow ###################################
152 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
153 skip "ldiskfs only test" && return
155 for num in $(seq $OSTCOUNT); do
156 dev=$(ostdevname $num)
157 log "run llverdev on the OST $dev"
158 do_rpc_nodes $(facet_host ost${num}) run_llverdev $dev -vpf ||
159 error "llverdev on $dev failed!"
161 # restore format overwritten by llverdev
164 run_test 1 "run llverdev on raw LUN"
171 for num in $(seq $OSTCOUNT); do
172 dev=$(ostdevname $num)
173 ostmnt=$(facet_mntpt ost${num})
174 fstype=$(facet_fstype ost${num})
176 # Mount the OST as an ldiskfs filesystem.
177 log "mount the OST $dev as a $fstype filesystem"
178 add ost${num} $(mkfs_opts ost${num} $dev) $FSTYPE_OPT \
179 --reformat $(ostdevname $num) \
180 $(ostvdevname $num) > /dev/null ||
181 error "format ost${num} error"
182 if [ $fstype == zfs ]; then
183 import_zpool ost${num}
184 do_facet ost${num} "$ZFS set canmount=on $dev; " \
185 "$ZFS set mountpoint=legacy $dev; $ZFS list $dev"
187 run_dumpfs ost${num} $dev
188 do_facet ost${num} mount -t $fstype $dev \
189 $ostmnt "$OST_MOUNT_OPTS"
191 # Run llverfs on the mounted ldiskfs filesystem in partial mode
192 # to ensure that the kernel can perform filesystem operations
193 # on the complete device without any errors.
194 log "run llverfs in partial mode on the OST $fstype $ostmnt"
195 do_rpc_nodes $(facet_host ost${num}) run_llverfs $ostmnt -vpl \
196 "no" || error "run_llverfs error on $fstype"
199 log "unmount the OST $dev"
202 # After llverfs is run on the ldiskfs filesystem in partial
203 # mode, a full e2fsck should be run to catch any errors early.
204 $RUN_FSCK && check_fsfacet ost${num}
207 log "full mode, mount the OST $dev as a $fstype again"
208 if [ $fstype == zfs ]; then
209 import_zpool ost${num}
211 do_facet ost${num} mount -t $(facet_fstype ost${num}) \
212 $dev $ostmnt "$OST_MOUNT_OPTS"
213 cleanup_dirs ost${num} $ostmnt
214 do_facet ost${num} "sync"
216 run_dumpfs ost${num} $dev
218 # Run llverfs on the mounted ldiskfs filesystem in full
219 # mode to ensure that the kernel can perform filesystem
220 # operations on the complete device without any errors.
221 log "run llverfs in full mode on OST $fstype $ostmnt"
222 do_rpc_nodes $(facet_host ost${num}) run_llverfs \
224 error "run_llverfs error on $fstype"
227 log "unmount the OST $dev"
230 # After llverfs is run on the ldiskfs filesystem in
231 # full mode, a full e2fsck should be run to catch any
233 $RUN_FSCK && check_fsfacet ost${num}
236 # there is no reason to continue using ost devices
237 # filled by llverfs as ldiskfs
240 run_test 2 "run llverfs on OST ldiskfs/zfs filesystem"
243 [ -z "$CLIENTS" ] && skip_env "CLIENTS not defined, skipping" && return
244 [ -z "$MPIRUN" ] && skip_env "MIPRUN not defined, skipping" && return
245 [ -z "$MDSRATE" ] && skip_env "MDSRATE not defined, skipping" && return
246 [ ! -x $MDSRATE ] && skip_env "$MDSRATE not built, skipping" && return
247 # Setup the Lustre filesystem.
248 log "setup the lustre filesystem"
249 REFORMAT="yes" check_and_setup_lustre
251 log "run mdsrate to use up the free inodes."
252 # Run the mdsrate test suite.
260 run_test 3 "use up free inodes on the OST with mdsrate"
263 # Setup the Lustre filesystem.
264 log "setup the lustre filesystem"
265 REFORMAT="yes" check_and_setup_lustre
268 for num in $(seq $OSTCOUNT); do
269 dev=$(ostdevname $num)
270 run_dumpfs ost${num} $dev
273 # Run llverfs on the mounted Lustre filesystem both in partial and
274 # full mode to to fill the filesystem and verify the file contents.
275 log "run llverfs in partial mode on the Lustre filesystem $MOUNT"
276 run_llverfs $MOUNT -vp "no" || error "run_llverfs error on lustre"
284 # Setup the Lustre filesystem again.
285 log "setup the lustre filesystem again"
288 cleanup_dirs client $MOUNT
292 for num in $(seq $OSTCOUNT); do
293 dev=$(ostdevname $num)
294 run_dumpfs ost${num} $dev
297 log "run llverfs in full mode on the Lustre filesystem $MOUNT"
298 run_llverfs $MOUNT -vl "no" ||
299 error "run_llverfs error on lustre"
307 run_test 4 "run llverfs on lustre filesystem"
310 $LARGE_LUN_RESTORE_MOUNT && setupall
311 check_and_cleanup_lustre