3 # lustre This shell script takes care of starting and stopping
7 # description: Part of the lustre file system.
9 # config: /etc/sysconfig/lustre
12 # Required-Start: lnet
15 # Description: Lustre init script
18 PATH=/sbin:/usr/sbin:/bin:/usr/bin
20 # Source function library.
21 . /etc/rc.d/init.d/functions
23 # Source networking configuration.
24 if [ ! -f /etc/sysconfig/network ]; then
28 . /etc/sysconfig/network
30 LDEV=${LDEV:-"/usr/sbin/ldev"}
31 ZPOOL_LAYOUT=/usr/bin/zpool_layout
32 UDEVADM=${UDEVADM:-/sbin/udevadm}
34 # Check that networking is up.
35 [ "${NETWORKING}" = "no" ] && exit 0
37 # Check for and source configuration file otherwise set defaults
38 [ -f /etc/sysconfig/lustre ] && . /etc/sysconfig/lustre
39 FSCK_ARGS=${FSCK_ARGS:-""}
40 MOUNT_OPTIONS=${MOUNT_OPTIONS:-""}
41 LOCAL_SRV=${LOCAL_SRV:-"`$LDEV -l 2>/dev/null`"}
42 FOREIGN_SRV=${FOREIGN_SRV:-"`$LDEV -f 2>/dev/null`"}
43 REQUIRE_MMP_FEATURE=${REQUIRE_MMP_FEATURE:-${FOREIGN_SRV:+"yes"}}
44 LOCAL_MOUNT_DIR=${LOCAL_MOUNT_DIR:-"/mnt/lustre/local"}
45 FOREIGN_MOUNT_DIR=${FOREIGN_MOUNT_DIR:-"/mnt/lustre/foreign"}
46 SETUP_DEVICES=${SETUP_DEVICES:-""}
47 ZPOOL_LAYOUT_BUSES=${ZPOOL_LAYOUT_BUSES:-""}
48 ZPOOL_LAYOUT_PORTS=${ZPOOL_LAYOUT_PORTS:-""}
49 ZPOOL_LAYOUT_MAP=${ZPOOL_LAYOUT_MAP:-""}
50 MOUNT_DELAY=${MOUNT_DELAY:-2}
51 LOAD_ZFS=${LOAD_ZFS:-""}
53 if [ -z "$TUNE2FS" ] ; then
54 TUNE2FS=`which tunefs.ldiskfs 2>/dev/null`
55 if [ -z "$TUNE2FS" ] ; then
56 TUNE2FS=`which tune2fs 2>/dev/null`
60 if [ -z "$PFSCK" ] ; then
61 PFSCK=`which pfsck.ldiskfs 2>/dev/null`
62 if [ -z "$PFSCK" ] ; then
63 PFSCK=`which fsck 2>/dev/null`
71 if [ -n "$ZPOOL_LAYOUT_BUSES" -a -n "$ZPOOL_LAYOUT_PORTS" ] ; then
72 MAP_ARG=${ZPOOL_LAYOUT_MAP:+"-m $ZPOOL_LAYOUT_MAP"}
73 $ZPOOL_LAYOUT -t -b "$ZPOOL_LAYOUT_BUSES" \
74 -p "$ZPOOL_LAYOUT_PORTS" $MAP_ARG
76 if [ "$LOAD_ZFS" = "yes" ] && ! modprobe zfs ; then
77 echo "Failed to load zfs module. Aborting."
86 for label in $labels; do
87 devtype=`$LDEV -t $label`
88 if [ "$devtype" = "zfs" ] ; then
90 elif [ "$devtype" = "md" ] ; then
91 dev=`label_to_device $label`
92 journal=`$LDEV -j $label`
94 stop_md_device $journal
103 local pool=`$LDEV -z $label`
104 local args="-N $ZPOOL_IMPORT_ARGS"
105 local cache=`$LDEV -r $label`
106 # -c is incompatible with -d
107 if [ -n "$cache" ] ; then
108 args="$args -c $cache"
109 elif [ -n "$ZPOOL_IMPORT_DIR" ] ; then
110 args="$args -d $ZPOOL_IMPORT_DIR"
113 if zpool status $pool >/dev/null 2>&1 ; then
115 elif [ -n "$pool" ] ; then
116 zpool import $pool $args 2>/dev/null
119 if [ $result -ne 0 ] ; then
120 echo "Unexpected return code from import of pool $pool: $result"
128 local pool=`$LDEV -z $label`
129 zpool export $pool 2>/dev/null
132 # Trigger udev and wait for it to settle.
135 if [ -x ${UDEVADM} ]; then
136 ${UDEVADM} trigger --action=change --subsystem-match=block
144 # Usage: run_preexec_check [ start | restart | condrestart ]
145 # The single parameter will be passed to the PREEXEC_SCRIPT
148 if [ -n "$PREEXEC_CHECK" ] && ! $PREEXEC_CHECK ; then
149 echo "Pre-exec check \"$PREEXEC_CHECK\" failed. Aborting."
153 if [ -n "$PREEXEC_SCRIPT" ] && ! "$PREEXEC_SCRIPT" "$1" ; then
154 echo "Pre-exec script \"$PREEXEC_SCRIPT\" failed. Aborting."
159 # Usage: run_postexec_check [ start | restart | condrestart ]
160 # The single parameter will be passed to the PREEXEC_SCRIPT
161 run_postexec_check ()
163 if [ -n "$POSTEXEC_CHECK" ] && ! $POSTEXEC_CHECK ; then
164 echo "Post-exec check \"$POSTEXEC_CHECK\" failed. Aborting."
168 if [ -n "$POSTEXEC_SCRIPT" ] && ! "$POSTEXEC_SCRIPT" "$1" ; then
169 echo "Post-exec script \"$POSTEXEC_SCRIPT\" failed. Aborting."
174 # Usage: adjust_scsi_timeout <dev>
175 adjust_scsi_timeout ()
179 if [ -n "$SCSI_DEVICE_TIMEOUT" ]; then
180 # make sure that it is actually a SCSI (sd) device
181 local name=`basename $dev`
182 local proc=/sys/block/${name}/device/timeout
183 local driver=`readlink /sys/block/${name}/device/driver`
184 if [ -n "$driver" ] && [ "`basename $driver`" == "sd" ]; then
185 if ! echo $SCSI_DEVICE_TIMEOUT >$proc; then
186 echo "FAILED: could not adjust ${dev} timeout"
194 # Usage: fsck_test <dev> [ <dev> ... ]
195 # Checks all devices in parallel if FSCK_ARGS is set.
200 # Filter out non-absolute paths, which are probably ZFS datasets
201 devices=`echo $devices |xargs -n 1|grep '^/'|xargs`
203 if [ -n "${FSCK_ARGS}" -a -n "$devices" ]; then
204 if [ -x $PFSCK ] ; then
205 echo "$PFSCK $devices -- ${FSCK_ARGS}"
206 $PFSCK $devices -- ${FSCK_ARGS}
207 if [ $? -ne 0 -a $? -ne 1 ] ; then
208 echo "FAILED: $PFSCK -- ${FSCK_ARGS}: $?"
212 echo "$PFSCK not found"
219 # Usage: test_feature_flag <dev> <flag>
227 for feature in `$TUNE2FS -l $dev 2>/dev/null \
228 | grep features: | sed -e 's/^.*: //'`; do
229 if [ "$feature" == "$flag" ]; then
238 # Usage: mmp_test <dev>
239 # Returns 0 if it is set or not required, 1 if unset and required or error.
245 if [ "$REQUIRE_MMP_FEATURE" == "yes" ]; then
246 if [ -x $TUNE2FS ]; then
247 if ! test_feature_flag $dev "mmp"; then
248 echo "mmp feature flag is not set on $dev"
252 echo "$TUNE2FS not found"
260 # Usage: label_to_mountpt <label>
261 # Prints mount point path, if label matches a local or foreign server.
267 for serv in $LOCAL_SRV; do
268 if [ "$serv" == "$label" ]; then
269 echo "$LOCAL_MOUNT_DIR/$label"
273 for serv in $FOREIGN_SRV; do
274 if [ "$serv" == "$label" ]; then
275 echo "$FOREIGN_MOUNT_DIR/$label"
281 # Usage: label_to_device <label>
282 # Prints canonical device path.
286 local path=/dev/disk/by-label/$label
288 if [ -h $path ] ; then
289 readlink --canonicalize $path
295 # helper for mountpt_is_active() and device_is_active()
296 declare -r awkprog='BEGIN {rc = 1;}
297 { if ($field == path) {rc = 0;} }
300 # Usage: mountpt_is_active <label>
301 # Return 1 (inactive) on invalid label.
304 local dir=`label_to_mountpt $1`
307 if [ -n "$dir" ]; then
308 cat /proc/mounts | awk "$awkprog" field=2 path=$dir
314 # Usage: device_is_active <label>
315 # Return 1 (inactive) on invalid label.
318 local dev=`label_to_device $1`
321 if [ -n "$dev" ]; then
322 cat /proc/mounts | awk "$awkprog" field=1 path=$dir
328 # Usage: mount_one_device <label> <successflag> [devtype]
329 # Remove <successflag> on error (trick to detect errors after parallel runs).
335 local dev=`label_to_device $label`
336 local dir=`label_to_mountpt $label`
338 # $dir and $dev have already been checked at ths point
339 if [ ! -d $dir ] && ! mkdir -p $dir; then
343 echo "Mounting $dev on $dir"
344 if ! mount -t lustre $MOUNT_OPTIONS $dev $dir; then
350 # Usage: assemble_md_device <device>
351 # Assemble the md device backing device.
352 # Return 0 if the array is assembled successfully or was already active,
353 # otherwise return error code from mdadm.
354 assemble_md_device ()
361 if [ -n "$raidtab" ] ; then
362 args="$args -c $raidtab"
365 if ! md_array_is_active $dev ; then
374 # Usage: stop_md_device <device>
375 # Stop the md device backing device.
376 # Return 0 if the array is stopped successfully or was not active,
377 # otherwise return error code from mdadm.
385 if [ -n "$raidtab" ] ; then
386 args="$args -c $raidtab"
389 if [ -e $dev ] && md_array_is_active $dev ; then
397 # Usage: md_array_is_active <device>
398 # return 0 if device is an active md RAID array, or 1 otherwise
399 md_array_is_active ()
403 [ -e "$device" ] || return 1
405 mdadm --detail -t $device > /dev/null 2>&1
406 if [ $? -eq 4 ] ; then
412 # Usage: start_services <label> [ <label> ... ]
413 # fsck and mount any devices listed as arguments (in parallel).
414 # Attempt to assemble software raid arrays or zfs pools backing
426 dir=`label_to_mountpt $label`
427 devtype=`$LDEV -t $label`
428 dev=`label_to_device $label`
429 journal=`$LDEV -j $label`
430 raidtab=`$LDEV -r $label`
432 if [ -z "$dir" ] || [ -z "$dev" ]; then
433 echo "$label is not a valid lustre label on this node"
438 if [ "$devtype" = "md" ] ; then
439 if ! assemble_md_device $dev $raidtab ; then
440 echo "failed to assemble array $dev backing $label"
444 elif [ "$devtype" = "zfs" ] ; then
445 if ! import_zpool $label ; then
450 # Journal device field in ldev.conf may be "-" or empty,
451 # so only attempt to assemble if its an absolute path.
452 # Ignore errors since the journal device may not be an
454 if echo $journal | grep -q ^/ ; then
455 assemble_md_device $journal $raidtab 2>/dev/null
458 if mountpt_is_active $label || \
459 device_is_active $label; then
460 echo "$label is already mounted"
465 if [ "x$devtype" != "xzfs" ] ; then
466 if ! mmp_test $dev; then
470 if ! adjust_scsi_timeout $dev; then
475 devices="$devices $dev"
476 labels="$labels $label"
478 if [ $result == 0 ]; then
479 fsck_test $devices || return 2
481 # Fork to handle multiple mount_one_device()'s in parallel.
482 # Errors occurred if $successflag comes up missing afterwards.
484 [ -e $successflag ] || return 2
485 for label in $labels; do
486 mount_one_device $label $successflag `$LDEV -t $label` &
487 # stagger to avoid module loading races
488 if [[ -n $MOUNT_DELAY && $MOUNT_DELAY -gt 0 ]] ; then
492 for label in $labels; do
495 [ -e $successflag ] || return 2
502 # Usage: stop_services <label> [ <label> ... ]
503 # Unmount any devices listed as arguments (serially).
504 # Any devices which are not mounted or don't exist are skipped with no error.
512 for label in $labels; do
513 dir=`label_to_mountpt $label`
514 if [ -z "$dir" ]; then
515 echo "$label is not a valid lustre label on this node"
519 if ! mountpt_is_active $label; then
520 #echo "$label is not mounted"
525 echo "Unmounting $dir"
528 if [ -z "$pids" ]; then
535 # wait for all umount processes to complete, report any errors
537 wait $pid || result=2
541 for label in $labels; do
542 if mountpt_is_active $label; then
543 dir=`label_to_mountpt $label`
544 echo "Mount point $dir is still active"
547 if device_is_active $label; then
548 dev=`label_to_device $label`
549 echo "Device $dev is still active"
558 # Usage: start_lustre_services [local|foreign|all|<label>]
559 # If no parameter is specified, local devices will be started.
560 start_lustre_services ()
571 all) labels="$LOCAL_SRV $FOREIGN_SRV"
576 # for use by heartbeat V1 resource agent:
577 # starting an already-started service must not be an error
578 start_services $labels || exit 2
581 # Usage: stop_lustre_services [local|foreign|all|<label>]
582 # If no parameter is specified all devices will be stopped.
583 stop_lustre_services ()
588 local) labels=$LOCAL_SRV
593 ""|all) labels="$LOCAL_SRV $FOREIGN_SRV"
598 # for use by heartbeat V1 resource agent:
599 # stopping already-stopped service must not be an error
600 stop_services $labels || exit 2
603 # General lustre health check - not device specific.
607 old_nullglob="`shopt -p nullglob`"
611 # LSB compliance - return 3 if service is not running
612 # Lustre-specific returns
613 # 150 - partial startup
614 # 151 - health_check unhealthy
617 egrep -q "libcfs|lvfs|portals" /proc/modules && STATE="loaded"
619 # check for any configured devices (may indicate partial startup)
620 VAR=$(lctl get_param version 2>&1)
622 VAR=$(lctl get_param -n devices 2>&1)
628 # check for either a server or a client filesystem
634 ! lctl get_param -n mgs.MGS.* >/dev/null 2>&1 || MGT="YES"
636 VAR=$(lctl get_param -n mdt.*.recovery_status 2>&1 | grep '^status:' )
641 VAR=$(lctl get_param -n obdfilter.*.recovery_status 2>&1 | grep '^status:')
646 VAR=$(lctl get_param -n llite.fs* 2>&1)
651 if [ "$MGT" -o "$MDT" -o "$OST" -o "$LLITE" ]; then
656 # check if this is a router
657 if [[ "$(lctl get_param -n routes)" =~ "Routing enabled" ]]; then
663 # check for server disconnections
664 VAR=$(lctl get_param -n *c.*.*server_uuid 2>&1)
666 DISCON="$(echo $VAR | grep -v FULL)"
667 if [ -n "$DISCON" ] ; then
673 # check for servers in recovery
674 if [ -n "$MDT$OST" ] && echo $MDT $OST | grep -q RECOV ; then
679 # check for error in health_check
680 local health_check=$(lctl get_param -n health_check)
681 if [[ "$health_check" =~ "NOT HEALTHY" ]]; then
687 if [[ "$health_check" =~ "LBUG" ]]; then
697 # Usage: status [local|foreign|all|<label>]
698 # If no parameter is specified, general lustre health status will be reported.
706 local) labels=$LOCAL_SRV;
711 all) labels="$LOCAL_SRV $FOREIGN_SRV"
713 "") # ASSUMPTION: this is not the heartbeat res agent
720 # for use by heartbeat V1 resource agent:
721 # print "running" if *anything* is running.
722 for label in $labels; do
723 dir=`label_to_device $label`
724 if [ -z "$dir" ]; then
725 echo "$label is not a valid lustre label on this node"
730 if mountpt_is_active $label || device_is_active $label; then
735 [ $valid_devs == 1 ] && echo "stopped"
742 Usage: lustre {start|stop|status|restart|reload|condrestart}
744 lustre start [local|foreign|<label>]
745 lustre stop [local|foreign|<label>]
746 lustre status [local|foreign|<label>]
751 # See how we were called.
754 if [ $# -gt 2 ] ; then
755 echo "ERROR: Too many arguments."
758 run_preexec_check "start"
759 start_lustre_services $2
760 run_postexec_check "start"
763 if [ $# -gt 2 ] ; then
764 echo "ERROR: Too many arguments."
767 run_preexec_check "stop"
768 stop_lustre_services $2
769 run_postexec_check "stop"
772 if [ $# -gt 2 ] ; then
773 echo "ERROR: Too many arguments."
787 if grep lustre /proc/mounts ; then