3 # lustre This shell script takes care of starting and stopping
7 # description: Part of the lustre file system.
9 # config: /etc/sysconfig/lustre
11 PATH=/sbin:/usr/sbin:/bin:/usr/bin
13 # Source function library.
14 . /etc/rc.d/init.d/functions
16 # Source networking configuration.
17 if [ ! -f /etc/sysconfig/network ]; then
21 . /etc/sysconfig/network
23 LDEV=${LDEV:-"/usr/sbin/ldev"}
24 ZPOOL_LAYOUT=/usr/bin/zpool_layout
25 UDEVADM=${UDEVADM:-/sbin/udevadm}
27 # Check that networking is up.
28 [ "${NETWORKING}" = "no" ] && exit 0
30 # Check for and source configuration file otherwise set defaults
31 [ -f /etc/sysconfig/lustre ] && . /etc/sysconfig/lustre
32 FSCK_ARGS=${FSCK_ARGS:-""}
33 MOUNT_OPTIONS=${MOUNT_OPTIONS:-""}
34 LOCAL_SRV=${LOCAL_SRV:-"`$LDEV -l 2>/dev/null`"}
35 FOREIGN_SRV=${FOREIGN_SRV:-"`$LDEV -f 2>/dev/null`"}
36 REQUIRE_MMP_FEATURE=${REQUIRE_MMP_FEATURE:-${FOREIGN_SRV:+"yes"}}
37 LOCAL_MOUNT_DIR=${LOCAL_MOUNT_DIR:-"/mnt/lustre/local"}
38 FOREIGN_MOUNT_DIR=${FOREIGN_MOUNT_DIR:-"/mnt/lustre/foreign"}
39 SETUP_DEVICES=${SETUP_DEVICES:-""}
40 ZPOOL_LAYOUT_BUSES=${ZPOOL_LAYOUT_BUSES:-""}
41 ZPOOL_LAYOUT_PORTS=${ZPOOL_LAYOUT_PORTS:-""}
42 ZPOOL_LAYOUT_MAP=${ZPOOL_LAYOUT_MAP:-""}
43 MOUNT_DELAY=${MOUNT_DELAY:-2}
44 LOAD_ZFS=${LOAD_ZFS:-""}
46 if [ -z "$TUNE2FS" ] ; then
47 TUNE2FS=`which tunefs.ldiskfs 2>/dev/null`
48 if [ -z "$TUNE2FS" ] ; then
49 TUNE2FS=`which tune2fs 2>/dev/null`
53 if [ -z "$PFSCK" ] ; then
54 PFSCK=`which pfsck.ldiskfs 2>/dev/null`
55 if [ -z "$PFSCK" ] ; then
56 PFSCK=`which fsck 2>/dev/null`
64 if [ -n "$ZPOOL_LAYOUT_BUSES" -a -n "$ZPOOL_LAYOUT_PORTS" ] ; then
65 MAP_ARG=${ZPOOL_LAYOUT_MAP:+"-m $ZPOOL_LAYOUT_MAP"}
66 $ZPOOL_LAYOUT -t -b "$ZPOOL_LAYOUT_BUSES" \
67 -p "$ZPOOL_LAYOUT_PORTS" $MAP_ARG
69 if [ "$LOAD_ZFS" = "yes" ] && ! modprobe zfs ; then
70 echo "Failed to load zfs module. Aborting."
80 for label in $labels; do
81 devtype=`$LDEV -t $label`
82 if [ "$devtype" = "zfs" ] ; then
84 elif [ "$devtype" = "md" ] ; then
85 dev=`label_to_device $label`
86 journal=`$LDEV -j $label`
88 stop_md_device $journal
97 local pool=`$LDEV -z $label`
98 local args="-N $ZPOOL_IMPORT_ARGS"
99 local cache=`$LDEV -r $label`
100 # -c is incompatible with -d
101 if [ -n "$cache" ] ; then
102 args="$args -c $cache"
103 elif [ -n "$ZPOOL_IMPORT_DIR" ] ; then
104 args="$args -d $ZPOOL_IMPORT_DIR"
105 elif [ -d "/dev/disk/by-vdev" ] ; then
106 args="$args -d /dev/disk/by-vdev"
107 elif [ -d "/dev/mapper" ] ; then
108 args="$args -d /dev/mapper"
111 if zpool status $pool >/dev/null 2>&1 ; then
113 elif [ -n "$pool" ] ; then
114 zpool import $pool $args 2>/dev/null
123 local pool=`$LDEV -z $label`
124 zpool export $pool 2>/dev/null
127 # Trigger udev and wait for it to settle.
130 if [ -x ${UDEVADM} ]; then
131 ${UDEVADM} trigger --action=change --subsystem-match=block
139 # Usage: run_preexec_check [ start | restart | condrestart ]
140 # The single parameter will be passed to the PREEXEC_SCRIPT
143 if [ -n "$PREEXEC_CHECK" ] && ! $PREEXEC_CHECK ; then
144 echo "Pre-exec check \"$PREEXEC_CHECK\" failed. Aborting."
148 if [ -n "$PREEXEC_SCRIPT" ] && ! "$PREEXEC_SCRIPT" "$1" ; then
149 echo "Pre-exec script \"$PREEXEC_SCRIPT\" failed. Aborting."
154 # Usage: run_postexec_check [ start | restart | condrestart ]
155 # The single parameter will be passed to the PREEXEC_SCRIPT
156 run_postexec_check ()
158 if [ -n "$POSTEXEC_CHECK" ] && ! $POSTEXEC_CHECK ; then
159 echo "Post-exec check \"$POSTEXEC_CHECK\" failed. Aborting."
163 if [ -n "$POSTEXEC_SCRIPT" ] && ! "$POSTEXEC_SCRIPT" "$1" ; then
164 echo "Post-exec script \"$POSTEXEC_SCRIPT\" failed. Aborting."
169 # Usage: adjust_scsi_timeout <dev>
170 adjust_scsi_timeout ()
174 if [ -n "$SCSI_DEVICE_TIMEOUT" ]; then
175 # make sure that it is actually a SCSI (sd) device
176 local name=`basename $dev`
177 local proc=/sys/block/${name}/device/timeout
178 local driver=`readlink /sys/block/${name}/device/driver`
179 if [ -n "$driver" ] && [ "`basename $driver`" == "sd" ]; then
180 if ! echo $SCSI_DEVICE_TIMEOUT >$proc; then
181 echo "FAILED: could not adjust ${dev} timeout"
189 # Usage: fsck_test <dev> [ <dev> ... ]
190 # Checks all devices in parallel if FSCK_ARGS is set.
195 # Filter out non-absolute paths, which are probably ZFS datasets
196 devices=`echo $devices |xargs -n 1|grep '^/'|xargs`
198 if [ -n "${FSCK_ARGS}" -a -n "$devices" ]; then
199 if [ -x $PFSCK ] ; then
200 echo "$PFSCK $devices -- ${FSCK_ARGS}"
201 $PFSCK $devices -- ${FSCK_ARGS}
202 if [ $? -ne 0 -a $? -ne 1 ] ; then
203 echo "FAILED: $PFSCK -- ${FSCK_ARGS}: $?"
207 echo "$PFSCK not found"
214 # Usage: test_feature_flag <dev> <flag>
222 for feature in `$TUNE2FS -l $dev 2>/dev/null \
223 | grep features: | sed -e 's/^.*: //'`; do
224 if [ "$feature" == "$flag" ]; then
233 # Usage: mmp_test <dev>
234 # Returns 0 if it is set or not required, 1 if unset and required or error.
240 if [ "$REQUIRE_MMP_FEATURE" == "yes" ]; then
241 if [ -x $TUNE2FS ]; then
242 if ! test_feature_flag $dev "mmp"; then
243 echo "mmp feature flag is not set on $dev"
247 echo "$TUNE2FS not found"
255 # Usage: label_to_mountpt <label>
256 # Prints mount point path, if label matches a local or foreign server.
262 for serv in $LOCAL_SRV; do
263 if [ "$serv" == "$label" ]; then
264 echo "$LOCAL_MOUNT_DIR/$label"
268 for serv in $FOREIGN_SRV; do
269 if [ "$serv" == "$label" ]; then
270 echo "$FOREIGN_MOUNT_DIR/$label"
276 # Usage: label_to_device <label>
277 # Prints canonical device path.
281 local path=/dev/disk/by-label/$label
283 if [ -h $path ] ; then
284 readlink --canonicalize $path
290 # helper for mountpt_is_active() and device_is_active()
291 declare -r awkprog='BEGIN {rc = 1;}
292 { if ($field == path) {rc = 0;} }
295 # Usage: mountpt_is_active <label>
296 # Return 1 (inactive) on invalid label.
299 local dir=`label_to_mountpt $1`
302 if [ -n "$dir" ]; then
303 cat /proc/mounts | awk "$awkprog" field=2 path=$dir
309 # Usage: device_is_active <label>
310 # Return 1 (inactive) on invalid label.
313 local dev=`label_to_device $1`
316 if [ -n "$dev" ]; then
317 cat /proc/mounts | awk "$awkprog" field=1 path=$dir
323 # Usage: mount_one_device <label> <successflag> [devtype]
324 # Remove <succesflag> on error (trick to detect errors after parallel runs).
330 local dev=`label_to_device $label`
331 local dir=`label_to_mountpt $label`
333 # $dir and $dev have already been checked at ths point
334 if [ ! -d $dir ] && ! mkdir -p $dir; then
338 echo "Mounting $dev on $dir"
339 if ! mount -t lustre $MOUNT_OPTIONS $dev $dir; then
345 # Usage: assemble_md_device <device>
346 # Assemble the md device backing device.
347 # Return 0 if the array is assembled successfully or was already active,
348 # otherwise return error code from mdadm.
349 assemble_md_device ()
356 if [ -n "$raidtab" ] ; then
357 args="$args -c $raidtab"
360 if ! md_array_is_active $dev ; then
369 # Usage: stop_md_device <device>
370 # Stop the md device backing device.
371 # Return 0 if the array is stopped successfully or was not active,
372 # otherwise return error code from mdadm.
380 if [ -n "$raidtab" ] ; then
381 args="$args -c $raidtab"
384 if [ -e $dev ] && md_array_is_active $dev ; then
392 # Usage: md_array_is_active <device>
393 # return 0 if device is an active md RAID array, or 1 otherwise
394 md_array_is_active ()
398 [ -e "$device" ] || return 1
400 mdadm --detail -t $device > /dev/null 2>&1
401 if [ $? -eq 4 ] ; then
407 # Usage: start_services <label> [ <label> ... ]
408 # fsck and mount any devices listed as arguments (in parallel).
409 # Attempt to assemble software raid arrays or zfs pools backing
421 dir=`label_to_mountpt $label`
422 devtype=`$LDEV -t $label`
423 dev=`label_to_device $label`
424 journal=`$LDEV -j $label`
425 raidtab=`$LDEV -r $label`
427 if [ -z "$dir" ] || [ -z "$dev" ]; then
428 echo "$label is not a valid lustre label on this node"
433 if [ "$devtype" = "md" ] ; then
434 if ! assemble_md_device $dev $raidtab ; then
435 echo "failed to assemble array $dev backing $label"
439 elif [ "$devtype" = "zfs" ] ; then
440 if ! import_zpool $label ; then
445 # Journal device field in ldev.conf may be "-" or empty,
446 # so only attempt to assemble if its an absolute path.
447 # Ignore errors since the journal device may not be an
449 if echo $journal | grep -q ^/ ; then
450 assemble_md_device $journal $raidtab 2>/dev/null
453 if [ "x$devtype" != "xzfs" ] ; then
454 if mountpt_is_active $label || \
455 device_is_active $label; then
456 echo "$label is already mounted"
460 if ! mmp_test $dev; then
464 if ! adjust_scsi_timeout $dev; then
469 devices="$devices $dev"
470 labels="$labels $label"
472 if [ $result == 0 ]; then
473 fsck_test $devices || return 2
475 # Fork to handle multiple mount_one_device()'s in parallel.
476 # Errors occurred if $successflag comes up missing afterwards.
478 [ -e $successflag ] || return 2
479 for label in $labels; do
480 mount_one_device $label $successflag `$LDEV -t $label` &
481 # stagger to avoid module loading races
482 if [[ -n $MOUNT_DELAY && $MOUNT_DELAY -gt 0 ]] ; then
486 for label in $labels; do
489 [ -e $successflag ] || return 2
496 # Usage: stop_services <label> [ <label> ... ]
497 # Unmount any devices listed as arguments (serially).
498 # Any devices which are not mounted or don't exist are skipped with no error.
505 for label in $labels; do
506 dir=`label_to_mountpt $label`
507 if [ -z "$dir" ]; then
508 echo "$label is not a valid lustre label on this node"
512 if ! mountpt_is_active $label; then
513 #echo "$label is not mounted"
517 echo "Unmounting $dir"
518 umount $dir || result=2
521 for label in $labels; do
522 if mountpt_is_active $label; then
523 dir=`label_to_mountpt $label`
524 echo "Mount point $dir is still active"
527 if device_is_active $label; then
528 dev=`label_to_device $label`
529 echo "Device $dev is still active"
538 # Usage: start_lustre_services [local|foreign|all|<label>]
539 # If no parameter is specified, local devices will be started.
540 start_lustre_services ()
551 all) labels="$LOCAL_SRV $FOREIGN_SRV"
556 # for use by heartbeat V1 resource agent:
557 # starting an already-started service must not be an error
558 start_services $labels || exit 2
561 # Usage: stop_lustre_services [local|foreign|all|<label>]
562 # If no parameter is specified all devices will be stopped.
563 stop_lustre_services ()
568 local) labels=$LOCAL_SRV
573 ""|all) labels="$LOCAL_SRV $FOREIGN_SRV"
578 # for use by heartbeat V1 resource agent:
579 # stopping already-stopped service must not be an error
580 stop_services $labels || exit 2
583 # General lustre health check - not device specific.
586 old_nullglob="`shopt -p nullglob`"
590 # LSB compliance - return 3 if service is not running
591 # Lustre-specific returns
592 # 150 - partial startup
593 # 151 - health_check unhealthy
596 egrep -q "libcfs|lvfs|portals" /proc/modules && STATE="loaded"
598 # check for any configured devices (may indicate partial startup)
599 if [ -d /proc/fs/lustre ]; then
600 if [ -n "`cat /proc/fs/lustre/devices 2> /dev/null`" ] ; then
605 # check for either a server or a client filesystem
606 MDT="`ls /proc/fs/lustre/mdt/*/recovery_status 2> /dev/null`"
607 OST="`ls /proc/fs/lustre/obdfilter/*/recovery_status \
609 LLITE="`ls /proc/fs/lustre/llite/fs* 2> /dev/null`"
610 if [ "$MDT" -o "$OST" -o "$LLITE" ]; then
615 # check if this is a router
616 if [ -d /proc/sys/lnet ]; then
617 ROUTER="`cat /proc/sys/lnet/routes | head -1 |
618 grep -i -c \"Routing enabled\"`"
619 if [[ ! -z ${ROUTER} && ${ROUTER} -ge 1 ]]; then
626 # check for server disconnections
627 DISCON="`grep -v FULL /proc/fs/lustre/*c/*/*server_uuid 2> /dev/null`"
628 if [ -n "$DISCON" ] ; then
633 # check for servers in recovery
634 if [ -n "$MDT$OST" ] && grep -q RECOV $MDT $OST ; then
639 # check for error in health_check
640 HEALTH="/proc/fs/lustre/health_check"
641 if [ -f "$HEALTH" ] && grep -q "NOT HEALTHY" $HEALTH ; then
647 if [ -f "$HEALTH" ] && grep -q "LBUG" $HEALTH ; then
657 # Usage: status [local|foreign|all|<label>]
658 # If no parameter is specified, general lustre health status will be reported.
666 local) labels=$LOCAL_SRV;
671 all) labels="$LOCAL_SRV $FOREIGN_SRV"
673 "") # ASSUMPTION: this is not the heartbeat res agent
680 # for use by heartbeat V1 resource agent:
681 # print "running" if *anything* is running.
682 for label in $labels; do
683 dir=`label_to_device $label`
684 if [ -z "$dir" ]; then
685 echo "$label is not a valid lustre label on this node"
690 if mountpt_is_active $label || device_is_active $label; then
695 [ $valid_devs == 1 ] && echo "stopped"
702 Usage: lustre {start|stop|status|restart|reload|condrestart}
704 lustre start [local|foreign|<label>]
705 lustre stop [local|foreign|<label>]
706 lustre status [local|foreign|<label>]
711 # See how we were called.
714 if [ $# -gt 2 ] ; then
715 echo "ERROR: Too many arguments."
718 run_preexec_check "start"
719 start_lustre_services $2
720 run_postexec_check "start"
723 if [ $# -gt 2 ] ; then
724 echo "ERROR: Too many arguments."
727 run_preexec_check "stop"
728 stop_lustre_services $2
729 run_postexec_check "stop"
732 if [ $# -gt 2 ] ; then
733 echo "ERROR: Too many arguments."
747 if grep lustre /proc/mounts ; then