3 # lustre This shell script takes care of starting and stopping
7 # description: Part of the lustre file system.
9 # config: /etc/sysconfig/lustre
12 # Required-Start: lnet
15 # Description: Lustre init script
18 PATH=/sbin:/usr/sbin:/bin:/usr/bin
20 # Source function library.
21 . /etc/rc.d/init.d/functions
23 # Source networking configuration.
24 if [ ! -f /etc/sysconfig/network ]; then
28 . /etc/sysconfig/network
30 LDEV=${LDEV:-"/usr/sbin/ldev"}
31 ZPOOL_LAYOUT=/usr/bin/zpool_layout
32 UDEVADM=${UDEVADM:-/sbin/udevadm}
34 # Check that networking is up.
35 [ "${NETWORKING}" = "no" ] && exit 0
37 # Check for and source configuration file otherwise set defaults
38 [ -f /etc/sysconfig/lustre ] && . /etc/sysconfig/lustre
39 FSCK_ARGS=${FSCK_ARGS:-""}
40 MOUNT_OPTIONS=${MOUNT_OPTIONS:-""}
41 LOCAL_SRV=${LOCAL_SRV:-"`$LDEV -l 2>/dev/null`"}
42 FOREIGN_SRV=${FOREIGN_SRV:-"`$LDEV -f 2>/dev/null`"}
43 REQUIRE_MMP_FEATURE=${REQUIRE_MMP_FEATURE:-${FOREIGN_SRV:+"yes"}}
44 LOCAL_MOUNT_DIR=${LOCAL_MOUNT_DIR:-"/mnt/lustre/local"}
45 FOREIGN_MOUNT_DIR=${FOREIGN_MOUNT_DIR:-"/mnt/lustre/foreign"}
46 SETUP_DEVICES=${SETUP_DEVICES:-""}
47 ZPOOL_LAYOUT_BUSES=${ZPOOL_LAYOUT_BUSES:-""}
48 ZPOOL_LAYOUT_PORTS=${ZPOOL_LAYOUT_PORTS:-""}
49 ZPOOL_LAYOUT_MAP=${ZPOOL_LAYOUT_MAP:-""}
50 MOUNT_DELAY=${MOUNT_DELAY:-2}
51 LOAD_ZFS=${LOAD_ZFS:-""}
53 if [ -z "$TUNE2FS" ] ; then
54 TUNE2FS=`which tunefs.ldiskfs 2>/dev/null`
55 if [ -z "$TUNE2FS" ] ; then
56 TUNE2FS=`which tune2fs 2>/dev/null`
60 if [ -z "$PFSCK" ] ; then
61 PFSCK=`which pfsck.ldiskfs 2>/dev/null`
62 if [ -z "$PFSCK" ] ; then
63 PFSCK=`which fsck 2>/dev/null`
71 if [ -n "$ZPOOL_LAYOUT_BUSES" -a -n "$ZPOOL_LAYOUT_PORTS" ] ; then
72 MAP_ARG=${ZPOOL_LAYOUT_MAP:+"-m $ZPOOL_LAYOUT_MAP"}
73 $ZPOOL_LAYOUT -t -b "$ZPOOL_LAYOUT_BUSES" \
74 -p "$ZPOOL_LAYOUT_PORTS" $MAP_ARG
76 if [ "$LOAD_ZFS" = "yes" ] && ! modprobe zfs ; then
77 echo "Failed to load zfs module. Aborting."
86 for label in $labels; do
87 devtype=`$LDEV -t $label`
88 if [ "$devtype" = "zfs" ] ; then
90 elif [ "$devtype" = "md" ] ; then
91 dev=`label_to_device $label`
92 journal=`$LDEV -j $label`
94 stop_md_device $journal
103 local pool=`$LDEV -z $label`
104 local args="-N $ZPOOL_IMPORT_ARGS"
105 local cache=`$LDEV -r $label`
106 # -c is incompatible with -d
107 if [ -n "$cache" ] ; then
108 args="$args -c $cache"
109 elif [ -n "$ZPOOL_IMPORT_DIR" ] ; then
110 args="$args -d $ZPOOL_IMPORT_DIR"
113 if zpool status $pool >/dev/null 2>&1 ; then
115 elif [ -n "$pool" ] ; then
116 zpool import $pool $args 2>/dev/null
119 if [ $result -ne 0 ] ; then
120 echo "Unexpected return code from import of pool $pool: $result"
128 local pool=`$LDEV -z $label`
129 zpool export $pool 2>/dev/null
132 # Trigger udev and wait for it to settle.
135 if [ -x ${UDEVADM} ]; then
136 ${UDEVADM} trigger --action=change --subsystem-match=block
144 # Usage: run_preexec_check [ start | restart | condrestart ]
145 # The single parameter will be passed to the PREEXEC_SCRIPT
148 if [ -n "$PREEXEC_CHECK" ] && ! $PREEXEC_CHECK ; then
149 echo "Pre-exec check \"$PREEXEC_CHECK\" failed. Aborting."
153 if [ -n "$PREEXEC_SCRIPT" ] && ! "$PREEXEC_SCRIPT" "$1" ; then
154 echo "Pre-exec script \"$PREEXEC_SCRIPT\" failed. Aborting."
159 # Usage: run_postexec_check [ start | restart | condrestart ]
160 # The single parameter will be passed to the PREEXEC_SCRIPT
161 run_postexec_check ()
163 if [ -n "$POSTEXEC_CHECK" ] && ! $POSTEXEC_CHECK ; then
164 echo "Post-exec check \"$POSTEXEC_CHECK\" failed. Aborting."
168 if [ -n "$POSTEXEC_SCRIPT" ] && ! "$POSTEXEC_SCRIPT" "$1" ; then
169 echo "Post-exec script \"$POSTEXEC_SCRIPT\" failed. Aborting."
174 # Usage: adjust_scsi_timeout <dev>
175 adjust_scsi_timeout ()
179 if [ -n "$SCSI_DEVICE_TIMEOUT" ]; then
180 # make sure that it is actually a SCSI (sd) device
181 local name=`basename $dev`
182 local proc=/sys/block/${name}/device/timeout
183 local driver=`readlink /sys/block/${name}/device/driver`
184 if [ -n "$driver" ] && [ "`basename $driver`" == "sd" ]; then
185 if ! echo $SCSI_DEVICE_TIMEOUT >$proc; then
186 echo "FAILED: could not adjust ${dev} timeout"
194 # Usage: fsck_test <dev> [ <dev> ... ]
195 # Checks all devices in parallel if FSCK_ARGS is set.
200 # Filter out non-absolute paths, which are probably ZFS datasets
201 devices=`echo $devices |xargs -n 1|grep '^/'|xargs`
203 if [ -n "${FSCK_ARGS}" -a -n "$devices" ]; then
204 if [ -x $PFSCK ] ; then
205 echo "$PFSCK $devices -- ${FSCK_ARGS}"
206 $PFSCK $devices -- ${FSCK_ARGS}
207 if [ $? -ne 0 -a $? -ne 1 ] ; then
208 echo "FAILED: $PFSCK -- ${FSCK_ARGS}: $?"
212 echo "$PFSCK not found"
219 # Usage: test_feature_flag <dev> <flag>
227 for feature in `$TUNE2FS -l $dev 2>/dev/null \
228 | grep features: | sed -e 's/^.*: //'`; do
229 if [ "$feature" == "$flag" ]; then
238 # Usage: mmp_test <dev>
239 # Returns 0 if it is set or not required, 1 if unset and required or error.
245 if [ "$REQUIRE_MMP_FEATURE" == "yes" ]; then
246 if [ -x $TUNE2FS ]; then
247 if ! test_feature_flag $dev "mmp"; then
248 echo "mmp feature flag is not set on $dev"
252 echo "$TUNE2FS not found"
260 # Usage: label_to_mountpt <label>
261 # Prints mount point path, if label matches a local or foreign server.
267 for serv in $LOCAL_SRV; do
268 if [ "$serv" == "$label" ]; then
269 echo "$LOCAL_MOUNT_DIR/$label"
273 for serv in $FOREIGN_SRV; do
274 if [ "$serv" == "$label" ]; then
275 echo "$FOREIGN_MOUNT_DIR/$label"
281 # Usage: label_to_device <label>
282 # Prints canonical device path.
286 local path=/dev/disk/by-label/$label
288 if [ -h $path ] ; then
289 readlink --canonicalize $path
295 # helper for mountpt_is_active() and device_is_active()
296 declare -r awkprog='BEGIN {rc = 1;}
297 { if ($field == path) {rc = 0;} }
300 # Usage: mountpt_is_active <label>
301 # Return 1 (inactive) on invalid label.
304 local dir=`label_to_mountpt $1`
307 if [ -n "$dir" ]; then
308 cat /proc/mounts | awk "$awkprog" field=2 path=$dir
314 # Usage: device_is_active <label>
315 # Return 1 (inactive) on invalid label.
318 local dev=`label_to_device $1`
321 if [ -n "$dev" ]; then
322 cat /proc/mounts | awk "$awkprog" field=1 path=$dir
328 # Usage: mount_one_device <label> <successflag> [devtype]
329 # Remove <successflag> on error (trick to detect errors after parallel runs).
335 local dev=`label_to_device $label`
336 local dir=`label_to_mountpt $label`
338 # $dir and $dev have already been checked at ths point
339 if [ ! -d $dir ] && ! mkdir -p $dir; then
343 echo "Mounting $dev on $dir"
344 if ! mount -t lustre $MOUNT_OPTIONS $dev $dir; then
350 # Usage: assemble_md_device <device>
351 # Assemble the md device backing device.
352 # Return 0 if the array is assembled successfully or was already active,
353 # otherwise return error code from mdadm.
354 assemble_md_device ()
361 if [ -n "$raidtab" ] ; then
362 args="$args -c $raidtab"
365 if ! md_array_is_active $dev ; then
374 # Usage: stop_md_device <device>
375 # Stop the md device backing device.
376 # Return 0 if the array is stopped successfully or was not active,
377 # otherwise return error code from mdadm.
385 if [ -n "$raidtab" ] ; then
386 args="$args -c $raidtab"
389 if [ -e $dev ] && md_array_is_active $dev ; then
397 # Usage: md_array_is_active <device>
398 # return 0 if device is an active md RAID array, or 1 otherwise
399 md_array_is_active ()
403 [ -e "$device" ] || return 1
405 mdadm --detail -t $device > /dev/null 2>&1
406 if [ $? -eq 4 ] ; then
412 # Usage: start_services <label> [ <label> ... ]
413 # fsck and mount any devices listed as arguments (in parallel).
414 # Attempt to assemble software raid arrays or zfs pools backing
426 dir=`label_to_mountpt $label`
427 devtype=`$LDEV -t $label`
428 dev=`label_to_device $label`
429 journal=`$LDEV -j $label`
430 raidtab=`$LDEV -r $label`
432 if [ -z "$dir" ] || [ -z "$dev" ]; then
433 echo "$label is not a valid lustre label on this node"
438 if [ "$devtype" = "md" ] ; then
439 if ! assemble_md_device $dev $raidtab ; then
440 echo "failed to assemble array $dev backing $label"
444 elif [ "$devtype" = "zfs" ] ; then
445 if ! import_zpool $label ; then
450 # Journal device field in ldev.conf may be "-" or empty,
451 # so only attempt to assemble if its an absolute path.
452 # Ignore errors since the journal device may not be an
454 if echo $journal | grep -q ^/ ; then
455 assemble_md_device $journal $raidtab 2>/dev/null
458 if [ "x$devtype" != "xzfs" ] ; then
459 if mountpt_is_active $label || \
460 device_is_active $label; then
461 echo "$label is already mounted"
465 if ! mmp_test $dev; then
469 if ! adjust_scsi_timeout $dev; then
474 devices="$devices $dev"
475 labels="$labels $label"
477 if [ $result == 0 ]; then
478 fsck_test $devices || return 2
480 # Fork to handle multiple mount_one_device()'s in parallel.
481 # Errors occurred if $successflag comes up missing afterwards.
483 [ -e $successflag ] || return 2
484 for label in $labels; do
485 mount_one_device $label $successflag `$LDEV -t $label` &
486 # stagger to avoid module loading races
487 if [[ -n $MOUNT_DELAY && $MOUNT_DELAY -gt 0 ]] ; then
491 for label in $labels; do
494 [ -e $successflag ] || return 2
501 # Usage: stop_services <label> [ <label> ... ]
502 # Unmount any devices listed as arguments (serially).
503 # Any devices which are not mounted or don't exist are skipped with no error.
511 for label in $labels; do
512 dir=`label_to_mountpt $label`
513 if [ -z "$dir" ]; then
514 echo "$label is not a valid lustre label on this node"
518 if ! mountpt_is_active $label; then
519 #echo "$label is not mounted"
524 echo "Unmounting $dir"
527 if [ -z "$pids" ]; then
534 # wait for all umount processes to complete, report any errors
536 wait $pid || result=2
540 for label in $labels; do
541 if mountpt_is_active $label; then
542 dir=`label_to_mountpt $label`
543 echo "Mount point $dir is still active"
546 if device_is_active $label; then
547 dev=`label_to_device $label`
548 echo "Device $dev is still active"
557 # Usage: start_lustre_services [local|foreign|all|<label>]
558 # If no parameter is specified, local devices will be started.
559 start_lustre_services ()
570 all) labels="$LOCAL_SRV $FOREIGN_SRV"
575 # for use by heartbeat V1 resource agent:
576 # starting an already-started service must not be an error
577 start_services $labels || exit 2
580 # Usage: stop_lustre_services [local|foreign|all|<label>]
581 # If no parameter is specified all devices will be stopped.
582 stop_lustre_services ()
587 local) labels=$LOCAL_SRV
592 ""|all) labels="$LOCAL_SRV $FOREIGN_SRV"
597 # for use by heartbeat V1 resource agent:
598 # stopping already-stopped service must not be an error
599 stop_services $labels || exit 2
602 # General lustre health check - not device specific.
606 old_nullglob="`shopt -p nullglob`"
610 # LSB compliance - return 3 if service is not running
611 # Lustre-specific returns
612 # 150 - partial startup
613 # 151 - health_check unhealthy
616 egrep -q "libcfs|lvfs|portals" /proc/modules && STATE="loaded"
618 # check for any configured devices (may indicate partial startup)
619 VAR=$(lctl get_param version 2>&1)
621 VAR=$(lctl get_param -n devices 2>&1)
627 # check for either a server or a client filesystem
633 ! lctl get_param -n mgs.MGS.* >/dev/null 2>&1 || MGT="YES"
635 VAR=$(lctl get_param -n mdt.*.recovery_status 2>&1 | grep '^status:' )
640 VAR=$(lctl get_param -n obdfilter.*.recovery_status 2>&1 | grep '^status:')
645 VAR=$(lctl get_param -n llite.fs* 2>&1)
650 if [ "$MGT" -o "$MDT" -o "$OST" -o "$LLITE" ]; then
655 # check if this is a router
656 if [[ "$(lctl get_param -n routes)" =~ "Routing enabled" ]]; then
662 # check for server disconnections
663 VAR=$(lctl get_param -n *c.*.*server_uuid 2>&1)
665 DISCON="$(echo $VAR | grep -v FULL)"
666 if [ -n "$DISCON" ] ; then
672 # check for servers in recovery
673 if [ -n "$MDT$OST" ] && echo $MDT $OST | grep -q RECOV ; then
678 # check for error in health_check
679 local health_check=$(lctl get_param -n health_check)
680 if [[ "$health_check" =~ "NOT HEALTHY" ]]; then
686 if [[ "$health_check" =~ "LBUG" ]]; then
696 # Usage: status [local|foreign|all|<label>]
697 # If no parameter is specified, general lustre health status will be reported.
705 local) labels=$LOCAL_SRV;
710 all) labels="$LOCAL_SRV $FOREIGN_SRV"
712 "") # ASSUMPTION: this is not the heartbeat res agent
719 # for use by heartbeat V1 resource agent:
720 # print "running" if *anything* is running.
721 for label in $labels; do
722 dir=`label_to_device $label`
723 if [ -z "$dir" ]; then
724 echo "$label is not a valid lustre label on this node"
729 if mountpt_is_active $label || device_is_active $label; then
734 [ $valid_devs == 1 ] && echo "stopped"
741 Usage: lustre {start|stop|status|restart|reload|condrestart}
743 lustre start [local|foreign|<label>]
744 lustre stop [local|foreign|<label>]
745 lustre status [local|foreign|<label>]
750 # See how we were called.
753 if [ $# -gt 2 ] ; then
754 echo "ERROR: Too many arguments."
757 run_preexec_check "start"
758 start_lustre_services $2
759 run_postexec_check "start"
762 if [ $# -gt 2 ] ; then
763 echo "ERROR: Too many arguments."
766 run_preexec_check "stop"
767 stop_lustre_services $2
768 run_postexec_check "stop"
771 if [ $# -gt 2 ] ; then
772 echo "ERROR: Too many arguments."
786 if grep lustre /proc/mounts ; then