3 # lustre This shell script takes care of starting and stopping
7 # description: Part of the lustre file system.
9 # config: /etc/sysconfig/lustre
11 PATH=/sbin:/usr/sbin:/bin:/usr/bin
13 # Source function library.
14 . /etc/rc.d/init.d/functions
16 # Source networking configuration.
17 if [ ! -f /etc/sysconfig/network ]; then
21 . /etc/sysconfig/network
23 LDEV=${LDEV:-"/usr/sbin/ldev"}
24 ZPOOL_LAYOUT=/usr/bin/zpool_layout
25 UDEVADM=${UDEVADM:-/sbin/udevadm}
27 # Check that networking is up.
28 [ "${NETWORKING}" = "no" ] && exit 0
30 # Check for and source configuration file otherwise set defaults
31 [ -f /etc/sysconfig/lustre ] && . /etc/sysconfig/lustre
32 FSCK_ARGS=${FSCK_ARGS:-""}
33 MOUNT_OPTIONS=${MOUNT_OPTIONS:-""}
34 LOCAL_SRV=${LOCAL_SRV:-"`$LDEV -l 2>/dev/null`"}
35 FOREIGN_SRV=${FOREIGN_SRV:-"`$LDEV -f 2>/dev/null`"}
36 REQUIRE_MMP_FEATURE=${REQUIRE_MMP_FEATURE:-${FOREIGN_SRV:+"yes"}}
37 LOCAL_MOUNT_DIR=${LOCAL_MOUNT_DIR:-"/mnt/lustre/local"}
38 FOREIGN_MOUNT_DIR=${FOREIGN_MOUNT_DIR:-"/mnt/lustre/foreign"}
39 SETUP_DEVICES=${SETUP_DEVICES:-""}
40 ZPOOL_LAYOUT_BUSES=${ZPOOL_LAYOUT_BUSES:-""}
41 ZPOOL_LAYOUT_PORTS=${ZPOOL_LAYOUT_PORTS:-""}
42 ZPOOL_LAYOUT_MAP=${ZPOOL_LAYOUT_MAP:-""}
43 MOUNT_DELAY=${MOUNT_DELAY:-2}
44 LOAD_ZFS=${LOAD_ZFS:-""}
46 if [ -z "$TUNE2FS" ] ; then
47 TUNE2FS=`which tunefs.ldiskfs 2>/dev/null`
48 if [ -z "$TUNE2FS" ] ; then
49 TUNE2FS=`which tune2fs 2>/dev/null`
53 if [ -z "$PFSCK" ] ; then
54 PFSCK=`which pfsck.ldiskfs 2>/dev/null`
55 if [ -z "$PFSCK" ] ; then
56 PFSCK=`which fsck 2>/dev/null`
64 if [ -n "$ZPOOL_LAYOUT_BUSES" -a -n "$ZPOOL_LAYOUT_PORTS" ] ; then
65 MAP_ARG=${ZPOOL_LAYOUT_MAP:+"-m $ZPOOL_LAYOUT_MAP"}
66 $ZPOOL_LAYOUT -t -b "$ZPOOL_LAYOUT_BUSES" \
67 -p "$ZPOOL_LAYOUT_PORTS" $MAP_ARG
69 if [ "$LOAD_ZFS" = "yes" ] && ! modprobe zfs ; then
70 echo "Failed to load zfs module. Aborting."
79 for label in $labels; do
80 devtype=`$LDEV -t $label`
81 if [ "$devtype" = "zfs" ] ; then
83 elif [ "$devtype" = "md" ] ; then
84 dev=`label_to_device $label`
85 journal=`$LDEV -j $label`
87 stop_md_device $journal
96 local pool=`$LDEV -z $label`
97 local args="-N $ZPOOL_IMPORT_ARGS"
98 local cache=`$LDEV -r $label`
99 # -c is incompatible with -d
100 if [ -n "$cache" ] ; then
101 args="$args -c $cache"
102 elif [ -n "$ZPOOL_IMPORT_DIR" ] ; then
103 args="$args -d $ZPOOL_IMPORT_DIR"
106 if zpool status $pool >/dev/null 2>&1 ; then
108 elif [ -n "$pool" ] ; then
109 zpool import $pool $args 2>/dev/null
112 if [ $result -ne 0 ] ; then
113 echo "Unexpected return code from import of pool $pool: $result"
121 local pool=`$LDEV -z $label`
122 zpool export $pool 2>/dev/null
125 # Trigger udev and wait for it to settle.
128 if [ -x ${UDEVADM} ]; then
129 ${UDEVADM} trigger --action=change --subsystem-match=block
137 # Usage: run_preexec_check [ start | restart | condrestart ]
138 # The single parameter will be passed to the PREEXEC_SCRIPT
141 if [ -n "$PREEXEC_CHECK" ] && ! $PREEXEC_CHECK ; then
142 echo "Pre-exec check \"$PREEXEC_CHECK\" failed. Aborting."
146 if [ -n "$PREEXEC_SCRIPT" ] && ! "$PREEXEC_SCRIPT" "$1" ; then
147 echo "Pre-exec script \"$PREEXEC_SCRIPT\" failed. Aborting."
152 # Usage: run_postexec_check [ start | restart | condrestart ]
153 # The single parameter will be passed to the PREEXEC_SCRIPT
154 run_postexec_check ()
156 if [ -n "$POSTEXEC_CHECK" ] && ! $POSTEXEC_CHECK ; then
157 echo "Post-exec check \"$POSTEXEC_CHECK\" failed. Aborting."
161 if [ -n "$POSTEXEC_SCRIPT" ] && ! "$POSTEXEC_SCRIPT" "$1" ; then
162 echo "Post-exec script \"$POSTEXEC_SCRIPT\" failed. Aborting."
167 # Usage: adjust_scsi_timeout <dev>
168 adjust_scsi_timeout ()
172 if [ -n "$SCSI_DEVICE_TIMEOUT" ]; then
173 # make sure that it is actually a SCSI (sd) device
174 local name=`basename $dev`
175 local proc=/sys/block/${name}/device/timeout
176 local driver=`readlink /sys/block/${name}/device/driver`
177 if [ -n "$driver" ] && [ "`basename $driver`" == "sd" ]; then
178 if ! echo $SCSI_DEVICE_TIMEOUT >$proc; then
179 echo "FAILED: could not adjust ${dev} timeout"
187 # Usage: fsck_test <dev> [ <dev> ... ]
188 # Checks all devices in parallel if FSCK_ARGS is set.
193 # Filter out non-absolute paths, which are probably ZFS datasets
194 devices=`echo $devices |xargs -n 1|grep '^/'|xargs`
196 if [ -n "${FSCK_ARGS}" -a -n "$devices" ]; then
197 if [ -x $PFSCK ] ; then
198 echo "$PFSCK $devices -- ${FSCK_ARGS}"
199 $PFSCK $devices -- ${FSCK_ARGS}
200 if [ $? -ne 0 -a $? -ne 1 ] ; then
201 echo "FAILED: $PFSCK -- ${FSCK_ARGS}: $?"
205 echo "$PFSCK not found"
212 # Usage: test_feature_flag <dev> <flag>
220 for feature in `$TUNE2FS -l $dev 2>/dev/null \
221 | grep features: | sed -e 's/^.*: //'`; do
222 if [ "$feature" == "$flag" ]; then
231 # Usage: mmp_test <dev>
232 # Returns 0 if it is set or not required, 1 if unset and required or error.
238 if [ "$REQUIRE_MMP_FEATURE" == "yes" ]; then
239 if [ -x $TUNE2FS ]; then
240 if ! test_feature_flag $dev "mmp"; then
241 echo "mmp feature flag is not set on $dev"
245 echo "$TUNE2FS not found"
253 # Usage: label_to_mountpt <label>
254 # Prints mount point path, if label matches a local or foreign server.
260 for serv in $LOCAL_SRV; do
261 if [ "$serv" == "$label" ]; then
262 echo "$LOCAL_MOUNT_DIR/$label"
266 for serv in $FOREIGN_SRV; do
267 if [ "$serv" == "$label" ]; then
268 echo "$FOREIGN_MOUNT_DIR/$label"
274 # Usage: label_to_device <label>
275 # Prints canonical device path.
279 local path=/dev/disk/by-label/$label
281 if [ -h $path ] ; then
282 readlink --canonicalize $path
288 # helper for mountpt_is_active() and device_is_active()
289 declare -r awkprog='BEGIN {rc = 1;}
290 { if ($field == path) {rc = 0;} }
293 # Usage: mountpt_is_active <label>
294 # Return 1 (inactive) on invalid label.
297 local dir=`label_to_mountpt $1`
300 if [ -n "$dir" ]; then
301 cat /proc/mounts | awk "$awkprog" field=2 path=$dir
307 # Usage: device_is_active <label>
308 # Return 1 (inactive) on invalid label.
311 local dev=`label_to_device $1`
314 if [ -n "$dev" ]; then
315 cat /proc/mounts | awk "$awkprog" field=1 path=$dir
321 # Usage: mount_one_device <label> <successflag> [devtype]
322 # Remove <successflag> on error (trick to detect errors after parallel runs).
328 local dev=`label_to_device $label`
329 local dir=`label_to_mountpt $label`
331 # $dir and $dev have already been checked at ths point
332 if [ ! -d $dir ] && ! mkdir -p $dir; then
336 echo "Mounting $dev on $dir"
337 if ! mount -t lustre $MOUNT_OPTIONS $dev $dir; then
343 # Usage: assemble_md_device <device>
344 # Assemble the md device backing device.
345 # Return 0 if the array is assembled successfully or was already active,
346 # otherwise return error code from mdadm.
347 assemble_md_device ()
354 if [ -n "$raidtab" ] ; then
355 args="$args -c $raidtab"
358 if ! md_array_is_active $dev ; then
367 # Usage: stop_md_device <device>
368 # Stop the md device backing device.
369 # Return 0 if the array is stopped successfully or was not active,
370 # otherwise return error code from mdadm.
378 if [ -n "$raidtab" ] ; then
379 args="$args -c $raidtab"
382 if [ -e $dev ] && md_array_is_active $dev ; then
390 # Usage: md_array_is_active <device>
391 # return 0 if device is an active md RAID array, or 1 otherwise
392 md_array_is_active ()
396 [ -e "$device" ] || return 1
398 mdadm --detail -t $device > /dev/null 2>&1
399 if [ $? -eq 4 ] ; then
405 # Usage: start_services <label> [ <label> ... ]
406 # fsck and mount any devices listed as arguments (in parallel).
407 # Attempt to assemble software raid arrays or zfs pools backing
419 dir=`label_to_mountpt $label`
420 devtype=`$LDEV -t $label`
421 dev=`label_to_device $label`
422 journal=`$LDEV -j $label`
423 raidtab=`$LDEV -r $label`
425 if [ -z "$dir" ] || [ -z "$dev" ]; then
426 echo "$label is not a valid lustre label on this node"
431 if [ "$devtype" = "md" ] ; then
432 if ! assemble_md_device $dev $raidtab ; then
433 echo "failed to assemble array $dev backing $label"
437 elif [ "$devtype" = "zfs" ] ; then
438 if ! import_zpool $label ; then
443 # Journal device field in ldev.conf may be "-" or empty,
444 # so only attempt to assemble if its an absolute path.
445 # Ignore errors since the journal device may not be an
447 if echo $journal | grep -q ^/ ; then
448 assemble_md_device $journal $raidtab 2>/dev/null
451 if [ "x$devtype" != "xzfs" ] ; then
452 if mountpt_is_active $label || \
453 device_is_active $label; then
454 echo "$label is already mounted"
458 if ! mmp_test $dev; then
462 if ! adjust_scsi_timeout $dev; then
467 devices="$devices $dev"
468 labels="$labels $label"
470 if [ $result == 0 ]; then
471 fsck_test $devices || return 2
473 # Fork to handle multiple mount_one_device()'s in parallel.
474 # Errors occurred if $successflag comes up missing afterwards.
476 [ -e $successflag ] || return 2
477 for label in $labels; do
478 mount_one_device $label $successflag `$LDEV -t $label` &
479 # stagger to avoid module loading races
480 if [[ -n $MOUNT_DELAY && $MOUNT_DELAY -gt 0 ]] ; then
484 for label in $labels; do
487 [ -e $successflag ] || return 2
494 # Usage: stop_services <label> [ <label> ... ]
495 # Unmount any devices listed as arguments (serially).
496 # Any devices which are not mounted or don't exist are skipped with no error.
504 for label in $labels; do
505 dir=`label_to_mountpt $label`
506 if [ -z "$dir" ]; then
507 echo "$label is not a valid lustre label on this node"
511 if ! mountpt_is_active $label; then
512 #echo "$label is not mounted"
517 echo "Unmounting $dir"
520 if [ -z "$pids" ]; then
527 # wait for all umount processes to complete, report any errors
529 wait $pid || result=2
533 for label in $labels; do
534 if mountpt_is_active $label; then
535 dir=`label_to_mountpt $label`
536 echo "Mount point $dir is still active"
539 if device_is_active $label; then
540 dev=`label_to_device $label`
541 echo "Device $dev is still active"
550 # Usage: start_lustre_services [local|foreign|all|<label>]
551 # If no parameter is specified, local devices will be started.
552 start_lustre_services ()
563 all) labels="$LOCAL_SRV $FOREIGN_SRV"
568 # for use by heartbeat V1 resource agent:
569 # starting an already-started service must not be an error
570 start_services $labels || exit 2
573 # Usage: stop_lustre_services [local|foreign|all|<label>]
574 # If no parameter is specified all devices will be stopped.
575 stop_lustre_services ()
580 local) labels=$LOCAL_SRV
585 ""|all) labels="$LOCAL_SRV $FOREIGN_SRV"
590 # for use by heartbeat V1 resource agent:
591 # stopping already-stopped service must not be an error
592 stop_services $labels || exit 2
595 # General lustre health check - not device specific.
599 old_nullglob="`shopt -p nullglob`"
603 # LSB compliance - return 3 if service is not running
604 # Lustre-specific returns
605 # 150 - partial startup
606 # 151 - health_check unhealthy
609 egrep -q "libcfs|lvfs|portals" /proc/modules && STATE="loaded"
611 # check for any configured devices (may indicate partial startup)
612 VAR=$(lctl get_param version 2>&1)
614 VAR=$(lctl get_param -n devices 2>&1)
620 # check for either a server or a client filesystem
625 VAR=$(lctl get_param -n mdt.*.recovery_status 2>&1 | grep '^status:' )
630 VAR=$(lctl get_param -n obdfilter.*.recovery_status 2>&1 | grep '^status:')
635 VAR=$(lctl get_param -n llite.fs* 2>&1)
640 if [ "$MDT" -o "$OST" -o "$LLITE" ]; then
645 # check if this is a router
646 if [[ "$(lctl get_param -n routes)" =~ "Routing enabled" ]]; then
652 # check for server disconnections
653 VAR=$(lctl get_param -n *c.*.*server_uuid 2>&1)
655 DISCON="$(echo $VAR | grep -v FULL)"
656 if [ -n "$DISCON" ] ; then
662 # check for servers in recovery
663 if [ -n "$MDT$OST" ] && echo $MDT $OST | grep -q RECOV ; then
668 # check for error in health_check
669 local health_check=$(lctl get_param -n health_check)
670 if [[ "$health_check" =~ "NOT HEALTHY" ]]; then
676 if [[ "$health_check" =~ "LBUG" ]]; then
686 # Usage: status [local|foreign|all|<label>]
687 # If no parameter is specified, general lustre health status will be reported.
695 local) labels=$LOCAL_SRV;
700 all) labels="$LOCAL_SRV $FOREIGN_SRV"
702 "") # ASSUMPTION: this is not the heartbeat res agent
709 # for use by heartbeat V1 resource agent:
710 # print "running" if *anything* is running.
711 for label in $labels; do
712 dir=`label_to_device $label`
713 if [ -z "$dir" ]; then
714 echo "$label is not a valid lustre label on this node"
719 if mountpt_is_active $label || device_is_active $label; then
724 [ $valid_devs == 1 ] && echo "stopped"
731 Usage: lustre {start|stop|status|restart|reload|condrestart}
733 lustre start [local|foreign|<label>]
734 lustre stop [local|foreign|<label>]
735 lustre status [local|foreign|<label>]
740 # See how we were called.
743 if [ $# -gt 2 ] ; then
744 echo "ERROR: Too many arguments."
747 run_preexec_check "start"
748 start_lustre_services $2
749 run_postexec_check "start"
752 if [ $# -gt 2 ] ; then
753 echo "ERROR: Too many arguments."
756 run_preexec_check "stop"
757 stop_lustre_services $2
758 run_postexec_check "stop"
761 if [ $# -gt 2 ] ; then
762 echo "ERROR: Too many arguments."
776 if grep lustre /proc/mounts ; then