3 # License: GNU General Public License (GPL)
6 # Description: Manages a lustre_server on a shared storage medium.
7 # Original Author: Eric Z. Ayers (eric.ayers@compgen.com)
8 # Original Release: 25 Oct 2000
10 # Rewritten Filesystem agent to lustre_server agent by bschubert@ddn.com
12 # usage: ./lustre_server {start|stop|status|monitor|validate-all|meta-data}
14 # OCF parameters are as below:
16 # OCF_RESKEY_directory
19 #OCF_RESKEY_device : name of block device for the filesystem. e.g. /dev/sda1, /dev/md0
20 # Or a -U or -L option for mount, or an NFS mount specification
21 #OCF_RESKEY_directory : the mount point for the filesystem
22 #OCF_RESKEY_options : options to be given to the mount command via -o
25 # NOTE: There is no locking (such as a SCSI reservation) being done here.
26 # I would if the SCSI driver could properly maintain the reservation,
27 # which it cannot, even with the 'scsi reservation' patch submitted
28 # earlier this year by James Bottomley. The patch minimizes the
29 # bus resets caused by a RESERVATION_CONFLICT return, and helps the
30 # reservation stay when 2 nodes contend for a reservation,
31 # but it does not attempt to recover the reservation in the
32 # case of a bus reset.
34 # What all this means is that if 2 nodes mount the same file system
35 # read-write, the filesystem is going to become corrupted. However
36 # Lustre provides the Multi-Mount-Protection feature (MMP).
38 # As a result, you should use this together with the stonith option
39 # and redundant, independent communications paths.
41 # If you don't do this, don't blame us when you scramble your disk.
43 # Note: the ServeRAID controller does prohibit concurrent acess
44 # In this case, you don't actually need STONITH, but redundant comm is
45 # still an excellent idea.
48 #######################################################################
51 . ${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs
53 #######################################################################
57 # FIXME: OCF_DEBUG officially supported in the the mean time?
62 usage: $0 {start|stop|status|monitor|validate-all|meta-data}
69 <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
70 <resource-agent name="lustre_server">
71 <version>1.0</version>
75 Resource script for lustre_server. It manages a lustre_server on a shared storage medium.
77 <shortdesc lang="en">lustre_server resource agent</shortdesc>
80 <parameter name="device" required="1">
82 The name of block device for the filesystem
84 <shortdesc lang="en">block device</shortdesc>
85 <content type="string" default="" />
88 <parameter name="directory" required="1">
90 The mount point for the filesystem.
92 <shortdesc lang="en">mount point</shortdesc>
93 <content type="string" default="" />
96 <parameter name="options">
98 Any extra options to be given as -o options to mount.
100 <shortdesc lang="en">options</shortdesc>
101 <content type="string" default="" />
106 <action name="start" timeout="300" />
107 <action name="stop" timeout="300" />
108 <action name="monitor" depth="0" timeout="300" interval="120" start-delay="10" />
109 <action name="validate-all" timeout="5" />
110 <action name="meta-data" timeout="5" />
117 # Make sure the kernel does the right thing with the FS buffers
118 # This function should be called after unmounting and before mounting
119 # It may not be necessary in 2.4 and later kernels, but it shouldn't hurt
122 # It's really a bug that you have to do this at all...
125 if have_binary $BLOCKDEV ; then
126 $BLOCKDEV --flushbufs $1
131 # Figure out the real device number of external journals
132 # we need to provide as mount options, since the block
133 # device only know major/minor of the the journal device and it
134 # might/will change dynamically on reboots and between servers
135 # It would be MUCH better if 'mount' would do it on its own
136 # and in fact mount is already linked against libblkid to do
138 get_external_journal_device()
140 UUID=`dumpe2fs -h $DEVICE 2>/dev/null | awk '/^Journal UUID/{print $3}'`
141 if [ -z "$UUID" ]; then
142 # device has internal journal, not need to proceed
146 # prefer /dev/mapper/
147 JDEV="`blkid -t UUID=$UUID /dev/mapper/* | awk -F: '{print $1}'`"
148 if [ -z "$JDEV" ]; then
149 JDEV="`blkid -t UUID=$UUID | awk -F: '{print $1}'`"
152 if [ -z "$JDEV" ]; then
153 # know the journal is on an external device, but we can't find it
154 ocf_log err "Cannot find device with journal UUID $UUID"
155 return $OCF_ERR_GENERIC
158 DEVNUM="`stat -c %02t%02T $JDEV`"
159 if [ -z "$DEVNUM" ]; then
160 ocf_log err "Failed to retrieve device number of Journal device"
161 return $OCF_ERR_GENERIC
164 # add 0x only here, because we couldn't check for an empty string otherwise
171 # Take advantage of /proc/mounts if present, use portable mount command
172 # otherwise. Normalize format to "dev mountpoint fstype".
175 if [ ! -f $mtab ]; then
176 ocf_log err "$mtab is missing!"
177 exit $OCF_ERR_GENERIC
179 cat $mtab | cut -d' ' -f1,2,3
182 lustre_health_check()
184 proc="/proc/fs/lustre/health_check"
185 # on first check the lustre modules are not loaded yet
186 if [ ! -e $proc ]; then
191 if [ "$check" = "healthy" ]; then
194 ocf_log err "$proc is $check"
200 # START: Start up the filesystem
202 lustre_server_start()
205 if [ $? -ne 0 ]; then
206 return ${OCF_ERR_GENERIC}
209 # See if the device is currently mounted
210 lustre_server_status >/dev/null 2>&1
212 if [ $rc -ne $OCF_NOT_RUNNING -a $rc -ne $OCF_SUCCESS ]; then
213 ocf_log err "Unexpected status ($c) before mounting $DEVICE to $MOUNTPOINT"
214 elif [ $rc -eq $OCF_SUCCESS ]; then
218 if [ ! -d "$MOUNTPOINT" ] ; then
219 ocf_log err "Couldn't find directory [$MOUNTPOINT] to use as a mount point"
225 jdev=`get_external_journal_device`
227 if [ $rc -ne 0 ]; then
228 ocf_log err "Failure to get journal device"
232 if [ -n "$jdev" ]; then
233 if [ -n "$options" ]; then
234 options="$options,journal_dev=$jdev"
236 options="-ojournal_dev=$jdev"
240 # Mount the filesystem.
241 cmd="$MOUNT -t $FSTYPE $options $DEVICE $MOUNTPOINT"
242 ocf_log info "Running $cmd"
244 if [ $? -ne 0 ]; then
245 ocf_log err "cmd \"$cmd\" failed: $res"
246 return $OCF_ERR_GENERIC
251 # end of lustre_server_start
254 # STOP: Unmount the filesystem
258 # See if the device is currently mounted
259 lustre_server_mounted >/dev/null 2>&1
260 if [ $? -eq $OCF_NOT_RUNNING ]; then
261 # Already unmounted, wonderful.
264 # Umount $MOUNTPOINT.
265 ocf_log info "Trying to unmount $MOUNTPOINT"
266 cmd="$UMOUNT $MOUNTPOINT"
267 ocf_log info "Running $cmd"
270 if [ $rc -eq 0 ] ; then
272 ocf_log info "unmounted $MOUNTPOINT successfully"
275 ocf_log err "\"$cmd\" failed ($rc): $res"
283 # end of lustre_server_stop
286 # MOUNTED: is the filesystem mounted or not?
288 lustre_server_mounted()
290 if list_mounts | grep -q " $MOUNTPOINT " >/dev/null 2>&1; then
292 msg="$MOUNTPOINT is mounted (running)"
295 msg="$MOUNTPOINT is unmounted (stopped)"
298 # check in all mntdevs if really not mounted
299 # lustre bug 21359 (https://bugzilla.lustre.org/show_bug.cgi?id=21359)
300 if [ $rc -eq $OCF_NOT_RUNNING ]; then
301 local list="/proc/fs/lustre/mds/* /proc/fs/lustre/obdfilter/*"
303 if [ -f ${i}/mntdev ]; then
304 MNTDEVS="$MNTDEVS ${i}/mntdev"
307 local mgsdev=/proc/fs/lustre/mgs/MGS/mntdev
308 if [ -f $mgsdev ]; then
309 MNTDEVS="$MNTDEVS $mgsdev"
311 for i in $MNTDEVS; do
313 if [ "$dev" = "$DEVICE" ]; then
314 ocf_log err "Bug21359, /proc/mounts claims device is not mounted, but $i proves this is wrong"
321 if [ "$OCF_DEBUG" = "yes" ]; then
326 status) ocf_log info "$msg"
329 if [ $rc -ne $OCF_SUCCESS ]; then
336 # end of lustre_server_mounted
339 # STATUS: is the filesystem mounted and healthy or not?
341 lustre_server_status()
344 if [ $? -ne 0 ]; then
345 return ${OCF_ERR_GENERIC}
348 lustre_server_mounted
353 # end of lustre_server_status
356 # Check if Lustre is available at all
358 lustre_server_validate_all()
360 proc="/proc/fs/lustre"
361 if [ ! -d $proc ]; then
364 while [ ! -d $proc -o $count -gt 10 ]; do
368 if [ ! -d $proc ]; then
369 ocf_log err "Failed to load the lustre module"
370 return $OCF_ERR_GENERIC
377 # Check the arguments passed to this script
378 if [ $# -ne 1 ]; then
383 # Check the OCF_RESKEY_ environment variables...
384 DEVICE=$OCF_RESKEY_device
385 if [ ! -z "$OCF_RESKEY_options" ]; then
386 options="-o $OCF_RESKEY_options"
391 if [ "$OCF_DEBUG" = "yes" ]; then
392 ocf_log info "OP = $OP"
395 # These operations do not require instance parameters
405 # Normalize instance parameters:
407 # It is possible that OCF_RESKEY_directory has one or even multiple trailing "/".
408 # But the output of `mount` and /proc/mounts do not.
409 if [ -z "$OCF_RESKEY_directory" ]; then
410 ocf_log err "Please specify the directory"
413 MOUNTPOINT=$(echo $OCF_RESKEY_directory | sed 's/\/*$//')
415 # At this stage, $MOUNTPOINT does not contain trailing "/" unless it is "/"
416 # TODO: / mounted via lustre_server sounds dangerous. On stop, we'll
417 # kill the whole system. Is that a good idea?
420 MOUNTPOINT=`echo $MOUNTPOINT | sed -e 's/\s*//'`
421 if [ -z "$MOUNTPOINT" ]; then
422 ocf_log err "Empty mount point!"
423 ocf_log err "Please specify the directory"
427 # Check to make sure the utilites are found
428 check_binary $MODPROBE
433 if [ "$OP" != "monitor" ]; then
434 ocf_log info "Running $OP for $DEVICE on $MOUNTPOINT"
437 # These operations do not require the clone checking + OCFS2
440 status|monitor) lustre_server_status
443 validate-all) lustre_server_validate_all
446 stop) lustre_server_stop
451 if [ -n "$OCF_RESKEY_CRM_meta_clone" ]; then
452 ocf_log err "DANGER! $FSTYPE on $DEVICE is NOT cluster-aware!"
453 ocf_log err "DO NOT RUN IT AS A CLONE!"
454 ocf_log err "Politely refusing to proceed to avoid data corruption."
455 exit $OCF_ERR_GENERIC
459 start) lustre_server_start
462 exit $OCF_ERR_UNIMPLEMENTED