lc_modprobe.sh
lc_hb.sh
lc_cluman.sh
+lc_md.sh
+lc_lvm.sh
# See the file COPYING in this distribution
# These are scripts that are generated from .in files
-genscripts = lustre_config.sh lc_modprobe.sh lc_net.sh lc_hb.sh lc_cluman.sh lustre_createcsv.sh
+genscripts = lustre_config.sh lc_modprobe.sh lc_net.sh lc_hb.sh lc_cluman.sh lustre_createcsv.sh lc_md.sh lc_lvm.sh
sbin_SCRIPTS = $(genscripts) lc_servip.sh lustre_up14.sh
GEN_CLUMGR_CONFIG=${SCRIPTS_PATH}/lc_cluman.sh
SCRIPT_VERIFY_SRVIP=${SCRIPTS_PATH}/lc_servip.sh
SCRIPT_GEN_MONCF=${SCRIPTS_PATH}/lc_mon.sh
+SCRIPT_CONFIG_MD=${SCRIPTS_PATH}/lc_md.sh
+SCRIPT_CONFIG_LVM=${SCRIPTS_PATH}/lc_lvm.sh
# Variables of HA software
HBVER_HBV1="hbv1" # Heartbeat version 1
FS_TYPE=${FS_TYPE:-"lustre"} # Lustre filesystem type
FILE_SUFFIX=${FILE_SUFFIX:-".lustre"} # Suffix of the generated config files
+# Marker of the MD device line
+MD_MARKER=${MD_MARKER:-"MD"}
+
+# Marker of the LVM device line
+PV_MARKER=${PV_MARKER:-"PV"}
+VG_MARKER=${VG_MARKER:-"VG"}
+LV_MARKER=${LV_MARKER:-"LV"}
+
declare -a CONFIG_ITEM # Items in each line of the csv file
+declare -a NODE_NAME # Hostnames of nodes have been configured
+
# verbose_output string
# Output verbose information $string
echo "$NAME"
fi
}
+
+# configured_host host_name
+#
+# Check whether the devices in $host_name has been configured or not
+configured_host() {
+ local host_name=$1
+ declare -i i
+
+ for ((i = 0; i < ${#NODE_NAME[@]}; i++)); do
+ [ "${host_name}" = "${NODE_NAME[i]}" ] && return 0
+ done
+
+ return 1
+}
--- /dev/null
+#!/bin/bash
+#
+# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
+#
+# lc_lvm.sh - configure Linux LVM devices from a csv file
+#
+################################################################################
+
+# Usage
+usage() {
+ cat >&2 <<EOF
+
+Usage: `basename $0` [-h] [-v] <csv file>
+
+ This script is used to configure Linux LVM devices in a Lustre cluster
+ from a csv file.
+
+ -h help and examples
+ -v verbose mode
+ csv file a spreadsheet that contains configuration parameters
+ (separated by commas) for each Linux LVM component
+ (PV, VG, LV) to be configured in a Lustre cluster
+
+EOF
+ exit 1
+}
+
+# Samples
+sample() {
+ cat <<EOF
+
+This script is used to configure Linux LVM devices in a Lustre cluster
+from a csv file.
+
+LVM is a Logical Volume Manager for the Linux operating system. The
+three-level components of it are PV (Physical Volume), VG (Volume Group)
+and LV (Logical Volume).
+
+Each line marked with "PV" in the csv file represents one or more PVs.
+The format is:
+hostname,PV,pv names,operation mode,options
+
+hostname hostname of the node in the cluster
+PV marker of PV line
+pv names devices or loopback files to be initialized for later
+ use by LVM or to be wiped the label, e.g. /dev/sda
+ Multiple devices or files are separated by space or by
+ using shell expansions, e.g. "/dev/sd{a,b,c}"
+operation mode create or remove, default is create
+options a "catchall" for other pvcreate/pvremove options
+ e.g. "-vv"
+
+Each line marked with "VG" in the csv file represents one VG.
+The format is:
+hostname,VG,vg name,operation mode,options,pv paths
+
+hostname hostname of the node in the cluster
+VG marker of VG line
+vg name name of the volume group, e.g. ost_vg
+operation mode create or remove, default is create
+options a "catchall" for other vgcreate/vgremove options
+ e.g. "-s 32M"
+pv paths physical volumes to construct this VG, required by
+ create mode
+ Multiple PVs are separated by space or by using
+ shell expansions, e.g. "/dev/sd[k-m]1"
+
+Each line marked with "LV" in the csv file represents one LV.
+The format is:
+hostname,LV,lv name,operation mode,options,lv size,vg name
+
+hostname hostname of the node in the cluster
+LV marker of LV line
+lv name name of the logical volume to be created (optional)
+ or path of the logical volume to be removed (required
+ by remove mode)
+operation mode create or remove, default is create
+options a "catchall" for other lvcreate/lvremove options
+ e.g. "-i 2 -I 128"
+lv size size [kKmMgGtT] to be allocated for the new LV
+ Default unit is megabytes.
+vg name name of the VG in which the new LV will be created
+
+Items left blank will be set to defaults.
+
+Example:
+-------------------------------------------------------
+# MD/LVM devices on mgsnode
+# Remove the LVM devices in the mgsnode
+mgsnode,LV,/dev/mgs_vg/mdt1,remove
+mgsnode,LV,/dev/mgs_vg/mdt2,remove
+mgsnode,VG,mgs_vg,remove
+mgsnode,PV,"/dev/sd{a,b}1",remove
+
+# Create MD device in the mgsnode
+mgsnode,MD,/dev/md0,,-q,1,/dev/sda1 /dev/sdb1
+
+
+# MD/LVM devices on ostnode
+# Create MD and LVM devices in the ostnode
+ostnode,MD,/dev/md0,,-q -c 128,5,"/dev/sd{a,b,c}"
+ostnode,MD,/dev/md1,,-q -c 128,5,"/dev/sd{d,e,f}"
+
+ostnode,PV,/dev/md0 /dev/md1
+ostnode,VG,ost_vg,,-s 32M,"/dev/md{0,1}"
+ostnode,LV,ost0,,-i 2 -I 128,300G,ost_vg
+ostnode,LV,ost1,,-i 2 -I 128,300G,ost_vg
+-------------------------------------------------------
+
+EOF
+ exit 0
+}
+
+# Get the library of functions
+. @scriptlibdir@/lc_common.sh
+
+#***************************** Global variables *****************************#
+# All the LVM device items in the csv file
+declare -a HOST_NAME LINE_MARKER LVM_NAME OP_MODE OP_OPTS SIXTH_ITEM SEVENTH_ITEM
+
+# Variables related to background executions
+declare -a REMOTE_CMD
+declare -a REMOTE_PID
+declare -i pid_num=0
+
+
+VERBOSE_OUTPUT=false
+# Get and check the positional parameters
+while getopts "hv" OPTION; do
+ case $OPTION in
+ h)
+ sample
+ ;;
+ v)
+ VERBOSE_OUTPUT=true
+ ;;
+ ?)
+ usage
+ esac
+done
+
+# Toss out the parameters we've already processed
+shift `expr $OPTIND - 1`
+
+# Here we expect the csv file
+if [ $# -eq 0 ]; then
+ echo >&2 "`basename $0`: Missing csv file!"
+ usage
+fi
+
+# check_lvm_item index
+#
+# Check the items required for managing LVM device ${LVM_NAME[index]}
+check_lvm_item() {
+ # Check argument
+ if [ $# -eq 0 ]; then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "Missing argument!"
+ return 1
+ fi
+
+ declare -i i=$1
+
+ # Check hostname
+ if [ -z "${HOST_NAME[i]}" ]; then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "hostname item has null value!"
+ return 1
+ fi
+
+ # Check LVM device name
+ if [ -z "${LVM_NAME[i]}" ] \
+ && [ "${LINE_MARKER[i]}" != "${LV_MARKER}" -a "${OP_MODE[i]}" != "remove" ]
+ then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "LVM component name item has null value!"
+ return 1
+ fi
+
+ # Check the operation mode
+ if [ -n "${OP_MODE[i]}" ] \
+ && [ "${OP_MODE[i]}" != "create" -a "${OP_MODE[i]}" != "remove" ]
+ then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "Invalid operation mode item - \"${OP_MODE[i]}\"!"
+ return 1
+ fi
+
+ # Check items required by create mode
+ if [ -z "${OP_MODE[i]}" -o "${OP_MODE[i]}" = "create" ]; then
+ if [ "${LINE_MARKER[i]}" = "${VG_MARKER}" -a -z "${SIXTH_ITEM[i]}" ]
+ then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "pv paths item of vg ${LVM_NAME[i]} has null value!"
+ return 1
+ fi
+
+ if [ "${LINE_MARKER[i]}" = "${LV_MARKER}" ]; then
+ if [ -z "${SIXTH_ITEM[i]}" ]; then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "lv size item has null value!"
+ return 1
+ fi
+
+ if [ -z "${SEVENTH_ITEM[i]}" ]; then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "vg name item has null value!"
+ return 1
+ fi
+ fi
+ fi
+
+ return 0
+}
+
+# get_lvm_items csv_file
+#
+# Get all the LVM device items in the $csv_file and do some checks.
+get_lvm_items() {
+ # Check argument
+ if [ $# -eq 0 ]; then
+ echo >&2 "`basename $0`: get_lvm_items() error: Missing csv file!"
+ return 1
+ fi
+
+ CSV_FILE=$1
+ local LINE line_marker
+ declare -i line_num=0
+ declare -i idx=0
+
+ while read -r LINE; do
+ let "line_num += 1"
+
+ # Skip the comment line
+ [ -z "`echo \"${LINE}\" | egrep -v \"([[:space:]]|^)#\"`" ] && continue
+
+ # Skip the non-LVM line
+ line_marker=`echo ${LINE} | awk -F, '{print $2}'`
+ [ "${line_marker}" != "${PV_MARKER}" ] \
+ && [ "${line_marker}" != "${VG_MARKER}" ] \
+ && [ "${line_marker}" != "${LV_MARKER}" ] && continue
+
+ # Parse the config line into CONFIG_ITEM
+ if ! parse_line $LINE; then
+ return 1
+ fi
+
+ HOST_NAME[idx]=${CONFIG_ITEM[0]}
+ LINE_MARKER[idx]=${CONFIG_ITEM[1]}
+ LVM_NAME[idx]=${CONFIG_ITEM[2]}
+ OP_MODE[idx]=${CONFIG_ITEM[3]}
+ OP_OPTS[idx]=${CONFIG_ITEM[4]}
+ SIXTH_ITEM[idx]=${CONFIG_ITEM[5]}
+ SEVENTH_ITEM[idx]=${CONFIG_ITEM[6]}
+
+ LVM_NAME[idx]=`echo "${LVM_NAME[idx]}" | sed 's/^"//' | sed 's/"$//'`
+ OP_OPTS[idx]=`echo "${OP_OPTS[idx]}" | sed 's/^"//' | sed 's/"$//'`
+ SIXTH_ITEM[idx]=`echo "${SIXTH_ITEM[idx]}" | sed 's/^"//' | sed 's/"$//'`
+
+ # Check some required items
+ if ! check_lvm_item $idx; then
+ echo >&2 "`basename $0`: check_lvm_item() error:"\
+ "Occurred on line ${line_num} in ${CSV_FILE}."
+ return 1
+ fi
+
+ let "idx += 1"
+ done < ${CSV_FILE}
+
+ return 0
+}
+
+# construct_lvm_create_cmdline index
+#
+# Construct the creation command line for ${LVM_NAME[index]}
+construct_lvm_create_cmdline() {
+ declare -i i=$1
+ local lvm_cmd
+
+ case "${LINE_MARKER[i]}" in
+ "${PV_MARKER}")
+ lvm_cmd="pvcreate -ff -y ${OP_OPTS[i]} ${LVM_NAME[i]}"
+ ;;
+ "${VG_MARKER}")
+ lvm_cmd="vgcreate ${OP_OPTS[i]} ${LVM_NAME[i]} ${SIXTH_ITEM[i]}"
+ ;;
+ "${LV_MARKER}")
+ if [ -z "${LVM_NAME[i]}" ]; then
+ lvm_cmd="lvcreate -L ${SIXTH_ITEM[i]} ${OP_OPTS[i]} ${SEVENTH_ITEM[i]}"
+ else
+ lvm_cmd="lvcreate -L ${SIXTH_ITEM[i]} -n ${LVM_NAME[i]} ${OP_OPTS[i]} ${SEVENTH_ITEM[i]}"
+ fi
+ ;;
+ esac
+
+ echo ${lvm_cmd}
+ return 0
+}
+
+# cmdline_rm_LVs vg_name
+#
+# Construct command line to remove all the LVs on $vg_name.
+# If $vg_name is null, then remove all the LVs in the host.
+cmdline_rm_LVs() {
+ local vg_name=$1
+ local lvm_rm_cmd
+
+ lvm_rm_cmd="vgchange -a n ${vg_name} &&"
+ lvm_rm_cmd=${lvm_rm_cmd}" vgdisplay -v ${vg_name} | grep \"LV Name\" | awk '{print \$3}' |"
+ lvm_rm_cmd=${lvm_rm_cmd}" while read lv; do lvremove -f \$lv; done"
+
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+# cmdline_rm_LV lv_path
+#
+# Construct command line to remove LV $lv_path
+cmdline_rm_LV() {
+ local lv_path=$1
+ local lvm_rm_cmd
+
+ lvm_rm_cmd="lvchange -a n ${lv_path} && lvremove -f ${lv_path}"
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+
+# cmdline_rm_VG vg_name
+#
+# Construct command line to remove VG $vg_name
+cmdline_rm_VG() {
+ local vg_name=$1
+ local lvm_rm_cmd
+
+ # Remove all the LVs on this VG
+ lvm_rm_cmd=$(cmdline_rm_LVs ${vg_name})
+
+ # Remove this VG
+ lvm_rm_cmd=${lvm_rm_cmd}" && vgremove ${vg_name}"
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+# cmdline_rm_VGs
+#
+# Construct command line to remove all the VGs in the host
+cmdline_rm_VGs() {
+ local lvm_rm_cmd
+
+ # Remove all the LVs in the host
+ lvm_rm_cmd=$(cmdline_rm_LVs)
+
+ # Remove all the VGs in the host
+ lvm_rm_cmd=${lvm_rm_cmd}" && vgdisplay | grep \"VG Name\" | awk '{print \$3}' |"
+ lvm_rm_cmd=${lvm_rm_cmd}" while read vg; do vgremove \$vg; done"
+
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+# cmdline_rm_PVs
+#
+# Construct command line to remove all the PVs in the host
+cmdline_rm_PVs() {
+ local lvm_rm_cmd
+
+ # Remove all the LVs and VGs in the host
+ lvm_rm_cmd=$(cmdline_rm_VGs)
+
+ # Remove all the PVs in the host
+ lvm_rm_cmd=${lvm_rm_cmd}" && pvdisplay | grep \"PV Name\" | awk '{print \$3}' |"
+ lvm_rm_cmd=${lvm_rm_cmd}" while read pv; do pvremove -ff -y \$pv; done"
+
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+# construct_lvm_teardown_cmdline index
+#
+# Construct the teardown command line for LVM devices in ${HOST_NAME[index]}
+construct_lvm_teardown_cmdline() {
+ declare -i i=$1
+ local lvm_rm_cmd
+
+ case "${LINE_MARKER[i]}" in
+ "${LV_MARKER}")
+ lvm_rm_cmd=$(cmdline_rm_LVs ${SEVENTH_ITEM[i]})
+ ;;
+ "${VG_MARKER}")
+ # Remove all the VGs in the host
+ lvm_rm_cmd=$(cmdline_rm_VGs)
+ ;;
+ "${PV_MARKER}")
+ # Remove all the PVs in the host
+ lvm_rm_cmd=$(cmdline_rm_PVs)
+ ;;
+ esac
+
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+# construct_lvm_rm_cmdline index
+#
+# Construct the remove command line for LVM device ${LVM_NAME[index]}
+construct_lvm_rm_cmdline() {
+ declare -i i=$1
+ local lvm_rm_cmd
+
+ case "${LINE_MARKER[i]}" in
+ "${LV_MARKER}")
+ lvm_rm_cmd=$(cmdline_rm_LV ${LVM_NAME[i]})
+ ;;
+ "${VG_MARKER}")
+ lvm_rm_cmd=$(cmdline_rm_VG ${LVM_NAME[i]})
+ ;;
+ "${PV_MARKER}")
+ lvm_rm_cmd="pvremove -ff -y ${LVM_NAME[i]}"
+ ;;
+ esac
+
+ echo ${lvm_rm_cmd}
+ return 0
+}
+
+# construct_lvm_cmdline host_name
+#
+# Construct the command line of LVM utilities to be run in the $host_name
+construct_lvm_cmdline() {
+ LVM_CMDLINE=
+ local host_name=$1
+ local lvm_cmd
+ declare -i i
+
+ # Construct command line
+ for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
+ lvm_cmd=
+ if [ "${host_name}" = "${HOST_NAME[i]}" ]; then
+ case "${OP_MODE[i]}" in
+ "" | create)
+ # Construct the create command line
+ lvm_cmd=$(construct_lvm_create_cmdline ${i})
+ ;;
+ remove)
+ if [ -z "${LVM_NAME[i]}" ]; then
+ # Construct the teardown command line
+ lvm_cmd=$(construct_lvm_teardown_cmdline ${i})
+ else # Remove instead of teardown
+ # Construct the remove command line
+ lvm_cmd=$(construct_lvm_rm_cmdline ${i})
+ fi
+ ;;
+ *)
+ echo >&2 "`basename $0`: construct_lvm_cmdline() error:"\
+ "Invalid operation mode - \"${OP_MODE[i]}\"!"
+ return 1
+ ;;
+ esac
+
+ if [ -z "${LVM_CMDLINE}" ]; then
+ LVM_CMDLINE=${lvm_cmd}
+ else
+ LVM_CMDLINE=${LVM_CMDLINE}" && "${lvm_cmd}
+ fi
+ fi
+ done
+
+ return 0
+}
+
+# config_lvm_devs host_name
+#
+# Run remote command to configure LVM devices in $host_name
+config_lvm_devs() {
+ local host_name=$1
+
+ # Construct the LVM utilities command line
+ if ! construct_lvm_cmdline ${host_name}; then
+ return 1
+ fi
+
+ if [ -z "${LVM_CMDLINE}" ]; then
+ verbose_output "There are no LVM devices on host ${host_name}"\
+ "needed to be configured."
+ return 0
+ fi
+
+ # Run remote command to configure LVM devices in $host_name
+ verbose_output "Configuring LVM devices in host ${host_name}..."
+ verbose_output "Configure command line is: \"${LVM_CMDLINE}\""
+ REMOTE_CMD[pid_num]="${REMOTE} ${host_name} \"${LVM_CMDLINE}\""
+ ${REMOTE} ${host_name} "(${EXPORT_PATH} ${LVM_CMDLINE})" >&2 &
+ REMOTE_PID[pid_num]=$!
+ let "pid_num += 1"
+
+ return 0
+}
+
+# Run remote command to configure all the LVM devices specified
+# in the csv file
+config_lvm() {
+ declare -i i=0
+ declare -i idx=0 # Index of NODE_NAME array
+ local host_name
+ local failed_status
+
+ # Initialize the NODE_NAME array
+ unset NODE_NAME
+
+ for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
+ host_name=${HOST_NAME[i]}
+ configured_host ${host_name} && continue
+
+ NODE_NAME[idx]=${host_name}
+ let "idx += 1"
+
+ # Run remote command to configure LVM devices in $host_name
+ if ! config_lvm_devs ${host_name}; then
+ return 1
+ fi
+ done
+
+ if [ ${#HOST_NAME[@]} -eq 0 -o ${#REMOTE_PID[@]} -eq 0 ]; then
+ verbose_output "There are no LVM devices to be configured."
+ return 0
+ fi
+
+ # Wait for the exit status of the background remote command
+ verbose_output "Waiting for the return of the remote command..."
+ failed_status=false
+ for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do
+ wait ${REMOTE_PID[${pid_num}]}
+ if [ $? -ne 0 ]; then
+ echo >&2 "`basename $0`: config_lvm() error: Failed"\
+ "to execute \"${REMOTE_CMD[${pid_num}]}\"!"
+ failed_status=true
+ fi
+ done
+
+ if ${failed_status}; then
+ return 1
+ fi
+
+ verbose_output "All the LVM devices are configured successfully!"
+ return 0
+}
+
+# Main flow
+# Check the csv file
+if ! check_file $1; then
+ exit 1
+fi
+
+# Get all the LVM device items from the csv file
+if ! get_lvm_items ${CSV_FILE}; then
+ exit 1
+fi
+
+# Configure the LVM devices
+if ! config_lvm; then
+ exit 1
+fi
+
+exit 0
--- /dev/null
+#!/bin/bash
+#
+# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
+#
+# lc_md.sh - configure Linux MD devices from a csv file
+#
+################################################################################
+
+# Usage
+usage() {
+ cat >&2 <<EOF
+
+Usage: `basename $0` [-h] [-v] <csv file>
+
+ This script is used to configure Linux MD devices in a Lustre cluster
+ from a csv file.
+
+ -h help and examples
+ -v verbose mode
+ csv file a spreadsheet that contains configuration parameters
+ (separated by commas) for each Linux MD device to be
+ configured in a Lustre cluster
+
+EOF
+ exit 1
+}
+
+# Samples
+sample() {
+ cat <<EOF
+
+This script is used to configure Linux MD devices in a Lustre cluster
+from a csv file.
+
+Each line marked with "MD" in the csv file represents one MD device.
+The format is:
+hostname,MD,md name,operation mode,options,raid level,component devices
+
+hostname hostname of the node in the cluster
+MD marker of MD device line
+md name MD device name, e.g. /dev/md0
+operation mode create or remove, default is create
+options a "catchall" for other mdadm options, e.g. "-c 128"
+raid level raid level: 0,1,4,5,6,10,linear and multipath
+component devices block devices to be combined into the MD device
+ Multiple devices are separated by space or by using
+ shell expansions, e.g. "/dev/sd{a,b,c}"
+
+Items left blank will be set to defaults.
+
+Example:
+-------------------------------------------------------
+# MD devices on mgsnode
+mgsnode,MD,/dev/md0,,-q -c 32,1,/dev/sda1 /dev/sdb1
+mgsnode,MD,/dev/md1,,-q -c 32,1,/dev/sdc1 /dev/sdd1
+mgsnode,MD,/dev/md2,,-q -c 32,0,/dev/md0 /dev/md1
+
+# MD device on ostnode
+ostnode,MD,/dev/md0,,-q -c 128,5,"/dev/sd{a,b,c,d,e}"
+-------------------------------------------------------
+
+EOF
+ exit 0
+}
+
+# Get the library of functions
+. @scriptlibdir@/lc_common.sh
+
+#***************************** Global variables *****************************#
+# Raid command path
+RAID_CMD_PATH=${RAID_CMD_PATH:-"/sbin"}
+MDADM=${MDADM:-"$RAID_CMD_PATH/mdadm"}
+
+# All the MD device items in the csv file
+declare -a HOST_NAME MD_NAME OP_MODE OP_OPTS RAID_LEVEL MD_DEVS
+
+# Variables related to background executions
+declare -a REMOTE_CMD
+declare -a REMOTE_PID
+declare -i pid_num=0
+
+
+VERBOSE_OUTPUT=false
+# Get and check the positional parameters
+while getopts "hv" OPTION; do
+ case $OPTION in
+ h)
+ sample
+ ;;
+ v)
+ VERBOSE_OUTPUT=true
+ ;;
+ ?)
+ usage
+ esac
+done
+
+# Toss out the parameters we've already processed
+shift `expr $OPTIND - 1`
+
+# Here we expect the csv file
+if [ $# -eq 0 ]; then
+ echo >&2 "`basename $0`: Missing csv file!"
+ usage
+fi
+
+# check_md_item index
+#
+# Check the items required for managing MD device ${MD_NAME[index]}
+check_md_item() {
+ # Check argument
+ if [ $# -eq 0 ]; then
+ echo >&2 "`basename $0`: check_md_item() error:"\
+ "Missing argument!"
+ return 1
+ fi
+
+ declare -i i=$1
+
+ # Check hostname
+ if [ -z "${HOST_NAME[i]}" ]; then
+ echo >&2 "`basename $0`: check_md_item() error:"\
+ "hostname item has null value!"
+ return 1
+ fi
+
+ # Check items required by create mode
+ if [ -z "${OP_MODE[i]}" -o "${OP_MODE[i]}" = "create" ]; then
+ # Check MD device name
+ if [ -z "${MD_NAME[i]}" ]; then
+ echo >&2 "`basename $0`: check_md_item() error:"\
+ "md name item has null value!"
+ return 1
+ fi
+
+ if [ -z "${RAID_LEVEL[i]}" ]; then
+ echo >&2 "`basename $0`: check_md_item() error:"\
+ "raid level item of MD device ${MD_NAME[i]} has null value!"
+ return 1
+ fi
+
+ if [ -z "${MD_DEVS[i]}" ]; then
+ echo >&2 "`basename $0`: check_md_item() error:"\
+ "component devices item of ${MD_NAME[i]} has null value!"
+ return 1
+ fi
+ fi
+
+ return 0
+}
+
+# get_md_items csv_file
+#
+# Get all the MD device items in the $csv_file and do some checks.
+get_md_items() {
+ # Check argument
+ if [ $# -eq 0 ]; then
+ echo >&2 "`basename $0`: get_md_items() error: Missing csv file!"
+ return 1
+ fi
+
+ CSV_FILE=$1
+ local LINE
+ declare -i line_num=0
+ declare -i idx=0
+
+ while read -r LINE; do
+ let "line_num += 1"
+
+ # Skip the comment line
+ [ -z "`echo \"${LINE}\" | egrep -v \"([[:space:]]|^)#\"`" ] && continue
+
+ # Skip the non-MD line
+ [ "`echo ${LINE}|awk -F, '{print $2}'`" != "${MD_MARKER}" ] && continue
+
+ # Parse the config line into CONFIG_ITEM
+ if ! parse_line $LINE; then
+ return 1
+ fi
+
+ HOST_NAME[idx]=${CONFIG_ITEM[0]}
+ MD_NAME[idx]=${CONFIG_ITEM[2]}
+ OP_MODE[idx]=${CONFIG_ITEM[3]}
+ OP_OPTS[idx]=${CONFIG_ITEM[4]}
+ RAID_LEVEL[idx]=${CONFIG_ITEM[5]}
+ MD_DEVS[idx]=${CONFIG_ITEM[6]}
+
+ OP_OPTS[idx]=`echo "${OP_OPTS[idx]}" | sed 's/^"//' | sed 's/"$//'`
+ MD_DEVS[idx]=`echo "${MD_DEVS[idx]}" | sed 's/^"//' | sed 's/"$//'`
+
+ # Check some required items
+ if ! check_md_item $idx; then
+ echo >&2 "`basename $0`: check_md_item() error:"\
+ "Occurred on line ${line_num} in ${CSV_FILE}."
+ return 1
+ fi
+
+ let "idx += 1"
+ done < ${CSV_FILE}
+
+ return 0
+}
+
+# md_is_active host_name md_name
+#
+# Run remote command to check whether $md_name is active in @host_name
+md_is_active() {
+ local host_name=$1
+ local md_name=$2
+ local cmd ret_str
+
+ cmd="grep -q ${md_name##*/} /proc/mdstat 2>&1"
+ ret_str=`${REMOTE} ${host_name} "${cmd}" 2>&1`
+ if [ $? -ne 0 ]; then
+ if [ -n "${ret_str}" ]; then
+ echo >&2 "`basename $0`: md_is_active() error:"\
+ "remote command to ${host_name} error: ${ret_str}!"
+ return 2 # Error occurred
+ else
+ return 1 # inactive
+ fi
+ fi
+
+ return 0 # active
+}
+
+# construct_mdadm_create_cmdline index
+#
+# Construct the create operation command line of mdadm for ${MD_NAME[index]}
+construct_mdadm_create_cmdline() {
+ declare -i i=$1
+ local cmd_line
+ local echo_disk disk line
+ declare -i alldisks=0
+ declare -i raiddisks=0
+ declare -i sparedisks=0
+
+ cmd_line="${MDADM} -C -R ${MD_NAME[i]} ${OP_OPTS[i]} -l ${RAID_LEVEL[i]}"
+
+ if [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#* -n*}" ]\
+ || [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#*--raid-devices*}" ]; then
+ cmd_line=${cmd_line}" ${MD_DEVS[i]}"
+ echo ${cmd_line}
+ return 0
+ fi
+
+ # FIXME: Get the number of component devices in the array
+ echo_disk="for disk in ${MD_DEVS[i]}; do echo $disk; done"
+ while read line; do
+ let "alldisks += 1"
+ done < <(${REMOTE} ${HOST_NAME[i]} "${echo_disk}")
+
+ if [ ${alldisks} -eq 0 ]; then
+ echo "`basename $0`: construct_mdadm_create_cmdline() error:"\
+ "Failed to execute remote command to get the number of"\
+ "component devices of array ${MD_NAME[i]} from host ${HOST_NAME[i]}!"
+ return 1
+ fi
+
+ # Get the specified number of spare (eXtra) devices
+ if [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#* -x*}" ]; then
+ sparedisks=`echo ${OP_OPTS[i]##* -x}|awk -F" " '{print $1}'`
+ elif [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#*--spare-devices*}" ]; then
+ sparedisks=`echo ${OP_OPTS[i]##*--spare-devices=}|awk -F" " '{print $1}'`
+ fi
+
+ # Get the number of raid devices in the array
+ # The number of raid devices in the array plus the number of spare devices
+ # listed on the command line must equal the number of component devices
+ # (including "missing" devices).
+ let "raiddisks = alldisks - sparedisks"
+
+ if [ ${raiddisks} -lt 1 ]; then
+ echo "`basename $0`: construct_mdadm_create_cmdline() error:"\
+ "Invalid number of raid devices in array ${MD_NAME[i]}: ${raiddisks}!"\
+ "Check the number of spare devices and whether all the component devices"\
+ "\"${MD_DEVS[i]}\" (except \"missing\" devices) exist in host ${HOST_NAME[i]}!"
+ return 1
+ fi
+
+ cmd_line=${cmd_line}" -n ${raiddisks} ${MD_DEVS[i]}"
+
+ echo ${cmd_line}
+ return 0
+}
+
+# construct_mdadm_rm_cmdline index
+#
+# Construct the remove operation command line of mdadm for ${MD_NAME[index]}
+construct_mdadm_rm_cmdline() {
+ declare -i i=$1
+ local mdadm_cmd
+ local real_devs
+
+ # Deactivate the MD array, releasing all resources
+ mdadm_cmd="${MDADM} -S ${MD_NAME[i]}"
+
+ if [ -n "${MD_DEVS[i]}" ]; then
+ # Remove the "missing" devices from the component devices
+ real_devs=`echo ${MD_DEVS[i]} | sed 's/missing//g'`
+ # Over-written the superblock with zeros
+ mdadm_cmd=${mdadm_cmd}" && ${MDADM} --zero-superblock ${real_devs}"
+ fi
+
+ echo ${mdadm_cmd}
+ return 0
+}
+
+# construct_mdadm_cmdline host_name
+#
+# Construct the command line of mdadm to be run in $host_name
+construct_mdadm_cmdline() {
+ MDADM_CMDLINE=
+ local host_name=$1
+ local mdadm_stop_cmd mdadm_cmd
+ local rc OK
+ declare -i i
+
+ # Construct command line
+ for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
+ mdadm_stop_cmd=
+ mdadm_cmd=
+ if [ "${host_name}" = "${HOST_NAME[i]}" ]; then
+ case "${OP_MODE[i]}" in
+ "" | create)
+ # Check the status of the MD array
+ md_is_active ${host_name} ${MD_NAME[i]}
+ rc=$?
+ if [ "$rc" -eq "2" ]; then
+ return 1
+ elif [ "$rc" -eq "0" ]; then
+ OK=
+ echo -n "`basename $0`: ${MD_NAME[i]} is active on"\
+ "${host_name}, go ahead to deactivate it and create"\
+ "the new array? [y/n]:"
+ read OK
+ if [ "${OK}" = "n" ]; then
+ echo "`basename $0`: ${MD_NAME[i]} on host"\
+ "${host_name} remains as it is."
+ continue
+ fi
+
+ # Construct the remove command line
+ mdadm_stop_cmd=$(construct_mdadm_rm_cmdline ${i})
+ fi
+
+ # Construct the create command line
+ mdadm_cmd=$(construct_mdadm_create_cmdline ${i})
+ if [ $? -ne 0 ]; then
+ echo >&2 "${mdadm_cmd}"
+ return 1
+ fi
+
+ [ -n "${mdadm_stop_cmd}" ] && mdadm_cmd=${mdadm_stop_cmd}" && "${mdadm_cmd}
+ ;;
+ remove)
+ if [ -z "${MD_NAME[i]}" ]; then
+ OK=
+ echo -n "`basename $0`: Do you really want to remove"\
+ "all the MD devices in the host ${HOST_NAME[i]}? [y/n]:"
+ read OK
+ if [ "${OK}" = "n" ]; then
+ echo "`basename $0`: MD devices on host"\
+ "${HOST_NAME[i]} remain as they are."
+ continue
+ fi
+
+ # Construct the teardown command line
+ mdadm_cmd="(cat /proc/mdstat | egrep \"^md[[:digit:]]\" |"
+ mdadm_cmd=${mdadm_cmd}" while read md rest; do ${MDADM} -S /dev/\$md; done)"
+ else
+ # Construct the remove command line
+ mdadm_cmd=$(construct_mdadm_rm_cmdline ${i})
+ fi
+ ;;
+ *)
+ # Other operations
+ mdadm_cmd="${MDADM} ${OP_MODE[i]} ${MD_NAME[i]} ${OP_OPTS[i]} ${MD_DEVS[i]}"
+ ;;
+ esac
+
+ if [ -z "${MDADM_CMDLINE}" ]; then
+ MDADM_CMDLINE=${mdadm_cmd}
+ else
+ MDADM_CMDLINE=${MDADM_CMDLINE}" && "${mdadm_cmd}
+ fi
+ fi
+ done
+
+ return 0
+}
+
+# config_md_devs host_name
+#
+# Run remote command to configure MD devices in $host_name
+config_md_devs() {
+ local host_name=$1
+
+ # Construct mdadm command line
+ if ! construct_mdadm_cmdline ${host_name}; then
+ return 1
+ fi
+
+ if [ -z "${MDADM_CMDLINE}" ]; then
+ verbose_output "There are no MD devices on host ${host_name}"\
+ "needed to be configured."
+ return 0
+ fi
+
+ # Run remote command to configure MD devices in $host_name
+ verbose_output "Configuring MD devices in host ${host_name}..."
+ verbose_output "Configure command line is: \"${MDADM_CMDLINE}\""
+ REMOTE_CMD[pid_num]="${REMOTE} ${host_name} \"${MDADM_CMDLINE}\""
+ ${REMOTE} ${host_name} "${MDADM_CMDLINE}" >&2 &
+ REMOTE_PID[pid_num]=$!
+ let "pid_num += 1"
+ sleep 1
+
+ return 0
+}
+
+# Run remote command to configure all the MD devices specified in the csv file
+config_md() {
+ declare -i i=0
+ declare -i idx=0 # Index of NODE_NAME array
+ local host_name
+ local failed_status
+
+ # Initialize the NODE_NAME array
+ unset NODE_NAME
+
+ for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
+ host_name=${HOST_NAME[i]}
+ configured_host ${host_name} && continue
+
+ NODE_NAME[idx]=${host_name}
+ let "idx += 1"
+
+ # Run remote command to configure MD devices in $host_name
+ if ! config_md_devs ${host_name}; then
+ return 1
+ fi
+ done
+
+ if [ ${#HOST_NAME[@]} -eq 0 -o ${#REMOTE_PID[@]} -eq 0 ]; then
+ verbose_output "There are no MD devices to be configured."
+ return 0
+ fi
+
+ # Wait for the exit status of the background remote command
+ verbose_output "Waiting for the return of the remote command..."
+ failed_status=false
+ for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do
+ wait ${REMOTE_PID[${pid_num}]}
+ if [ $? -ne 0 ]; then
+ echo >&2 "`basename $0`: config_md() error: Failed"\
+ "to execute \"${REMOTE_CMD[${pid_num}]}\"!"
+ failed_status=true
+ fi
+ done
+
+ if ${failed_status}; then
+ return 1
+ fi
+
+ verbose_output "All the MD devices are configured successfully!"
+ return 0
+}
+
+# Main flow
+# Check the csv file
+if ! check_file $1; then
+ exit 1
+fi
+
+# Get all the MD device items from the csv file
+if ! get_md_items ${CSV_FILE}; then
+ exit 1
+fi
+
+# Configure the MD devices
+if ! config_md; then
+ exit 1
+fi
+
+exit 0
# that will be part of the Lustre cluster.
#
# In addition, it can also verify the network connectivity and hostnames in
-# the cluster and produce High-Availability software configurations for
-# Heartbeat or CluManager.
+# the cluster, configure Linux MD/LVM devices and produce High-Availability
+# software configurations for Heartbeat or CluManager.
#
################################################################################
usage() {
cat >&2 <<EOF
-Usage: `basename $0` [-t HAtype] [-n] [-f] [-m] [-h] [-v] <csv file>
+Usage: `basename $0` [-t HAtype] [-n] [-d] [-f] [-m] [-h] [-v] <csv file>
This script is used to format and set up multiple lustre servers from a
csv file.
and hbv2 (Heartbeat version 2).
-n no net - don't verify network connectivity and hostnames
in the cluster
+ -d configure Linux MD/LVM devices before formatting the
+ Lustre targets
-f force-format the Lustre targets using --reformat option
-m no fstab change - don't modify /etc/fstab to add the new
Lustre targets
It can also optionally:
* verify the network connectivity and hostnames in the cluster
+ * configure Linux MD/LVM devices
* modify /etc/modprobe.conf to add Lustre networking info
* add the Lustre server info to /etc/fstab
* produce configurations for Heartbeat or CluManager.
-Each line in the csv file represents one Lustre target. The format is:
+There are 5 kinds of line formats in the csv file. They represent the following
+targets:
+1) Linux MD device
+The format is:
+hostname,MD,md name,operation mode,options,raid level,component devices
+
+hostname hostname of the node in the cluster
+MD marker of MD device line
+md name MD device name, e.g. /dev/md0
+operation mode create or remove, default is create
+options a "catchall" for other mdadm options, e.g. "-c 128"
+raid level raid level: 0,1,4,5,6,10,linear and multipath
+component devices block devices to be combined into the MD device
+ Multiple devices are separated by space or by using
+ shell expansions, e.g. "/dev/sd{a,b,c}"
+
+2) Linux LVM PV (Physical Volume)
+The format is:
+hostname,PV,pv names,operation mode,options
+
+hostname hostname of the node in the cluster
+PV marker of PV line
+pv names devices or loopback files to be initialized for later
+ use by LVM or to be wiped the label, e.g. /dev/sda
+ Multiple devices or files are separated by space or by
+ using shell expansions, e.g. "/dev/sd{a,b,c}"
+operation mode create or remove, default is create
+options a "catchall" for other pvcreate/pvremove options
+ e.g. "-vv"
+
+3) Linux LVM VG (Volume Group)
+The format is:
+hostname,VG,vg name,operation mode,options,pv paths
+
+hostname hostname of the node in the cluster
+VG marker of VG line
+vg name name of the volume group, e.g. ost_vg
+operation mode create or remove, default is create
+options a "catchall" for other vgcreate/vgremove options
+ e.g. "-s 32M"
+pv paths physical volumes to construct this VG, required by
+ create mode
+ Multiple PVs are separated by space or by using
+ shell expansions, e.g. "/dev/sd[k-m]1"
+
+4) Linux LVM LV (Logical Volume)
+The format is:
+hostname,LV,lv name,operation mode,options,lv size,vg name
+
+hostname hostname of the node in the cluster
+LV marker of LV line
+lv name name of the logical volume to be created (optional)
+ or path of the logical volume to be removed (required
+ by remove mode)
+operation mode create or remove, default is create
+options a "catchall" for other lvcreate/lvremove options
+ e.g. "-i 2 -I 128"
+lv size size [kKmMgGtT] to be allocated for the new LV
+ Default unit is megabytes.
+vg name name of the VG in which the new LV will be created
+
+5) Lustre target
+The format is:
hostname,module_opts,device name,mount point,device type,fsname,mgs nids,index,
format options,mkfs options,mount options,failover nids
lustre-ost2,options lnet networks=tcp,/tmp/ost2,/mnt/ost2,ost,,"lustre-mgs1@tcp0:lustre-mgs2@tcp0",,--quiet --device-size=10240 --noformat,,,lustre-ost1@tcp0
-------------------------------------------------------------------------------
+Example 4 - Configure Linux MD/LVM devices before formatting Lustre targets:
+-------------------------------------------------------------------------------
+# MD device on mgsnode
+mgsnode,MD,/dev/md0,,-q,1,/dev/sda1 /dev/sdb1
+
+# MD/LVM devices on ostnode
+ostnode,MD,/dev/md0,,-q -c 128,5,"/dev/sd{a,b,c}"
+ostnode,MD,/dev/md1,,-q -c 128,5,"/dev/sd{d,e,f}"
+ostnode,PV,/dev/md0 /dev/md1
+ostnode,VG,ost_vg,,-s 32M,/dev/md0 /dev/md1
+ostnode,LV,ost0,,-i 2 -I 128,300G,ost_vg
+ostnode,LV,ost1,,-i 2 -I 128,300G,ost_vg
+
+# combo mgs/mdt
+mgsnode,options lnet networks=tcp,/dev/md0,/mnt/mgs,mgs|mdt,,,,--quiet
+
+# ost0
+ostnode,options lnet networks=tcp,/dev/ost_vg/ost0,/mnt/ost0,ost,,mgsnode,,--quiet
+
+# ost1
+ostnode,options lnet networks=tcp,/dev/ost_vg/ost1,/mnt/ost1,ost,,mgsnode,,--quiet
+-------------------------------------------------------------------------------
+
EOF
exit 0
}
VERIFY_CONNECT=true
+CONFIG_MD_LVM=false
MODIFY_FSTAB=true
VERBOSE_OUTPUT=false
# Get and check the positional parameters
-while getopts "t:nfmhv" OPTION; do
+while getopts "t:ndfmhv" OPTION; do
case $OPTION in
t)
HATYPE_OPT=$OPTARG
n)
VERIFY_CONNECT=false
;;
+ d)
+ CONFIG_MD_LVM=true
+ ;;
f)
REFORMAT_OPTION=$"--reformat "
;;
CSV_FILE=$1
local LINE
+ local marker
declare -i line_num=0
declare -i idx=0
continue
fi
+ # Skip the Linux MD/LVM line
+ marker=`echo ${LINE} | awk -F, '{print $2}'`
+ if [ "${marker}" = "${MD_MARKER}" -o "${marker}" = "${PV_MARKER}" ] \
+ || [ "${marker}" = "${VG_MARKER}" -o "${marker}" = "${LV_MARKER}" ]; then
+ continue
+ fi
+
# Parse the config line into CONFIG_ITEM
if ! parse_line $LINE; then
echo >&2 $"`basename $0`: parse_line() error: Occurred"\
echo
fi
+if ${CONFIG_MD_LVM}; then
+# Configure Linux MD/LVM devices
+ echo "`basename $0`: Configuring Linux MD/LVM devices..."
+ if ! ${SCRIPT_CONFIG_MD} ${VERBOSE_OPT} ${CSV_FILE}; then
+ exit 1
+ fi
+
+ if ! ${SCRIPT_CONFIG_LVM} ${VERBOSE_OPT} ${CSV_FILE}; then
+ exit 1
+ fi
+ echo "`basename $0`: Configure Linux MD/LVM devices OK!"
+ echo
+fi
+
# Configure the Lustre cluster
echo "`basename $0`: ******** Lustre cluster configuration START ********"
if ! get_items ${CSV_FILE}; then
git://git.whamcloud.com / fs / lustre-release.git / commitdiff