From 271d74a3daaad0b59ac47ecfcd19aff97923cb6e Mon Sep 17 00:00:00 2001 From: yujian Date: Fri, 14 Jul 2006 03:14:48 +0000 Subject: [PATCH] b=10677 r=nathan configure Linux MD and LVM devices from the CSV file --- lustre/scripts/.cvsignore | 2 + lustre/scripts/Makefile.am | 2 +- lustre/scripts/lc_common.sh | 26 ++ lustre/scripts/lc_lvm.sh.in | 565 +++++++++++++++++++++++++++++++++++++ lustre/scripts/lc_md.sh.in | 486 +++++++++++++++++++++++++++++++ lustre/scripts/lustre_config.sh.in | 124 +++++++- 6 files changed, 1199 insertions(+), 6 deletions(-) create mode 100644 lustre/scripts/lc_lvm.sh.in create mode 100644 lustre/scripts/lc_md.sh.in diff --git a/lustre/scripts/.cvsignore b/lustre/scripts/.cvsignore index 8d51d00..9dae161 100644 --- a/lustre/scripts/.cvsignore +++ b/lustre/scripts/.cvsignore @@ -14,3 +14,5 @@ lc_net.sh lc_modprobe.sh lc_hb.sh lc_cluman.sh +lc_md.sh +lc_lvm.sh diff --git a/lustre/scripts/Makefile.am b/lustre/scripts/Makefile.am index 8a3b3cd..287691a 100644 --- a/lustre/scripts/Makefile.am +++ b/lustre/scripts/Makefile.am @@ -4,7 +4,7 @@ # See the file COPYING in this distribution # These are scripts that are generated from .in files -genscripts = lustre_config.sh lc_modprobe.sh lc_net.sh lc_hb.sh lc_cluman.sh lustre_createcsv.sh +genscripts = lustre_config.sh lc_modprobe.sh lc_net.sh lc_hb.sh lc_cluman.sh lustre_createcsv.sh lc_md.sh lc_lvm.sh sbin_SCRIPTS = $(genscripts) lc_servip.sh lustre_up14.sh diff --git a/lustre/scripts/lc_common.sh b/lustre/scripts/lc_common.sh index cfe34e5..36781c0 100644 --- a/lustre/scripts/lc_common.sh +++ b/lustre/scripts/lc_common.sh @@ -27,6 +27,8 @@ GEN_HB_CONFIG=${SCRIPTS_PATH}/lc_hb.sh GEN_CLUMGR_CONFIG=${SCRIPTS_PATH}/lc_cluman.sh SCRIPT_VERIFY_SRVIP=${SCRIPTS_PATH}/lc_servip.sh SCRIPT_GEN_MONCF=${SCRIPTS_PATH}/lc_mon.sh +SCRIPT_CONFIG_MD=${SCRIPTS_PATH}/lc_md.sh +SCRIPT_CONFIG_LVM=${SCRIPTS_PATH}/lc_lvm.sh # Variables of HA software HBVER_HBV1="hbv1" # Heartbeat version 1 @@ -55,7 +57,17 @@ TMP_DIRS="${HB_TMP_DIR} ${CLUMGR_TMP_DIR}" FS_TYPE=${FS_TYPE:-"lustre"} # Lustre filesystem type FILE_SUFFIX=${FILE_SUFFIX:-".lustre"} # Suffix of the generated config files +# Marker of the MD device line +MD_MARKER=${MD_MARKER:-"MD"} + +# Marker of the LVM device line +PV_MARKER=${PV_MARKER:-"PV"} +VG_MARKER=${VG_MARKER:-"VG"} +LV_MARKER=${LV_MARKER:-"LV"} + declare -a CONFIG_ITEM # Items in each line of the csv file +declare -a NODE_NAME # Hostnames of nodes have been configured + # verbose_output string # Output verbose information $string @@ -178,3 +190,17 @@ fcanon() { echo "$NAME" fi } + +# configured_host host_name +# +# Check whether the devices in $host_name has been configured or not +configured_host() { + local host_name=$1 + declare -i i + + for ((i = 0; i < ${#NODE_NAME[@]}; i++)); do + [ "${host_name}" = "${NODE_NAME[i]}" ] && return 0 + done + + return 1 +} diff --git a/lustre/scripts/lc_lvm.sh.in b/lustre/scripts/lc_lvm.sh.in new file mode 100644 index 0000000..35b7a40 --- /dev/null +++ b/lustre/scripts/lc_lvm.sh.in @@ -0,0 +1,565 @@ +#!/bin/bash +# +# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: +# +# lc_lvm.sh - configure Linux LVM devices from a csv file +# +################################################################################ + +# Usage +usage() { + cat >&2 < + + This script is used to configure Linux LVM devices in a Lustre cluster + from a csv file. + + -h help and examples + -v verbose mode + csv file a spreadsheet that contains configuration parameters + (separated by commas) for each Linux LVM component + (PV, VG, LV) to be configured in a Lustre cluster + +EOF + exit 1 +} + +# Samples +sample() { + cat <&2 "`basename $0`: Missing csv file!" + usage +fi + +# check_lvm_item index +# +# Check the items required for managing LVM device ${LVM_NAME[index]} +check_lvm_item() { + # Check argument + if [ $# -eq 0 ]; then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "Missing argument!" + return 1 + fi + + declare -i i=$1 + + # Check hostname + if [ -z "${HOST_NAME[i]}" ]; then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "hostname item has null value!" + return 1 + fi + + # Check LVM device name + if [ -z "${LVM_NAME[i]}" ] \ + && [ "${LINE_MARKER[i]}" != "${LV_MARKER}" -a "${OP_MODE[i]}" != "remove" ] + then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "LVM component name item has null value!" + return 1 + fi + + # Check the operation mode + if [ -n "${OP_MODE[i]}" ] \ + && [ "${OP_MODE[i]}" != "create" -a "${OP_MODE[i]}" != "remove" ] + then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "Invalid operation mode item - \"${OP_MODE[i]}\"!" + return 1 + fi + + # Check items required by create mode + if [ -z "${OP_MODE[i]}" -o "${OP_MODE[i]}" = "create" ]; then + if [ "${LINE_MARKER[i]}" = "${VG_MARKER}" -a -z "${SIXTH_ITEM[i]}" ] + then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "pv paths item of vg ${LVM_NAME[i]} has null value!" + return 1 + fi + + if [ "${LINE_MARKER[i]}" = "${LV_MARKER}" ]; then + if [ -z "${SIXTH_ITEM[i]}" ]; then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "lv size item has null value!" + return 1 + fi + + if [ -z "${SEVENTH_ITEM[i]}" ]; then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "vg name item has null value!" + return 1 + fi + fi + fi + + return 0 +} + +# get_lvm_items csv_file +# +# Get all the LVM device items in the $csv_file and do some checks. +get_lvm_items() { + # Check argument + if [ $# -eq 0 ]; then + echo >&2 "`basename $0`: get_lvm_items() error: Missing csv file!" + return 1 + fi + + CSV_FILE=$1 + local LINE line_marker + declare -i line_num=0 + declare -i idx=0 + + while read -r LINE; do + let "line_num += 1" + + # Skip the comment line + [ -z "`echo \"${LINE}\" | egrep -v \"([[:space:]]|^)#\"`" ] && continue + + # Skip the non-LVM line + line_marker=`echo ${LINE} | awk -F, '{print $2}'` + [ "${line_marker}" != "${PV_MARKER}" ] \ + && [ "${line_marker}" != "${VG_MARKER}" ] \ + && [ "${line_marker}" != "${LV_MARKER}" ] && continue + + # Parse the config line into CONFIG_ITEM + if ! parse_line $LINE; then + return 1 + fi + + HOST_NAME[idx]=${CONFIG_ITEM[0]} + LINE_MARKER[idx]=${CONFIG_ITEM[1]} + LVM_NAME[idx]=${CONFIG_ITEM[2]} + OP_MODE[idx]=${CONFIG_ITEM[3]} + OP_OPTS[idx]=${CONFIG_ITEM[4]} + SIXTH_ITEM[idx]=${CONFIG_ITEM[5]} + SEVENTH_ITEM[idx]=${CONFIG_ITEM[6]} + + LVM_NAME[idx]=`echo "${LVM_NAME[idx]}" | sed 's/^"//' | sed 's/"$//'` + OP_OPTS[idx]=`echo "${OP_OPTS[idx]}" | sed 's/^"//' | sed 's/"$//'` + SIXTH_ITEM[idx]=`echo "${SIXTH_ITEM[idx]}" | sed 's/^"//' | sed 's/"$//'` + + # Check some required items + if ! check_lvm_item $idx; then + echo >&2 "`basename $0`: check_lvm_item() error:"\ + "Occurred on line ${line_num} in ${CSV_FILE}." + return 1 + fi + + let "idx += 1" + done < ${CSV_FILE} + + return 0 +} + +# construct_lvm_create_cmdline index +# +# Construct the creation command line for ${LVM_NAME[index]} +construct_lvm_create_cmdline() { + declare -i i=$1 + local lvm_cmd + + case "${LINE_MARKER[i]}" in + "${PV_MARKER}") + lvm_cmd="pvcreate -ff -y ${OP_OPTS[i]} ${LVM_NAME[i]}" + ;; + "${VG_MARKER}") + lvm_cmd="vgcreate ${OP_OPTS[i]} ${LVM_NAME[i]} ${SIXTH_ITEM[i]}" + ;; + "${LV_MARKER}") + if [ -z "${LVM_NAME[i]}" ]; then + lvm_cmd="lvcreate -L ${SIXTH_ITEM[i]} ${OP_OPTS[i]} ${SEVENTH_ITEM[i]}" + else + lvm_cmd="lvcreate -L ${SIXTH_ITEM[i]} -n ${LVM_NAME[i]} ${OP_OPTS[i]} ${SEVENTH_ITEM[i]}" + fi + ;; + esac + + echo ${lvm_cmd} + return 0 +} + +# cmdline_rm_LVs vg_name +# +# Construct command line to remove all the LVs on $vg_name. +# If $vg_name is null, then remove all the LVs in the host. +cmdline_rm_LVs() { + local vg_name=$1 + local lvm_rm_cmd + + lvm_rm_cmd="vgchange -a n ${vg_name} &&" + lvm_rm_cmd=${lvm_rm_cmd}" vgdisplay -v ${vg_name} | grep \"LV Name\" | awk '{print \$3}' |" + lvm_rm_cmd=${lvm_rm_cmd}" while read lv; do lvremove -f \$lv; done" + + echo ${lvm_rm_cmd} + return 0 +} + +# cmdline_rm_LV lv_path +# +# Construct command line to remove LV $lv_path +cmdline_rm_LV() { + local lv_path=$1 + local lvm_rm_cmd + + lvm_rm_cmd="lvchange -a n ${lv_path} && lvremove -f ${lv_path}" + echo ${lvm_rm_cmd} + return 0 +} + + +# cmdline_rm_VG vg_name +# +# Construct command line to remove VG $vg_name +cmdline_rm_VG() { + local vg_name=$1 + local lvm_rm_cmd + + # Remove all the LVs on this VG + lvm_rm_cmd=$(cmdline_rm_LVs ${vg_name}) + + # Remove this VG + lvm_rm_cmd=${lvm_rm_cmd}" && vgremove ${vg_name}" + echo ${lvm_rm_cmd} + return 0 +} + +# cmdline_rm_VGs +# +# Construct command line to remove all the VGs in the host +cmdline_rm_VGs() { + local lvm_rm_cmd + + # Remove all the LVs in the host + lvm_rm_cmd=$(cmdline_rm_LVs) + + # Remove all the VGs in the host + lvm_rm_cmd=${lvm_rm_cmd}" && vgdisplay | grep \"VG Name\" | awk '{print \$3}' |" + lvm_rm_cmd=${lvm_rm_cmd}" while read vg; do vgremove \$vg; done" + + echo ${lvm_rm_cmd} + return 0 +} + +# cmdline_rm_PVs +# +# Construct command line to remove all the PVs in the host +cmdline_rm_PVs() { + local lvm_rm_cmd + + # Remove all the LVs and VGs in the host + lvm_rm_cmd=$(cmdline_rm_VGs) + + # Remove all the PVs in the host + lvm_rm_cmd=${lvm_rm_cmd}" && pvdisplay | grep \"PV Name\" | awk '{print \$3}' |" + lvm_rm_cmd=${lvm_rm_cmd}" while read pv; do pvremove -ff -y \$pv; done" + + echo ${lvm_rm_cmd} + return 0 +} + +# construct_lvm_teardown_cmdline index +# +# Construct the teardown command line for LVM devices in ${HOST_NAME[index]} +construct_lvm_teardown_cmdline() { + declare -i i=$1 + local lvm_rm_cmd + + case "${LINE_MARKER[i]}" in + "${LV_MARKER}") + lvm_rm_cmd=$(cmdline_rm_LVs ${SEVENTH_ITEM[i]}) + ;; + "${VG_MARKER}") + # Remove all the VGs in the host + lvm_rm_cmd=$(cmdline_rm_VGs) + ;; + "${PV_MARKER}") + # Remove all the PVs in the host + lvm_rm_cmd=$(cmdline_rm_PVs) + ;; + esac + + echo ${lvm_rm_cmd} + return 0 +} + +# construct_lvm_rm_cmdline index +# +# Construct the remove command line for LVM device ${LVM_NAME[index]} +construct_lvm_rm_cmdline() { + declare -i i=$1 + local lvm_rm_cmd + + case "${LINE_MARKER[i]}" in + "${LV_MARKER}") + lvm_rm_cmd=$(cmdline_rm_LV ${LVM_NAME[i]}) + ;; + "${VG_MARKER}") + lvm_rm_cmd=$(cmdline_rm_VG ${LVM_NAME[i]}) + ;; + "${PV_MARKER}") + lvm_rm_cmd="pvremove -ff -y ${LVM_NAME[i]}" + ;; + esac + + echo ${lvm_rm_cmd} + return 0 +} + +# construct_lvm_cmdline host_name +# +# Construct the command line of LVM utilities to be run in the $host_name +construct_lvm_cmdline() { + LVM_CMDLINE= + local host_name=$1 + local lvm_cmd + declare -i i + + # Construct command line + for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do + lvm_cmd= + if [ "${host_name}" = "${HOST_NAME[i]}" ]; then + case "${OP_MODE[i]}" in + "" | create) + # Construct the create command line + lvm_cmd=$(construct_lvm_create_cmdline ${i}) + ;; + remove) + if [ -z "${LVM_NAME[i]}" ]; then + # Construct the teardown command line + lvm_cmd=$(construct_lvm_teardown_cmdline ${i}) + else # Remove instead of teardown + # Construct the remove command line + lvm_cmd=$(construct_lvm_rm_cmdline ${i}) + fi + ;; + *) + echo >&2 "`basename $0`: construct_lvm_cmdline() error:"\ + "Invalid operation mode - \"${OP_MODE[i]}\"!" + return 1 + ;; + esac + + if [ -z "${LVM_CMDLINE}" ]; then + LVM_CMDLINE=${lvm_cmd} + else + LVM_CMDLINE=${LVM_CMDLINE}" && "${lvm_cmd} + fi + fi + done + + return 0 +} + +# config_lvm_devs host_name +# +# Run remote command to configure LVM devices in $host_name +config_lvm_devs() { + local host_name=$1 + + # Construct the LVM utilities command line + if ! construct_lvm_cmdline ${host_name}; then + return 1 + fi + + if [ -z "${LVM_CMDLINE}" ]; then + verbose_output "There are no LVM devices on host ${host_name}"\ + "needed to be configured." + return 0 + fi + + # Run remote command to configure LVM devices in $host_name + verbose_output "Configuring LVM devices in host ${host_name}..." + verbose_output "Configure command line is: \"${LVM_CMDLINE}\"" + REMOTE_CMD[pid_num]="${REMOTE} ${host_name} \"${LVM_CMDLINE}\"" + ${REMOTE} ${host_name} "(${EXPORT_PATH} ${LVM_CMDLINE})" >&2 & + REMOTE_PID[pid_num]=$! + let "pid_num += 1" + + return 0 +} + +# Run remote command to configure all the LVM devices specified +# in the csv file +config_lvm() { + declare -i i=0 + declare -i idx=0 # Index of NODE_NAME array + local host_name + local failed_status + + # Initialize the NODE_NAME array + unset NODE_NAME + + for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do + host_name=${HOST_NAME[i]} + configured_host ${host_name} && continue + + NODE_NAME[idx]=${host_name} + let "idx += 1" + + # Run remote command to configure LVM devices in $host_name + if ! config_lvm_devs ${host_name}; then + return 1 + fi + done + + if [ ${#HOST_NAME[@]} -eq 0 -o ${#REMOTE_PID[@]} -eq 0 ]; then + verbose_output "There are no LVM devices to be configured." + return 0 + fi + + # Wait for the exit status of the background remote command + verbose_output "Waiting for the return of the remote command..." + failed_status=false + for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do + wait ${REMOTE_PID[${pid_num}]} + if [ $? -ne 0 ]; then + echo >&2 "`basename $0`: config_lvm() error: Failed"\ + "to execute \"${REMOTE_CMD[${pid_num}]}\"!" + failed_status=true + fi + done + + if ${failed_status}; then + return 1 + fi + + verbose_output "All the LVM devices are configured successfully!" + return 0 +} + +# Main flow +# Check the csv file +if ! check_file $1; then + exit 1 +fi + +# Get all the LVM device items from the csv file +if ! get_lvm_items ${CSV_FILE}; then + exit 1 +fi + +# Configure the LVM devices +if ! config_lvm; then + exit 1 +fi + +exit 0 diff --git a/lustre/scripts/lc_md.sh.in b/lustre/scripts/lc_md.sh.in new file mode 100644 index 0000000..a2a2228 --- /dev/null +++ b/lustre/scripts/lc_md.sh.in @@ -0,0 +1,486 @@ +#!/bin/bash +# +# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: +# +# lc_md.sh - configure Linux MD devices from a csv file +# +################################################################################ + +# Usage +usage() { + cat >&2 < + + This script is used to configure Linux MD devices in a Lustre cluster + from a csv file. + + -h help and examples + -v verbose mode + csv file a spreadsheet that contains configuration parameters + (separated by commas) for each Linux MD device to be + configured in a Lustre cluster + +EOF + exit 1 +} + +# Samples +sample() { + cat <&2 "`basename $0`: Missing csv file!" + usage +fi + +# check_md_item index +# +# Check the items required for managing MD device ${MD_NAME[index]} +check_md_item() { + # Check argument + if [ $# -eq 0 ]; then + echo >&2 "`basename $0`: check_md_item() error:"\ + "Missing argument!" + return 1 + fi + + declare -i i=$1 + + # Check hostname + if [ -z "${HOST_NAME[i]}" ]; then + echo >&2 "`basename $0`: check_md_item() error:"\ + "hostname item has null value!" + return 1 + fi + + # Check items required by create mode + if [ -z "${OP_MODE[i]}" -o "${OP_MODE[i]}" = "create" ]; then + # Check MD device name + if [ -z "${MD_NAME[i]}" ]; then + echo >&2 "`basename $0`: check_md_item() error:"\ + "md name item has null value!" + return 1 + fi + + if [ -z "${RAID_LEVEL[i]}" ]; then + echo >&2 "`basename $0`: check_md_item() error:"\ + "raid level item of MD device ${MD_NAME[i]} has null value!" + return 1 + fi + + if [ -z "${MD_DEVS[i]}" ]; then + echo >&2 "`basename $0`: check_md_item() error:"\ + "component devices item of ${MD_NAME[i]} has null value!" + return 1 + fi + fi + + return 0 +} + +# get_md_items csv_file +# +# Get all the MD device items in the $csv_file and do some checks. +get_md_items() { + # Check argument + if [ $# -eq 0 ]; then + echo >&2 "`basename $0`: get_md_items() error: Missing csv file!" + return 1 + fi + + CSV_FILE=$1 + local LINE + declare -i line_num=0 + declare -i idx=0 + + while read -r LINE; do + let "line_num += 1" + + # Skip the comment line + [ -z "`echo \"${LINE}\" | egrep -v \"([[:space:]]|^)#\"`" ] && continue + + # Skip the non-MD line + [ "`echo ${LINE}|awk -F, '{print $2}'`" != "${MD_MARKER}" ] && continue + + # Parse the config line into CONFIG_ITEM + if ! parse_line $LINE; then + return 1 + fi + + HOST_NAME[idx]=${CONFIG_ITEM[0]} + MD_NAME[idx]=${CONFIG_ITEM[2]} + OP_MODE[idx]=${CONFIG_ITEM[3]} + OP_OPTS[idx]=${CONFIG_ITEM[4]} + RAID_LEVEL[idx]=${CONFIG_ITEM[5]} + MD_DEVS[idx]=${CONFIG_ITEM[6]} + + OP_OPTS[idx]=`echo "${OP_OPTS[idx]}" | sed 's/^"//' | sed 's/"$//'` + MD_DEVS[idx]=`echo "${MD_DEVS[idx]}" | sed 's/^"//' | sed 's/"$//'` + + # Check some required items + if ! check_md_item $idx; then + echo >&2 "`basename $0`: check_md_item() error:"\ + "Occurred on line ${line_num} in ${CSV_FILE}." + return 1 + fi + + let "idx += 1" + done < ${CSV_FILE} + + return 0 +} + +# md_is_active host_name md_name +# +# Run remote command to check whether $md_name is active in @host_name +md_is_active() { + local host_name=$1 + local md_name=$2 + local cmd ret_str + + cmd="grep -q ${md_name##*/} /proc/mdstat 2>&1" + ret_str=`${REMOTE} ${host_name} "${cmd}" 2>&1` + if [ $? -ne 0 ]; then + if [ -n "${ret_str}" ]; then + echo >&2 "`basename $0`: md_is_active() error:"\ + "remote command to ${host_name} error: ${ret_str}!" + return 2 # Error occurred + else + return 1 # inactive + fi + fi + + return 0 # active +} + +# construct_mdadm_create_cmdline index +# +# Construct the create operation command line of mdadm for ${MD_NAME[index]} +construct_mdadm_create_cmdline() { + declare -i i=$1 + local cmd_line + local echo_disk disk line + declare -i alldisks=0 + declare -i raiddisks=0 + declare -i sparedisks=0 + + cmd_line="${MDADM} -C -R ${MD_NAME[i]} ${OP_OPTS[i]} -l ${RAID_LEVEL[i]}" + + if [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#* -n*}" ]\ + || [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#*--raid-devices*}" ]; then + cmd_line=${cmd_line}" ${MD_DEVS[i]}" + echo ${cmd_line} + return 0 + fi + + # FIXME: Get the number of component devices in the array + echo_disk="for disk in ${MD_DEVS[i]}; do echo $disk; done" + while read line; do + let "alldisks += 1" + done < <(${REMOTE} ${HOST_NAME[i]} "${echo_disk}") + + if [ ${alldisks} -eq 0 ]; then + echo "`basename $0`: construct_mdadm_create_cmdline() error:"\ + "Failed to execute remote command to get the number of"\ + "component devices of array ${MD_NAME[i]} from host ${HOST_NAME[i]}!" + return 1 + fi + + # Get the specified number of spare (eXtra) devices + if [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#* -x*}" ]; then + sparedisks=`echo ${OP_OPTS[i]##* -x}|awk -F" " '{print $1}'` + elif [ "${OP_OPTS[i]}" != "${OP_OPTS[i]#*--spare-devices*}" ]; then + sparedisks=`echo ${OP_OPTS[i]##*--spare-devices=}|awk -F" " '{print $1}'` + fi + + # Get the number of raid devices in the array + # The number of raid devices in the array plus the number of spare devices + # listed on the command line must equal the number of component devices + # (including "missing" devices). + let "raiddisks = alldisks - sparedisks" + + if [ ${raiddisks} -lt 1 ]; then + echo "`basename $0`: construct_mdadm_create_cmdline() error:"\ + "Invalid number of raid devices in array ${MD_NAME[i]}: ${raiddisks}!"\ + "Check the number of spare devices and whether all the component devices"\ + "\"${MD_DEVS[i]}\" (except \"missing\" devices) exist in host ${HOST_NAME[i]}!" + return 1 + fi + + cmd_line=${cmd_line}" -n ${raiddisks} ${MD_DEVS[i]}" + + echo ${cmd_line} + return 0 +} + +# construct_mdadm_rm_cmdline index +# +# Construct the remove operation command line of mdadm for ${MD_NAME[index]} +construct_mdadm_rm_cmdline() { + declare -i i=$1 + local mdadm_cmd + local real_devs + + # Deactivate the MD array, releasing all resources + mdadm_cmd="${MDADM} -S ${MD_NAME[i]}" + + if [ -n "${MD_DEVS[i]}" ]; then + # Remove the "missing" devices from the component devices + real_devs=`echo ${MD_DEVS[i]} | sed 's/missing//g'` + # Over-written the superblock with zeros + mdadm_cmd=${mdadm_cmd}" && ${MDADM} --zero-superblock ${real_devs}" + fi + + echo ${mdadm_cmd} + return 0 +} + +# construct_mdadm_cmdline host_name +# +# Construct the command line of mdadm to be run in $host_name +construct_mdadm_cmdline() { + MDADM_CMDLINE= + local host_name=$1 + local mdadm_stop_cmd mdadm_cmd + local rc OK + declare -i i + + # Construct command line + for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do + mdadm_stop_cmd= + mdadm_cmd= + if [ "${host_name}" = "${HOST_NAME[i]}" ]; then + case "${OP_MODE[i]}" in + "" | create) + # Check the status of the MD array + md_is_active ${host_name} ${MD_NAME[i]} + rc=$? + if [ "$rc" -eq "2" ]; then + return 1 + elif [ "$rc" -eq "0" ]; then + OK= + echo -n "`basename $0`: ${MD_NAME[i]} is active on"\ + "${host_name}, go ahead to deactivate it and create"\ + "the new array? [y/n]:" + read OK + if [ "${OK}" = "n" ]; then + echo "`basename $0`: ${MD_NAME[i]} on host"\ + "${host_name} remains as it is." + continue + fi + + # Construct the remove command line + mdadm_stop_cmd=$(construct_mdadm_rm_cmdline ${i}) + fi + + # Construct the create command line + mdadm_cmd=$(construct_mdadm_create_cmdline ${i}) + if [ $? -ne 0 ]; then + echo >&2 "${mdadm_cmd}" + return 1 + fi + + [ -n "${mdadm_stop_cmd}" ] && mdadm_cmd=${mdadm_stop_cmd}" && "${mdadm_cmd} + ;; + remove) + if [ -z "${MD_NAME[i]}" ]; then + OK= + echo -n "`basename $0`: Do you really want to remove"\ + "all the MD devices in the host ${HOST_NAME[i]}? [y/n]:" + read OK + if [ "${OK}" = "n" ]; then + echo "`basename $0`: MD devices on host"\ + "${HOST_NAME[i]} remain as they are." + continue + fi + + # Construct the teardown command line + mdadm_cmd="(cat /proc/mdstat | egrep \"^md[[:digit:]]\" |" + mdadm_cmd=${mdadm_cmd}" while read md rest; do ${MDADM} -S /dev/\$md; done)" + else + # Construct the remove command line + mdadm_cmd=$(construct_mdadm_rm_cmdline ${i}) + fi + ;; + *) + # Other operations + mdadm_cmd="${MDADM} ${OP_MODE[i]} ${MD_NAME[i]} ${OP_OPTS[i]} ${MD_DEVS[i]}" + ;; + esac + + if [ -z "${MDADM_CMDLINE}" ]; then + MDADM_CMDLINE=${mdadm_cmd} + else + MDADM_CMDLINE=${MDADM_CMDLINE}" && "${mdadm_cmd} + fi + fi + done + + return 0 +} + +# config_md_devs host_name +# +# Run remote command to configure MD devices in $host_name +config_md_devs() { + local host_name=$1 + + # Construct mdadm command line + if ! construct_mdadm_cmdline ${host_name}; then + return 1 + fi + + if [ -z "${MDADM_CMDLINE}" ]; then + verbose_output "There are no MD devices on host ${host_name}"\ + "needed to be configured." + return 0 + fi + + # Run remote command to configure MD devices in $host_name + verbose_output "Configuring MD devices in host ${host_name}..." + verbose_output "Configure command line is: \"${MDADM_CMDLINE}\"" + REMOTE_CMD[pid_num]="${REMOTE} ${host_name} \"${MDADM_CMDLINE}\"" + ${REMOTE} ${host_name} "${MDADM_CMDLINE}" >&2 & + REMOTE_PID[pid_num]=$! + let "pid_num += 1" + sleep 1 + + return 0 +} + +# Run remote command to configure all the MD devices specified in the csv file +config_md() { + declare -i i=0 + declare -i idx=0 # Index of NODE_NAME array + local host_name + local failed_status + + # Initialize the NODE_NAME array + unset NODE_NAME + + for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do + host_name=${HOST_NAME[i]} + configured_host ${host_name} && continue + + NODE_NAME[idx]=${host_name} + let "idx += 1" + + # Run remote command to configure MD devices in $host_name + if ! config_md_devs ${host_name}; then + return 1 + fi + done + + if [ ${#HOST_NAME[@]} -eq 0 -o ${#REMOTE_PID[@]} -eq 0 ]; then + verbose_output "There are no MD devices to be configured." + return 0 + fi + + # Wait for the exit status of the background remote command + verbose_output "Waiting for the return of the remote command..." + failed_status=false + for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do + wait ${REMOTE_PID[${pid_num}]} + if [ $? -ne 0 ]; then + echo >&2 "`basename $0`: config_md() error: Failed"\ + "to execute \"${REMOTE_CMD[${pid_num}]}\"!" + failed_status=true + fi + done + + if ${failed_status}; then + return 1 + fi + + verbose_output "All the MD devices are configured successfully!" + return 0 +} + +# Main flow +# Check the csv file +if ! check_file $1; then + exit 1 +fi + +# Get all the MD device items from the csv file +if ! get_md_items ${CSV_FILE}; then + exit 1 +fi + +# Configure the MD devices +if ! config_md; then + exit 1 +fi + +exit 0 diff --git a/lustre/scripts/lustre_config.sh.in b/lustre/scripts/lustre_config.sh.in index 1ee3a4d..92493b2 100644 --- a/lustre/scripts/lustre_config.sh.in +++ b/lustre/scripts/lustre_config.sh.in @@ -10,8 +10,8 @@ # that will be part of the Lustre cluster. # # In addition, it can also verify the network connectivity and hostnames in -# the cluster and produce High-Availability software configurations for -# Heartbeat or CluManager. +# the cluster, configure Linux MD/LVM devices and produce High-Availability +# software configurations for Heartbeat or CluManager. # ################################################################################ @@ -19,7 +19,7 @@ usage() { cat >&2 < +Usage: `basename $0` [-t HAtype] [-n] [-d] [-f] [-m] [-h] [-v] This script is used to format and set up multiple lustre servers from a csv file. @@ -32,6 +32,8 @@ Usage: `basename $0` [-t HAtype] [-n] [-f] [-m] [-h] [-v] and hbv2 (Heartbeat version 2). -n no net - don't verify network connectivity and hostnames in the cluster + -d configure Linux MD/LVM devices before formatting the + Lustre targets -f force-format the Lustre targets using --reformat option -m no fstab change - don't modify /etc/fstab to add the new Lustre targets @@ -56,11 +58,74 @@ that will be part of the Lustre cluster. It can also optionally: * verify the network connectivity and hostnames in the cluster + * configure Linux MD/LVM devices * modify /etc/modprobe.conf to add Lustre networking info * add the Lustre server info to /etc/fstab * produce configurations for Heartbeat or CluManager. -Each line in the csv file represents one Lustre target. The format is: +There are 5 kinds of line formats in the csv file. They represent the following +targets: +1) Linux MD device +The format is: +hostname,MD,md name,operation mode,options,raid level,component devices + +hostname hostname of the node in the cluster +MD marker of MD device line +md name MD device name, e.g. /dev/md0 +operation mode create or remove, default is create +options a "catchall" for other mdadm options, e.g. "-c 128" +raid level raid level: 0,1,4,5,6,10,linear and multipath +component devices block devices to be combined into the MD device + Multiple devices are separated by space or by using + shell expansions, e.g. "/dev/sd{a,b,c}" + +2) Linux LVM PV (Physical Volume) +The format is: +hostname,PV,pv names,operation mode,options + +hostname hostname of the node in the cluster +PV marker of PV line +pv names devices or loopback files to be initialized for later + use by LVM or to be wiped the label, e.g. /dev/sda + Multiple devices or files are separated by space or by + using shell expansions, e.g. "/dev/sd{a,b,c}" +operation mode create or remove, default is create +options a "catchall" for other pvcreate/pvremove options + e.g. "-vv" + +3) Linux LVM VG (Volume Group) +The format is: +hostname,VG,vg name,operation mode,options,pv paths + +hostname hostname of the node in the cluster +VG marker of VG line +vg name name of the volume group, e.g. ost_vg +operation mode create or remove, default is create +options a "catchall" for other vgcreate/vgremove options + e.g. "-s 32M" +pv paths physical volumes to construct this VG, required by + create mode + Multiple PVs are separated by space or by using + shell expansions, e.g. "/dev/sd[k-m]1" + +4) Linux LVM LV (Logical Volume) +The format is: +hostname,LV,lv name,operation mode,options,lv size,vg name + +hostname hostname of the node in the cluster +LV marker of LV line +lv name name of the logical volume to be created (optional) + or path of the logical volume to be removed (required + by remove mode) +operation mode create or remove, default is create +options a "catchall" for other lvcreate/lvremove options + e.g. "-i 2 -I 128" +lv size size [kKmMgGtT] to be allocated for the new LV + Default unit is megabytes. +vg name name of the VG in which the new LV will be created + +5) Lustre target +The format is: hostname,module_opts,device name,mount point,device type,fsname,mgs nids,index, format options,mkfs options,mount options,failover nids @@ -128,6 +193,29 @@ lustre-ost1,options lnet networks=tcp,/tmp/ost1,/mnt/ost1,ost,,"lustre-mgs1@tcp0 lustre-ost2,options lnet networks=tcp,/tmp/ost2,/mnt/ost2,ost,,"lustre-mgs1@tcp0:lustre-mgs2@tcp0",,--quiet --device-size=10240 --noformat,,,lustre-ost1@tcp0 ------------------------------------------------------------------------------- +Example 4 - Configure Linux MD/LVM devices before formatting Lustre targets: +------------------------------------------------------------------------------- +# MD device on mgsnode +mgsnode,MD,/dev/md0,,-q,1,/dev/sda1 /dev/sdb1 + +# MD/LVM devices on ostnode +ostnode,MD,/dev/md0,,-q -c 128,5,"/dev/sd{a,b,c}" +ostnode,MD,/dev/md1,,-q -c 128,5,"/dev/sd{d,e,f}" +ostnode,PV,/dev/md0 /dev/md1 +ostnode,VG,ost_vg,,-s 32M,/dev/md0 /dev/md1 +ostnode,LV,ost0,,-i 2 -I 128,300G,ost_vg +ostnode,LV,ost1,,-i 2 -I 128,300G,ost_vg + +# combo mgs/mdt +mgsnode,options lnet networks=tcp,/dev/md0,/mnt/mgs,mgs|mdt,,,,--quiet + +# ost0 +ostnode,options lnet networks=tcp,/dev/ost_vg/ost0,/mnt/ost0,ost,,mgsnode,,--quiet + +# ost1 +ostnode,options lnet networks=tcp,/dev/ost_vg/ost1,/mnt/ost1,ost,,mgsnode,,--quiet +------------------------------------------------------------------------------- + EOF exit 0 } @@ -150,10 +238,11 @@ declare -a MGS_NIDS INDEX FORMAT_OPTIONS MKFS_OPTIONS MOUNT_OPTIONS FAILOVERS VERIFY_CONNECT=true +CONFIG_MD_LVM=false MODIFY_FSTAB=true VERBOSE_OUTPUT=false # Get and check the positional parameters -while getopts "t:nfmhv" OPTION; do +while getopts "t:ndfmhv" OPTION; do case $OPTION in t) HATYPE_OPT=$OPTARG @@ -168,6 +257,9 @@ while getopts "t:nfmhv" OPTION; do n) VERIFY_CONNECT=false ;; + d) + CONFIG_MD_LVM=true + ;; f) REFORMAT_OPTION=$"--reformat " ;; @@ -640,6 +732,7 @@ get_items() { CSV_FILE=$1 local LINE + local marker declare -i line_num=0 declare -i idx=0 @@ -658,6 +751,13 @@ get_items() { continue fi + # Skip the Linux MD/LVM line + marker=`echo ${LINE} | awk -F, '{print $2}'` + if [ "${marker}" = "${MD_MARKER}" -o "${marker}" = "${PV_MARKER}" ] \ + || [ "${marker}" = "${VG_MARKER}" -o "${marker}" = "${LV_MARKER}" ]; then + continue + fi + # Parse the config line into CONFIG_ITEM if ! parse_line $LINE; then echo >&2 $"`basename $0`: parse_line() error: Occurred"\ @@ -1024,6 +1124,20 @@ if ${VERIFY_CONNECT}; then echo fi +if ${CONFIG_MD_LVM}; then +# Configure Linux MD/LVM devices + echo "`basename $0`: Configuring Linux MD/LVM devices..." + if ! ${SCRIPT_CONFIG_MD} ${VERBOSE_OPT} ${CSV_FILE}; then + exit 1 + fi + + if ! ${SCRIPT_CONFIG_LVM} ${VERBOSE_OPT} ${CSV_FILE}; then + exit 1 + fi + echo "`basename $0`: Configure Linux MD/LVM devices OK!" + echo +fi + # Configure the Lustre cluster echo "`basename $0`: ******** Lustre cluster configuration START ********" if ! get_items ${CSV_FILE}; then -- 1.8.3.1