lustre/scripts/lustre_createcsv.in

   1 #!/bin/bash
   2
   3 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
   4
   5 #
   6 # lustre_createcsv - generate a csv file from a running lustre cluster
   7 #
   8 # This script is used to collect lustre target informations, linux MD/LVM device
   9 # informations and HA software configurations in a lustre cluster to generate a
  10 # csv file. In reverse, the csv file could be parsed by lustre_config to
  11 # configure multiple lustre servers in parallel.
  12 #
  13 # This script should be run on the MGS node.
  14 #
  15 ################################################################################
  16
  17 # Usage
  18 usage() {
  19         cat <<EOF
  20
  21 Usage:  `basename $0` [-t HAtype] [-d] [-h] [-v] [-f csv_filename]
  22
  23         This script is used to collect lustre target informations, linux MD/LVM
  24         device informations and HA software configurations from a running lustre
  25         cluster to generate a csv file. It should be run on the MGS node.
  26
  27         -t HAtype       collect High-Availability software configurations
  28                         The argument following -t is used to indicate the High-
  29                         Availability software type. The HA software types which
  30                         are currently supported are: hbv1 (Heartbeat version 1)
  31                         and hbv2 (Heartbeat version 2).
  32         -d              collect linux MD/LVM device informations
  33         -h              help
  34         -v              verbose mode
  35         -f csv_filename designate a name for the csv file
  36                         Default is lustre_config.csv.
  37
  38 EOF
  39 }
  40
  41 # Get the library of functions
  42 . @scriptlibdir@/lc_common
  43
  44 #**************************** Global variables ****************************#
  45 # csv file
  46 LUSTRE_CSV_FILE=${LUSTRE_CSV_FILE:-"lustre_config.csv"}
  47
  48 # Lustre proc files
  49 LUSTRE_PROC=${LUSTRE_PROC:-"/proc/fs/lustre"}
  50 LUSTRE_PROC_DEVICES=${LUSTRE_PROC}/devices
  51
  52 LNET_PROC=${LNET_PROC:-"/proc/sys/lnet"}
  53 LNET_PROC_PEERS=${LNET_PROC}/peers
  54
  55 # Default network module options
  56 DEFAULT_MOD_OPTS=${DEFAULT_MOD_OPTS:-"options lnet networks=tcp"}
  57
  58 # Lustre target obd device types
  59 MGS_TYPE=${MGS_TYPE:-"mgs"}
  60 MDT_TYPE=${MDT_TYPE:-"mds"}
  61 OST_TYPE=${OST_TYPE:-"obdfilter"}
  62
  63 # The obd name of MGS target server
  64 MGS_SVNAME=${MGS_SVNAME:-"MGS"}
  65
  66 # Hostnames of the lustre cluster nodes
  67 declare -a HOST_NAMES
  68 MGS_HOSTNAME=${MGS_HOSTNAME:-"`hostname`"} # Hostname of the MGS node
  69
  70 # Configs of lustre targets in one cluster node
  71 declare -a TARGET_CONFIGS
  72 declare -a TARGET_SVNAMES TARGET_DEVNAMES TARGET_DEVSIZES TARGET_MNTPNTS
  73 declare -a TARGET_DEVTYPES TARGET_FSNAMES TARGET_MGSNIDS TARGET_INDEXES
  74 declare -a TARGET_FMTOPTS TARGET_MKFSOPTS TARGET_MNTOPTS TARGET_FAILNIDS
  75 declare -a HA_CONFIGS
  76 declare -a ALL_TARGET_SVNAMES           # All the target services in the cluster
  77 declare -a FAILOVER_FMTOPTS             # "--noformat"
  78
  79 # Informations of linux MD/LVM devices in one cluster node
  80 declare -a MD_NAME MD_LEVEL MD_DEVS     # MD
  81 declare -a VG_NAME VG_PVNAMES           # VG
  82 declare -a LV_NAME LV_SIZE LV_VGNAME    # LV
  83
  84 # Lustre target service types
  85 let "SVTYPE_MDT = 0x0001"
  86 let "SVTYPE_OST = 0x0002"
  87 let "SVTYPE_MGS = 0x0004"
  88
  89 # Permanent mount options for ext3 or ldiskfs
  90 ALWAYS_MNTOPTS=${ALWAYS_MNTOPTS:-"errors=remount-ro"}
  91 MDT_MGS_ALWAYS_MNTOPTS=${MDT_MGS_ALWAYS_MNTOPTS:-",iopen_nopriv,user_xattr"}
  92 OST_ALWAYS_MNTOPTS=${OST_ALWAYS_MNTOPTS:-",asyncdel"}
  93 OST_DEFAULT_MNTOPTS=${OST_DEFAULT_MNTOPTS:-",extents,mballoc"}
  94
  95 # User-settable parameter keys
  96 PARAM_MGSNODE=${PARAM_MGSNODE:-"mgsnode="}
  97 PARAM_FAILNODE=${PARAM_FAILNODE:-"failover.node="}
  98
  99 # Block size
 100 L_BLOCK_SIZE=4096
 101
 102 # Option string of mkfs.lustre
 103 OPTSTR_STRIPE_COUNT=${OPTSTR_STRIPE_COUNT:-"--stripe-count-hint="}
 104
 105
 106 # Get and check the positional parameters
 107 VERBOSE_OUTPUT=false
 108 GET_MDLVM_INFO=false
 109 while getopts "t:dhvf:" OPTION; do
 110         case $OPTION in
 111         t)
 112                 HATYPE_OPT=$OPTARG
 113                 if [ "${HATYPE_OPT}" != "${HBVER_HBV1}" ] \
 114                 && [ "${HATYPE_OPT}" != "${HBVER_HBV2}" ] \
 115                 && [ "${HATYPE_OPT}" != "${HATYPE_CLUMGR}" ]; then
 116                         error_output "Invalid HA software type" \
 117                                  "- ${HATYPE_OPT}!"
 118                         usage 1>&2
 119             exit 1
 120                 fi
 121                 ;;
 122         d)      GET_MDLVM_INFO=true;;
 123         h)      usage && exit 0;;
 124         v)      VERBOSE_OUTPUT=true;;
 125         f)      LUSTRE_CSV_FILE=$OPTARG;;
 126     ?)  usage 1>&2 && exit 1;;
 127         esac
 128 done
 129
 130 # Verify the local host is the MGS node
 131 mgs_node() {
 132         if [ ! -e ${LUSTRE_PROC_DEVICES} ]; then
 133                 error_output "${LUSTRE_PROC_DEVICES} does" \
 134                          "not exist. Lustre kernel modules may not be loaded!"
 135                 return 1
 136         fi
 137
 138         if [ -z "`cat ${LUSTRE_PROC_DEVICES}`" ]; then
 139                 error_output "${LUSTRE_PROC_DEVICES} is" \
 140                          "empty. Lustre services may not be started!"
 141                 return 1
 142         fi
 143
 144         if [ -z "`grep ${MGS_TYPE} ${LUSTRE_PROC_DEVICES}`" ]; then
 145                 error_output "This node is not a MGS node." \
 146                          "The script should be run on the MGS node!"
 147                 return 1
 148         fi
 149
 150         return 0
 151 }
 152
 153 # get_hostnames
 154 # Get lustre cluster node names
 155 get_hostnames() {
 156         declare -a HOST_NIDS
 157         declare -i idx          # Index of HOST_NIDS array
 158         declare -i i            # Index of HOST_NAMES array
 159
 160         if ! mgs_node; then
 161                 return 1
 162         fi
 163
 164         if [ ! -e ${LNET_PROC_PEERS} ]; then
 165                 error_output "${LNET_PROC_PEERS} does not" \
 166                          "exist. LNET kernel modules may not be loaded" \
 167                          "or LNET network may not be up!"
 168                 return 1
 169         fi
 170
 171         HOST_NAMES[0]=${MGS_HOSTNAME} # MGS node
 172         HOST_NIDS[0]=${HOST_NAMES[0]}
 173
 174         # Get the nids of the nodes which have contacted MGS
 175         idx=1
 176         for nid in `cat ${LNET_PROC_PEERS} | awk '{print $1}'`; do
 177                 if [ "${nid}" = "nid" ]; then
 178                         continue
 179                 fi
 180
 181                 HOST_NIDS[idx]=${nid}
 182                 let "idx += 1"
 183         done
 184
 185         if [ ${idx} -eq 1 ]; then
 186                 verbose_output "Only one node running in the lustre cluster." \
 187                                "It's ${HOST_NAMES[0]}."
 188                 return 0
 189         fi
 190
 191         # Get the hostnames of the nodes
 192         for ((idx = 1, i = 1; idx < ${#HOST_NIDS[@]}; idx++, i++)); do
 193                 if [ -z "${HOST_NIDS[idx]}" ]; then
 194                         error_output "get_hostnames():" \
 195                                  "Invalid nid - \"${HOST_NIDS[idx]}\"!"
 196                         return 1
 197                 fi
 198
 199                 HOST_NAMES[i]=$(nid2hostname ${HOST_NIDS[idx]})
 200                 if [ $? -ne 0 ]; then
 201                         error_output "${HOST_NAMES[i]}"
 202                         return 1
 203                 fi
 204
 205                 if [ "${HOST_NAMES[i]}" = "${HOST_NAMES[0]}" ]; then
 206                         unset HOST_NAMES[i]
 207                         let "i -= 1"
 208                 fi
 209         done
 210
 211         return 0
 212 }
 213
 214 #********************** Linux MD/LVM device informations **********************#
 215 # get_md_configs hostname
 216 # Get all the active MD device informations from the node @hostname
 217 get_md_configs() {
 218         declare -i i=0
 219         declare -i j=0
 220         local host_name=$1
 221         local ret_line line first_item
 222
 223         # Initialize the arrays
 224         unset MD_NAME
 225         unset MD_LEVEL
 226         unset MD_DEVS
 227
 228         # Execute remote command to the node ${host_name} and get all the
 229         # active MD device informations.
 230         while read -r ret_line; do
 231                 if is_pdsh; then
 232                         set -- ${ret_line}
 233                         shift
 234                         line="$*"
 235                 else
 236                         line="${ret_line}"
 237                 fi
 238
 239                 first_item=`echo "${line}" | awk '{print $1}'`
 240
 241                 # Get the MD device name and raid level
 242                 if [ "${first_item}" = "ARRAY" ]; then
 243                         MD_NAME[i]=`echo "${line}" | awk '{print $2}'`
 244                         MD_LEVEL[i]=`echo "${line}" | awk '{print $3}' | sed -e 's/level=//'`
 245                         let "j = i"
 246                         let "i += 1"
 247                 fi
 248
 249                 # Get the MD component devices
 250                 if [ "${first_item}" != "${first_item#devices=}" ]; then
 251                         MD_DEVS[j]=`echo "${line}" | sed -e 's/devices=//' -e 's/,/ /g'`
 252                 fi
 253         done < <(${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin
 254         ${MDADM} --detail --scan --verbose")
 255
 256         if [ $i -eq 0 ]; then
 257                 verbose_output "There are no active MD devices" \
 258                                "in the host ${host_name}!"
 259         fi
 260
 261         return 0
 262 }
 263
 264 # get_pv_configs hostname
 265 # Get all the LVM PV informations from the node @hostname
 266 get_pv_configs() {
 267         PV_NAMES=
 268         local host_name=$1
 269         local cmd ret_str
 270
 271         # Execute remote command to get all the PV informations.
 272         cmd="PATH=\$PATH:/sbin:/usr/sbin \
 273 pvdisplay -c | awk -F: '{print \$1}' | xargs"
 274         ret_str=`${REMOTE} ${host_name} "${cmd}" 2>&1`
 275         if [ $? -ne 0 ]; then
 276                 if [ -n "${ret_str}" ]; then
 277                         error_output "get_pv_configs():" \
 278                         "remote command to ${host_name} error: ${ret_str}"
 279                 else
 280                         remote_error "get_pv_configs" ${host_name}
 281                 fi
 282                 return 1
 283         fi
 284
 285         PV_NAMES=`echo "${ret_str}" | sed -e 's/^'${host_name}':[[:space:]]//'`
 286         if [ -z "${PV_NAMES}" ]; then
 287                 verbose_output "There are no PVs in the host ${host_name}!"
 288                 return 0
 289         fi
 290
 291         return 0
 292 }
 293
 294 # get_vg_pvnames hostname vgname
 295 # Get the PVs contained in @vgname from the node @hostname
 296 get_vg_pvnames() {
 297         local host_name=$1
 298         local vg_name=$2
 299         local pv_names=
 300         local cmd ret_str
 301
 302         # Execute remote command to get the PV names.
 303         cmd="PATH=\$PATH:/sbin:/usr/sbin vgdisplay -v ${vg_name} 2>/dev/null\
 304              | grep \"PV Name\" | awk '{print \$3}' | xargs"
 305         ret_str=`${REMOTE} ${host_name} "${cmd}" 2>&1`
 306         if [ $? -ne 0 ]; then
 307                 if [ -n "${ret_str}" ]; then
 308                         echo "`basename $0`: get_vg_pvnames() error:" \
 309                         "remote command to ${host_name} error: ${ret_str}"
 310                 else
 311                         remote_error "get_vg_pvnames" ${host_name}
 312                 fi
 313                 return 1
 314         fi
 315
 316         pv_names=`echo "${ret_str}" | sed -e 's/^'${host_name}':[[:space:]]//'`
 317         if [ -z "${pv_names}" ]; then
 318                 echo "`basename $0`: get_vg_pvnames() error:" \
 319                 "There are no PVs in VG ${vg_name} in the host ${host_name}!"\
 320                 "Or VG ${vg_name} does not exist."
 321                 return 1
 322         fi
 323
 324         echo "${pv_names}"
 325         return 0
 326 }
 327
 328 # get_vg_configs hostname
 329 # Get all the LVM VG informations from the node @hostname
 330 get_vg_configs() {
 331         declare -i i=0
 332         local host_name=$1
 333         local cmd ret_str
 334         local vg_name
 335
 336         # Initialize the arrays
 337         unset VG_NAME
 338         unset VG_PVNAMES
 339
 340         # Execute remote command to get all the VG names.
 341         cmd="PATH=\$PATH:/sbin:/usr/sbin vgdisplay \
 342              | grep \"VG Name\" | awk '{print \$3}' | xargs"
 343         ret_str=`${REMOTE} ${host_name} "${cmd}" 2>&1`
 344         if [ $? -ne 0 ]; then
 345                 if [ -n "${ret_str}" ]; then
 346                         error_output "get_vg_configs():" \
 347                         "remote command to ${host_name} error: ${ret_str}"
 348                 else
 349                         remote_error "get_vg_configs" ${host_name}
 350                 fi
 351                 return 1
 352         fi
 353
 354         if [ -z "${ret_str}" ] \
 355         || [ "${ret_str}" != "${ret_str#*No volume groups found*}" ]; then
 356                 verbose_output "There are no VGs in the host ${host_name}!"
 357                 return 0
 358         fi
 359
 360         # Get all the VG informations
 361         for vg_name in `echo "${ret_str}" | sed -e 's/^'${host_name}'://'`; do
 362                 VG_NAME[i]=${vg_name}
 363                 VG_PVNAMES[i]=$(get_vg_pvnames ${host_name} ${VG_NAME[i]})
 364                 if [ $? -ne 0 ]; then
 365                         error_output "${VG_PVNAMES[i]}"
 366                         return 1
 367                 fi
 368                 let "i += 1"
 369         done
 370
 371         return 0
 372 }
 373
 374 # get_lv_configs hostname
 375 # Get all the LVM LV informations from the node @hostname
 376 get_lv_configs() {
 377         declare -i i=0
 378         local host_name=$1
 379         local ret_line line
 380
 381         # Initialize the arrays
 382         unset LV_NAME
 383         unset LV_SIZE
 384         unset LV_VGNAME
 385
 386         # Execute remote command to get all the LV informations.
 387         while read -r ret_line; do
 388                 if is_pdsh; then
 389                         set -- ${ret_line}
 390                         shift
 391                         line="$*"
 392                 else
 393                         line="${ret_line}"
 394                 fi
 395
 396                 [ "${line}" != "${line#*volume group*}" ] && break
 397
 398                 LV_NAME[i]=`echo "${line}" | awk -F: '{print $1}' | sed -e 's/.*\///g'`
 399                 LV_VGNAME[i]=`echo "${line}" | awk -F: '{print $2}'`
 400                 LV_SIZE[i]=`echo "${line}" | awk -F: '{print $7}' | sed -e 's/.*/&K/'`
 401
 402                 let "i += 1"
 403         done < <(${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin lvdisplay -c")
 404
 405         if [ $i -eq 0 ]; then
 406                 verbose_output "There are no LVs in the host ${host_name}"
 407         fi
 408
 409         return 0
 410 }
 411
 412 #*************************** Network module options ***************************#
 413 # last_is_backslash line
 414 # Check whether the last effective letter of @line is a backslash
 415 last_is_backslash() {
 416         local line="$*"
 417         declare -i i
 418         declare -i length
 419         local letter last_letter
 420
 421         length=${#line}
 422         for ((i = ${length}-1; i >= 0; i--)); do
 423                 letter=${line:${i}:1}
 424                 [ "x${letter}" != "x " -a "x${letter}" != "x    " -a -n "${letter}" ]\
 425                 && last_letter=${letter} && break
 426         done
 427
 428         [ "x${last_letter}" = "x\\" ] && return 0
 429
 430         return 1
 431 }
 432
 433 # get_module_opts hostname
 434 # Get the network module options from the node @hostname
 435 get_module_opts() {
 436         local host_name=$1
 437         local ret_str
 438         local MODULE_CONF KERNEL_VER
 439         local ret_line line find_options
 440         local continue_flag
 441
 442         MODULE_OPTS=${DEFAULT_MOD_OPTS}
 443
 444         # Execute remote command to get the kernel version
 445         ret_str=`${REMOTE} ${host_name} "uname -r" 2>&1`
 446         if [ $? -ne 0 -a -n "${ret_str}" ]; then
 447                 error_output "get_module_opts():" \
 448                          "remote command error: ${ret_str}"
 449                 return 1
 450         fi
 451         remote_error "get_module_opts" ${host_name} "${ret_str}" && return 1
 452
 453         if is_pdsh; then
 454                 KERNEL_VER=`echo ${ret_str} | awk '{print $2}'`
 455         else
 456                 KERNEL_VER=`echo ${ret_str} | awk '{print $1}'`
 457         fi
 458
 459         # Get the module configuration file name
 460         if [ "${KERNEL_VER:0:3}" = "2.4" ]; then
 461                 MODULE_CONF=/etc/modules.conf
 462         else
 463                 MODULE_CONF=/etc/modprobe.conf
 464         fi
 465
 466         # Execute remote command to get the lustre network module options
 467         continue_flag=false
 468         find_options=false
 469         while read -r ret_line; do
 470                 if is_pdsh; then
 471                         set -- ${ret_line}
 472                         shift
 473                         line="$*"
 474                 else
 475                         line="${ret_line}"
 476                 fi
 477
 478                 # Get rid of the comment line
 479                 [ -z "`echo \"${line}\"|egrep -v \"^#\"`" ] && continue
 480
 481                 if [ "${line}" != "${line#*options lnet*}" ]; then
 482                         if ! ${find_options}; then
 483                                 find_options=true
 484                                 MODULE_OPTS=${line}
 485                         else
 486                                 MODULE_OPTS=${MODULE_OPTS}$" \n "${line}
 487                         fi
 488
 489                         last_is_backslash "${line}" && continue_flag=true \
 490                         || continue_flag=false
 491                         continue
 492                 fi
 493
 494                 if ${continue_flag}; then
 495                         MODULE_OPTS=${MODULE_OPTS}$" \n "${line}
 496                         ! last_is_backslash "${line}" && continue_flag=false
 497
 498                 fi
 499         done < <(${REMOTE} ${host_name} "cat ${MODULE_CONF}")
 500
 501         if [ -z "${MODULE_OPTS}" ]; then
 502                 MODULE_OPTS=${DEFAULT_MOD_OPTS}
 503         fi
 504
 505         return 0
 506 }
 507
 508 #************************ HA software configurations ************************#
 509 # is_ha_target hostname target_devname
 510 # Check whether the target @target_devname was made to be high-available
 511 is_ha_target() {
 512         local host_name=$1
 513         local target_svname=$2
 514         local res_file
 515         local ret_str
 516
 517         case "${HATYPE_OPT}" in
 518         "${HBVER_HBV1}")        res_file=${HA_RES};;
 519         "${HBVER_HBV2}")        res_file=${HA_CIB};;
 520         "${HATYPE_CLUMGR}")     res_file=${CLUMAN_CONFIG};;
 521         esac
 522
 523         # Execute remote command to check the resource file
 524         ret_str=`${REMOTE} ${host_name} \
 525                 "grep ${target_svname} ${res_file}" 2>&1`
 526         if [ $? -ne 0 -a -n "${ret_str}" ]; then
 527                 error_output "is_ha_target():" \
 528                          "remote command error: ${ret_str}"
 529                 return 1
 530         fi
 531
 532         [ "${ret_str}" = "${ret_str#*${target_svname}*}" ] && return 1
 533
 534         return 0
 535 }
 536
 537 # get_hb_configs hostname
 538 # Get the Heartbeat configurations from the node @hostname
 539 get_hb_configs() {
 540         local host_name=$1
 541         local ret_line line
 542         declare -i i
 543
 544         unset HA_CONFIGS
 545         HB_CHANNELS=
 546         SRV_IPADDRS=
 547         HB_OPTIONS=
 548
 549         # Execute remote command to get the configs of Heartbeat channels, etc
 550         while read -r ret_line; do
 551                 if is_pdsh; then
 552                         set -- ${ret_line}
 553                         shift
 554                         line="$*"
 555                 else
 556                         line="${ret_line}"
 557                 fi
 558
 559                 # Get rid of the comment line
 560                 [ -z "`echo \"${line}\"|egrep -v \"^#\"`" ] && continue
 561
 562                 if [ "${line}" != "${line#*serial*}" ] \
 563                 || [ "${line}" != "${line#*cast*}" ]; then
 564                         if [ -z "${HB_CHANNELS}" ]; then
 565                                 HB_CHANNELS=${line}
 566                         else
 567                                 HB_CHANNELS=${HB_CHANNELS}:${line}
 568                         fi
 569                 fi
 570
 571                 if [ "${line}" != "${line#*stonith*}" ] \
 572                 || [ "${line}" != "${line#*ping*}" ] \
 573                 || [ "${line}" != "${line#*respawn*}" ] \
 574                 || [ "${line}" != "${line#*apiauth*}" ] \
 575                 || [ "${line}" != "${line#*compression*}" ]; then
 576                         if [ -z "${HB_OPTIONS}" ]; then
 577                                 HB_OPTIONS=${line}
 578                         else
 579                                 HB_OPTIONS=${HB_OPTIONS}:${line}
 580                         fi
 581                 fi
 582         done < <(${REMOTE} ${host_name} "cat ${HA_CF}")
 583
 584         if [ -z "${HB_CHANNELS}" ]; then
 585                 error_output "get_hb_configs():" \
 586                          "There are no heartbeat channel configs in ${HA_CF}" \
 587                          "of host ${host_name} or ${HA_CF} does not exist!"
 588                 return 0
 589         fi
 590
 591         # Execute remote command to get Heartbeat service address
 592         if [ "${HATYPE_OPT}" = "${HBVER_HBV1}" ]; then
 593                 while read -r ret_line; do
 594                         if is_pdsh; then
 595                                 set -- ${ret_line}
 596                                 shift
 597                                 line="$*"
 598                         else
 599                                 line="${ret_line}"
 600                         fi
 601
 602                         # Get rid of the empty line
 603                         [ -z "`echo ${line}|awk '/[[:alnum:]]/ {print $0}'`" ]\
 604                         && continue
 605
 606                         # Get rid of the comment line
 607                         [ -z "`echo \"${line}\"|egrep -v \"^#\"`" ] && continue
 608
 609                         SRV_IPADDRS=`echo ${line} | awk '{print $2}'`
 610                         [ -n "${SRV_IPADDRS}" ] \
 611                         && [ "`echo ${line} | awk '{print $1}'`" = "${host_name}" ] && break
 612                 done < <(${REMOTE} ${host_name} "cat ${HA_RES}")
 613
 614                 if [ -z "${SRV_IPADDRS}" ]; then
 615                         error_output "get_hb_configs(): There"\
 616                                  "are no service address in ${HA_RES} of host"\
 617                                  "${host_name} or ${HA_RES} does not exist!"
 618                         return 0
 619                 fi
 620         fi
 621
 622         # Construct HA configuration items
 623         for ((i = 0; i < ${#TARGET_DEVNAMES[@]}; i++)); do
 624                 [ -z "${TARGET_DEVNAMES[i]}" ] && continue
 625
 626                 # Execute remote command to check whether this target service
 627                 # was made to be high-available
 628                 if is_ha_target ${host_name} ${TARGET_DEVNAMES[i]}; then
 629                         HA_CONFIGS[i]=${HB_CHANNELS},${SRV_IPADDRS},${HB_OPTIONS}
 630                 fi
 631         done
 632
 633         return 0
 634 }
 635
 636 # get_cluman_channel hostname
 637 # Get the Heartbeat channel of CluManager from the node @hostname
 638 get_cluman_channel() {
 639         local host_name=$1
 640         local ret_line line
 641         local cluman_channel=
 642         local mcast_ipaddr
 643
 644         while read -r ret_line; do
 645                 if is_pdsh; then
 646                         set -- ${ret_line}
 647                         shift
 648                         line="$*"
 649                 else
 650                         line="${ret_line}"
 651                 fi
 652
 653                 if [ "${line}" != "${line#*broadcast*}" ] \
 654                 && [ "`echo ${line}|awk '{print $3}'`" = "yes" ]; then
 655                         cluman_channel="broadcast"
 656                         break
 657                 fi
 658
 659                 if [ "${line}" != "${line#*multicast_ipaddress*}" ]; then
 660                         mcast_ipaddr=`echo ${line}|awk '{print $3}'`
 661                         if [ "${mcast_ipaddr}" != "225.0.0.11" ]; then
 662                                 cluman_channel="multicast ${mcast_ipaddr}"
 663                                 break
 664                         fi
 665                 fi
 666         done < <(${REMOTE} ${host_name} "${CONFIG_CMD} --clumembd")
 667
 668         echo ${cluman_channel}
 669         return 0
 670 }
 671
 672 # get_cluman_srvaddr hostname target_svname
 673 # Get the service IP addresses of @target_svname from the node @hostname
 674 get_cluman_srvaddr() {
 675         local host_name=$1
 676         local target_svname=$2
 677         local ret_line line
 678         local srvaddr cluman_srvaddr=
 679
 680         while read -r ret_line; do
 681                 if is_pdsh; then
 682                         set -- ${ret_line}
 683                         shift
 684                         line="$*"
 685                 else
 686                         line="${ret_line}"
 687                 fi
 688
 689                 if [ "${line}" != "${line#*ipaddress = *}" ]; then
 690                         srvaddr=`echo ${line}|awk '{print $3}'`
 691                         if [ -z "${cluman_srvaddr}" ]; then
 692                                 cluman_srvaddr=${srvaddr}
 693                         else
 694                                 cluman_srvaddr=${cluman_srvaddr}:${srvaddr}
 695                         fi
 696                 fi
 697         done < <(${REMOTE} ${host_name} "${CONFIG_CMD} \
 698                 --service=${target_svname} --service_ipaddresses")
 699
 700         if [ -z "${cluman_srvaddr}" ]; then
 701                 echo "`basename $0`: get_cluman_srvaddr() error: Cannot" \
 702                 "get the service IP addresses of ${target_svname} in" \
 703                 "${host_name}! Check ${CONFIG_CMD} command!"
 704                 return 1
 705         fi
 706
 707         echo ${cluman_srvaddr}
 708         return 0
 709 }
 710
 711 # get_cluman_configs hostname
 712 # Get the CluManager configurations from the node @hostname
 713 get_cluman_configs() {
 714         local host_name=$1
 715         local ret_str
 716         declare -i i
 717
 718         unset HA_CONFIGS
 719
 720         # Execute remote command to get the configs of CluManager
 721         for ((i = 0; i < ${#TARGET_DEVNAMES[@]}; i++)); do
 722                 HB_CHANNELS=
 723                 SRV_IPADDRS=
 724                 HB_OPTIONS=
 725                 [ -z "${TARGET_DEVNAMES[i]}" ] && continue
 726
 727                 # Execute remote command to check whether this target service
 728                 # was made to be high-available
 729                 ! is_ha_target ${host_name} ${TARGET_DEVNAMES[i]} && continue
 730
 731                 # Execute remote command to get Heartbeat channel
 732                 HB_CHANNELS=$(get_cluman_channel ${host_name})
 733                 if [ $? -ne 0 ]; then
 734                         error_output "${HB_CHANNELS}"
 735                 fi
 736
 737                 # Execute remote command to get service IP address
 738                 SRV_IPADDRS=$(get_cluman_srvaddr ${host_name} \
 739                               ${TARGET_SVNAMES[i]})
 740                 if [ $? -ne 0 ]; then
 741                         error_output "${SRV_IPADDRS}"
 742                         return 0
 743                 fi
 744
 745                 HA_CONFIGS[i]=${HB_CHANNELS},${SRV_IPADDRS},${HB_OPTIONS}
 746         done
 747
 748         return 0
 749 }
 750
 751 # get_ha_configs hostname
 752 # Get the HA software configurations from the node @hostname
 753 get_ha_configs() {
 754         local host_name=$1
 755
 756         unset HA_CONFIGS
 757
 758         if [ -z "${HATYPE_OPT}" ]; then
 759                 return 0
 760         fi
 761
 762         verbose_output "Collecting HA software configurations from host $1..."
 763
 764         case "${HATYPE_OPT}" in
 765         "${HBVER_HBV1}" | "${HBVER_HBV2}") # Heartbeat
 766                 if ! get_hb_configs ${host_name}; then
 767                         return 1
 768                 fi
 769                 ;;
 770         "${HATYPE_CLUMGR}") # CluManager
 771                 if ! get_cluman_configs ${host_name}; then
 772                         return 1
 773                 fi
 774                 ;;
 775         esac
 776
 777         return 0
 778 }
 779
 780 #*********************** Lustre targets configurations ***********************#
 781
 782 # is_failover_service target_svname
 783 # Check whether a target service @target_svname is a failover service.
 784 is_failover_service() {
 785         local target_svname=$1
 786         declare -i i
 787
 788         for ((i = 0; i < ${#ALL_TARGET_SVNAMES[@]}; i++)); do
 789                 [ "${target_svname}" = "${ALL_TARGET_SVNAMES[i]}" ] && return 0
 790         done
 791
 792         return 1
 793 }
 794
 795 # get_svnames hostname
 796 # Get the lustre target server obd names from the node @hostname
 797 get_svnames(){
 798         declare -i i
 799         declare -i j
 800         local host_name=$1
 801         local ret_line line
 802
 803         # Initialize the TARGET_SVNAMES array
 804         unset TARGET_SVNAMES
 805         unset FAILOVER_FMTOPTS
 806
 807         # Execute remote command to the node @hostname and figure out what
 808         # lustre services are running.
 809         i=0
 810         j=${#ALL_TARGET_SVNAMES[@]}
 811         while read -r ret_line; do
 812                 if is_pdsh; then
 813                         set -- ${ret_line}
 814                         shift
 815                         line="$*"
 816                 else
 817                         line="${ret_line}"
 818                 fi
 819
 820                 if [ -z "`echo ${line} | grep ${MGS_TYPE}`" ] \
 821                 && [ -z "`echo ${line} | grep ${MDT_TYPE}`" ] \
 822                 && [ -z "`echo ${line} | grep ${OST_TYPE}`" ]; then
 823                         continue
 824                 fi
 825
 826                 # Get target server name
 827                 TARGET_SVNAMES[i]=`echo ${line} | awk '{print $4}'`
 828                 if [ -n "${TARGET_SVNAMES[i]}" ]; then
 829                         if is_failover_service ${TARGET_SVNAMES[i]}; then
 830                                 FAILOVER_FMTOPTS[i]="--noformat"
 831                         fi
 832                         ALL_TARGET_SVNAMES[j]=${TARGET_SVNAMES[i]}
 833                         let "i += 1"
 834                         let "j += 1"
 835                 else
 836                         error_output "get_svnames(): Invalid"\
 837                               "line in ${host_name}'s ${LUSTRE_PROC_DEVICES}"\
 838                               "- \"${line}\"!"
 839                         return 1
 840                 fi
 841         done < <(${REMOTE} ${host_name} "cat ${LUSTRE_PROC_DEVICES}")
 842
 843         if [ $i -eq 0 ]; then
 844                 verbose_output "There are no lustre services running" \
 845                                "on the node ${host_name}!"
 846         fi
 847
 848         return 0
 849 }
 850
 851 # is_loopdev devname
 852 # Check whether a device @devname is a loop device or not
 853 is_loopdev() {
 854         local devname=$1
 855
 856         if [ -z "${devname}" ] || \
 857         [ -z "`echo ${devname}|awk '/\/dev\/loop[[:digit:]]/ {print $0}'`" ]
 858         then
 859                 return 1
 860         fi
 861
 862         return 0
 863 }
 864
 865 # get_devname hostname svname
 866 # Get the device name of lustre target @svname from node @hostname
 867 get_devname() {
 868         local host_name=$1
 869         local target_svname=$2
 870         local target_devname=
 871         local ret_str
 872         local target_type target_obdtype mntdev_file
 873
 874         if [ "${target_svname}" = "${MGS_SVNAME}" ]; then
 875                 # Execute remote command to get the device name of mgs target
 876                 ret_str=`${REMOTE} ${host_name} \
 877                         "PATH=\$PATH:/sbin:/usr/sbin findfs LABEL=${target_svname}" 2>&1`
 878                 if [ $? -ne 0 -a -n "${ret_str}" ]; then
 879                         if [ "${ret_str}" = "${ret_str#*Unable to resolve*}" ]
 880                         then
 881                                 echo "`basename $0`: get_devname() error:" \
 882                                      "remote command error: ${ret_str}"
 883                                 return 1
 884                         fi
 885                 fi
 886
 887                 if [ "${ret_str}" = "${ret_str#*Unable to resolve*}" ]; then
 888                         if is_pdsh; then
 889                                 target_devname=`echo ${ret_str} | awk '{print $2}'`
 890                         else
 891                                 target_devname=`echo ${ret_str} | awk '{print $1}'`
 892                         fi
 893                 fi
 894         else    # Execute remote command to get the device name of mdt/ost target
 895                 target_type=`echo ${target_svname} | cut -d - -f 2`
 896                 target_obdtype=${target_type:0:3}_TYPE
 897
 898                 mntdev_file=${LUSTRE_PROC}/${!target_obdtype}/${target_svname}/mntdev
 899
 900                 ret_str=`${REMOTE} ${host_name} "cat ${mntdev_file}" 2>&1`
 901                 if [ $? -ne 0 -a -n "${ret_str}" ]; then
 902                         echo "`basename $0`: get_devname() error:" \
 903                              "remote command error: ${ret_str}"
 904                         return 1
 905                 fi
 906
 907                 if [ "${ret_str}" != "${ret_str#*No such file*}" ]; then
 908                         echo "`basename $0`: get_devname() error:"\
 909                              "${mntdev_file} does not exist in ${host_name}!"
 910                         return 1
 911                 else
 912                         if is_pdsh; then
 913                                 target_devname=`echo ${ret_str} | awk '{print $2}'`
 914                         else
 915                                 target_devname=`echo ${ret_str} | awk '{print $1}'`
 916                         fi
 917                 fi
 918         fi
 919
 920         echo ${target_devname}
 921         return 0
 922 }
 923
 924 # get_devsize hostname target_devname
 925 # Get the device size (KB) of @target_devname from node @hostname
 926 get_devsize() {
 927         local host_name=$1
 928         local target_devname=$2
 929         local target_devsize=
 930         local ret_str
 931
 932         # Execute remote command to get the device size
 933         ret_str=`${REMOTE} ${host_name} \
 934                 "PATH=\$PATH:/sbin:/usr/sbin blockdev --getsize ${target_devname}" 2>&1`
 935         if [ $? -ne 0 -a -n "${ret_str}" ]; then
 936                 echo "`basename $0`: get_devsize() error:" \
 937                      "remote command error: ${ret_str}"
 938                 return 1
 939         fi
 940
 941         if is_pdsh; then
 942                 target_devsize=`echo ${ret_str} | awk '{print $2}'`
 943         else
 944                 target_devsize=`echo ${ret_str} | awk '{print $1}'`
 945         fi
 946
 947         if [ -z "`echo ${target_devsize}|awk '/^[[:digit:]]/ {print $0}'`" ]
 948         then
 949                 echo "`basename $0`: get_devsize() error: can't" \
 950                 "get device size of ${target_devname} in ${host_name}!"
 951                 return 1
 952         fi
 953
 954         let " target_devsize /= 2"
 955
 956         echo ${target_devsize}
 957         return 0
 958 }
 959
 960 # get_realdevname hostname loop_dev
 961 # Get the real device name of loop device @loop_dev from node @hostname
 962 get_realdevname() {
 963         local host_name=$1
 964         local loop_dev=$2
 965         local target_devname=
 966         local ret_str
 967
 968         # Execute remote command to get the real device name
 969         ret_str=`${REMOTE} ${host_name} \
 970                 "PATH=\$PATH:/sbin:/usr/sbin losetup ${loop_dev}" 2>&1`
 971         if [ $? -ne 0 -a -n "${ret_str}" ]; then
 972                 echo "`basename $0`: get_realdevname() error:" \
 973                      "remote command error: ${ret_str}"
 974                 return 1
 975         fi
 976
 977         if is_pdsh; then
 978                 target_devname=`echo ${ret_str} | awk '{print $4}' \
 979                                 | sed 's/^(//' | sed 's/)$//'`
 980         else
 981                 target_devname=`echo ${ret_str} | awk '{print $3}' \
 982                                 | sed 's/^(//' | sed 's/)$//'`
 983         fi
 984
 985         if [ "${ret_str}" != "${ret_str#*No such*}" ] \
 986         || [ -z "${target_devname}" ]; then
 987                 echo "`basename $0`: get_realdevname() error: can't" \
 988                 "get info on device ${loop_dev} in ${host_name}!"
 989                 return 1
 990         fi
 991
 992         echo ${target_devname}
 993         return 0
 994 }
 995
 996 # get_mntpnt hostname target_devname
 997 # Get the lustre target mount point from the node @hostname
 998 get_mntpnt(){
 999         local host_name=$1
1000         local target_devname=$2
1001         local mnt_point=
1002         local ret_str
1003
1004         # Execute remote command to get the mount point
1005         ret_str=`${REMOTE} ${host_name} \
1006                 "cat /etc/mtab | grep ${target_devname}" 2>&1`
1007         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1008                 echo "`basename $0`: get_mntpnt() error:" \
1009                      "remote command error: ${ret_str}"
1010                 return 1
1011         fi
1012
1013         if is_pdsh; then
1014                 mnt_point=`echo ${ret_str} | awk '{print $3}'`
1015         else
1016                 mnt_point=`echo ${ret_str} | awk '{print $2}'`
1017         fi
1018
1019         if [ -z "${mnt_point}" ]; then
1020                 echo "`basename $0`: get_mntpnt() error: can't" \
1021                 "get the mount point of ${target_devname} in ${host_name}!"
1022                 return 1
1023         fi
1024
1025         echo ${mnt_point}
1026         return 0
1027 }
1028
1029 # get_devnames hostname
1030 # Get the lustre target device names, mount points
1031 # and loop device sizes from the node @hostname
1032 get_devnames(){
1033         declare -i i
1034         local host_name=$1
1035         local ret_line line
1036
1037         # Initialize the arrays
1038         unset TARGET_DEVNAMES
1039         unset TARGET_DEVSIZES
1040         unset TARGET_MNTPNTS
1041
1042         for ((i = 0; i < ${#TARGET_SVNAMES[@]}; i++)); do
1043                 TARGET_DEVNAMES[i]=$(get_devname ${host_name} \
1044                                      ${TARGET_SVNAMES[i]})
1045                 if [ $? -ne 0 ]; then
1046                         error_output "${TARGET_DEVNAMES[i]}"
1047                         return 1
1048                 fi
1049
1050                 if [ -z "${TARGET_DEVNAMES[i]}" ]; then
1051                         if [ "${TARGET_SVNAMES[i]}" = "${MGS_SVNAME}" ]; then
1052                                 verbose_output "There exists combo mgs/mdt"\
1053                                                "target in ${host_name}."
1054                                 continue
1055                         else
1056                                 error_output "get_devname():"\
1057                                          "No device corresponding to target" \
1058                                          "${TARGET_SVNAMES[i]} in ${host_name}!"
1059                                 return 1
1060                         fi
1061                 fi
1062
1063                 # Get the mount point of the target
1064                 TARGET_MNTPNTS[i]=$(get_mntpnt ${host_name} \
1065                                      ${TARGET_DEVNAMES[i]})
1066                 if [ $? -ne 0 ]; then
1067                         error_output "${TARGET_MNTPNTS[i]}"
1068                         return 1
1069                 fi
1070
1071                 # The target device is a loop device?
1072                 if [ -n "${TARGET_DEVNAMES[i]}" ] \
1073                 && is_loopdev ${TARGET_DEVNAMES[i]}; then
1074                         # Get the device size
1075                         TARGET_DEVSIZES[i]=$(get_devsize ${host_name} \
1076                                              ${TARGET_DEVNAMES[i]})
1077                         if [ $? -ne 0 ]; then
1078                                 error_output "${TARGET_DEVSIZES[i]}"
1079                                 return 1
1080                         fi
1081
1082                         # Get the real device name
1083                         TARGET_DEVNAMES[i]=$(get_realdevname ${host_name} \
1084                                              ${TARGET_DEVNAMES[i]})
1085                         if [ $? -ne 0 ]; then
1086                                 error_output "${TARGET_DEVNAMES[i]}"
1087                                 return 1
1088                         fi
1089                 fi
1090         done
1091
1092         return 0
1093 }
1094
1095 # is_target target_svtype ldd_flags
1096 # Check the service type of a lustre target
1097 is_target() {
1098         case "$1" in
1099         "mdt") let "ret = $2 & SVTYPE_MDT";;
1100         "ost") let "ret = $2 & SVTYPE_OST";;
1101         "mgs") let "ret = $2 & SVTYPE_MGS";;
1102         "*")
1103                 error_output "is_target(): Invalid" \
1104                 "target service type - \"$1\"!"
1105                 return 1
1106                 ;;
1107         esac
1108
1109         if [ ${ret} -eq 0 ]; then
1110                 return 1
1111         fi
1112
1113         return 0
1114 }
1115
1116 # get_devtype ldd_flags
1117 # Get the service type of a lustre target from @ldd_flags
1118 get_devtype() {
1119         local target_devtype=
1120
1121         if [ -z "${flags}" ]; then
1122                 echo "`basename $0`: get_devtype() error: Invalid" \
1123                         "ldd_flags - it's value is null!"
1124                 return 1
1125         fi
1126
1127         if is_target "mgs" $1; then
1128                 if is_target "mdt" $1; then
1129                         target_devtype="mgs|mdt"
1130                 else
1131                         target_devtype="mgs"
1132                 fi
1133         elif is_target "mdt" $1; then
1134                 target_devtype="mdt"
1135         elif is_target "ost" $1; then
1136                 target_devtype="ost"
1137         else
1138                 echo "`basename $0`: get_devtype() error: Invalid" \
1139                 "ldd_flags - \"$1\"!"
1140                 return 1
1141         fi
1142
1143         echo ${target_devtype}
1144         return 0
1145 }
1146
1147 # get_mntopts ldd_mount_opts
1148 # Get the user-specified lustre target mount options from @ldd_mount_opts
1149 get_mntopts() {
1150         local mount_opts=
1151         local ldd_mount_opts=$1
1152
1153         mount_opts="${ldd_mount_opts#${ALWAYS_MNTOPTS}}"
1154         mount_opts="${mount_opts#${MDT_MGS_ALWAYS_MNTOPTS}}"
1155         mount_opts="${mount_opts#${OST_ALWAYS_MNTOPTS}}"
1156         mount_opts="${mount_opts#${OST_DEFAULT_MNTOPTS}}"
1157         mount_opts="`echo \"${mount_opts}\" | sed 's/^,//'`"
1158
1159         [ "${mount_opts}" != "${mount_opts#*,*}" ] && echo "\""${mount_opts}"\"" \
1160         || echo ${mount_opts}
1161
1162         return 0
1163 }
1164
1165 # get_mgsnids ldd_params
1166 # Get the mgs nids of lustre target from @ldd_params
1167 get_mgsnids() {
1168         local mgs_nids=         # mgs nids in one mgs node
1169         local all_mgs_nids=     # mgs nids in all mgs failover nodes
1170         local param=
1171         local ldd_params="$*"
1172
1173         for param in ${ldd_params}; do
1174                 if [ -n "`echo ${param}|awk '/mgsnode=/ {print $0}'`" ]; then
1175                         mgs_nids=`echo ${param#${PARAM_MGSNODE}}`
1176
1177                         if [ -n "${all_mgs_nids}" ]; then
1178                                 all_mgs_nids=${all_mgs_nids}:${mgs_nids}
1179                         else
1180                                 all_mgs_nids=${mgs_nids}
1181                         fi
1182                 fi
1183         done
1184
1185         [ "${all_mgs_nids}" != "${all_mgs_nids#*,*}" ] \
1186         && echo "\""${all_mgs_nids}"\"" || echo ${all_mgs_nids}
1187
1188         return 0
1189 }
1190
1191 # get_failnids ldd_params
1192 # Get the failover nids of lustre target from @ldd_params
1193 get_failnids() {
1194         local fail_nids=        # failover nids in one failover node
1195         local all_fail_nids=    # failover nids in all failover nodes
1196                                 # of this target
1197         local param=
1198         local ldd_params="$*"
1199
1200         for param in ${ldd_params}; do
1201                 if [ -n "`echo ${param}|awk '/failover.node=/ {print $0}'`" ]; then
1202                         fail_nids=`echo ${param#${PARAM_FAILNODE}}`
1203
1204                         if [ -n "${all_fail_nids}" ]; then
1205                                 all_fail_nids=${all_fail_nids}:${fail_nids}
1206                         else
1207                                 all_fail_nids=${fail_nids}
1208                         fi
1209                 fi
1210         done
1211
1212         [ "${all_fail_nids}" != "${all_fail_nids#*,*}" ] \
1213         && echo "\""${all_fail_nids}"\"" || echo ${all_fail_nids}
1214
1215         return 0
1216 }
1217
1218 # get_fmtopts target_devname hostname ldd_params
1219 # Get other format options of the lustre target @target_devname from @ldd_params
1220 get_fmtopts() {
1221         local target_devname=$1
1222         local host_name=$2
1223         shift
1224         shift
1225         local ldd_params="$*"
1226         local param=
1227         local fmt_opts=
1228
1229         for param in ${ldd_params}; do
1230                 [ -n "`echo ${param}|awk '/mgsnode=/ {print $0}'`" ] && continue
1231                 [ -n "`echo ${param}|awk '/failover.node=/ {print $0}'`" ] && continue
1232
1233                 if [ -n "${param}" ]; then
1234                         if [ -n "${fmt_opts}" ]; then
1235                                 fmt_opts=${fmt_opts}" --param=\""${param}"\""
1236                         else
1237                                 fmt_opts="--param=\""${param}"\""
1238                         fi
1239                 fi
1240         done
1241
1242         echo ${fmt_opts}
1243         return 0
1244 }
1245
1246 # get_stripecount host_name target_fsname
1247 # Get the stripe count for @target_fsname
1248 get_stripecount() {
1249         local host_name=$1
1250         local target_fsname=$2
1251         local stripe_count=
1252         local stripecount_file
1253         local ret_str
1254
1255         # Get the stripe count
1256         stripecount_file=${LUSTRE_PROC}/lov/${target_fsname}-mdtlov/stripecount
1257         ret_str=`${REMOTE} ${host_name} "cat ${stripecount_file}" 2>&1`
1258         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1259                 echo "`basename $0`: get_stripecount() error:" \
1260                 "remote command to ${host_name} error: ${ret_str}"
1261                 return 1
1262         fi
1263
1264         if is_pdsh; then
1265                 stripe_count=`echo ${ret_str} | awk '{print $2}'`
1266         else
1267                 stripe_count=`echo ${ret_str} | awk '{print $1}'`
1268         fi
1269
1270     if [ "$stripe_count" != "-1" ] && \
1271     [ -z "`echo ${stripe_count}|awk '/^[[:digit:]]/ {print $0}'`" ]; then
1272                 echo "`basename $0`: get_stripecount() error: can't" \
1273                 "get stripe count of ${target_fsname} in ${host_name}!"
1274                 return 1
1275         fi
1276
1277         echo ${stripe_count}
1278         return 0
1279 }
1280
1281 # get_stripecount_opt host_name target_fsname
1282 # Get the stripe count option for lustre mdt target
1283 get_stripecount_opt() {
1284         local host_name=$1
1285         local target_fsname=$2
1286         local stripe_count=
1287         local stripecount_opt=
1288
1289         # Get the stripe count
1290         [ -z "${target_fsname}" ] && target_fsname="lustre"
1291         stripe_count=$(get_stripecount ${host_name} ${target_fsname})
1292         if [ $? -ne 0 ]; then
1293                 echo "${stripe_count}"
1294                 return 1
1295         fi
1296
1297         if [ "${stripe_count}" != "1" ]; then
1298                 stripecount_opt=${OPTSTR_STRIPE_COUNT}${stripe_count}
1299         fi
1300
1301         echo ${stripecount_opt}
1302         return 0
1303 }
1304
1305 # get_ldds hostname
1306 # Get the lustre target disk data from the node @hostname
1307 get_ldds(){
1308         declare -i i
1309         local host_name=$1
1310         local ret_line line
1311         local flags mnt_opts params
1312         local stripecount_opt
1313
1314         # Initialize the arrays
1315         unset TARGET_DEVTYPES TARGET_FSNAMES TARGET_MGSNIDS TARGET_INDEXES
1316         unset TARGET_FMTOPTS  TARGET_MNTOPTS TARGET_FAILNIDS
1317
1318         # Get lustre target device type, fsname, index, etc.
1319         # from MOUNT_DATA_FILE. Using tunefs.lustre to read it.
1320         for ((i = 0; i < ${#TARGET_DEVNAMES[@]}; i++)); do
1321                 flags=
1322                 mnt_opts=
1323                 params=
1324                 stripecount_opt=
1325                 [ -z "${TARGET_DEVNAMES[i]}" ] && continue
1326
1327                 # Execute remote command to read MOUNT_DATA_FILE
1328                 while read -r ret_line; do
1329                         if is_pdsh; then
1330                                 set -- ${ret_line}
1331                                 shift
1332                                 line="$*"
1333                         else
1334                                 line="${ret_line}"
1335                         fi
1336
1337                         if [ -n "`echo ${line}|awk '/Index:/ {print $0}'`" ]; then
1338                                 TARGET_INDEXES[i]=`echo ${line}|awk '{print $2}'`
1339                                 continue
1340                         fi
1341
1342                         if [ -n "`echo ${line}|awk '/Lustre FS:/ {print $0}'`" ]; then
1343                                 TARGET_FSNAMES[i]=`echo ${line}|awk '{print $3}'`
1344                                 continue
1345                         fi
1346
1347                         if [ -n "`echo ${line}|awk '/Flags:/ {print $0}'`" ]; then
1348                                 flags=`echo ${line}|awk '{print $2}'`
1349                                 continue
1350                         fi
1351
1352                         if [ -n "`echo ${line}|awk '/Persistent mount opts:/ {print $0}'`" ]; then
1353                                 mnt_opts=`echo ${line}|awk '{print $0}'`
1354                                 mnt_opts=`echo ${mnt_opts#Persistent mount opts: }`
1355                                 continue
1356                         fi
1357
1358                         if [ -n "`echo ${line}|awk '/Parameters:/ {print $0}'`" ]; then
1359                                 params=`echo ${line}|awk '{print $0}'`
1360                                 params=`echo ${params#Parameters:}`
1361                                 break
1362                         fi
1363                 done < <(${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin
1364                 ${TUNEFS} --print --verbose ${TARGET_DEVNAMES[i]} 2>/dev/null")
1365
1366                 if [ -z "${flags}" ]; then
1367                         error_output "get_ldds(): Invalid" \
1368                                  "ldd_flags of target ${TARGET_DEVNAMES[i]}" \
1369                                  "in host ${host_name} - it's value is null!"\
1370                                  "Check ${TUNEFS} command!"
1371                         return 1
1372                 fi
1373
1374                 if [ "${TARGET_INDEXES[i]}" = "unassigned" ] \
1375                 || is_target "mgs" ${flags}; then
1376                         TARGET_INDEXES[i]=
1377                 fi
1378
1379                 [ "${TARGET_FSNAMES[i]}" = "lustre" ] && TARGET_FSNAMES[i]=
1380
1381                 # Get the lustre target service type
1382                 TARGET_DEVTYPES[i]=$(get_devtype ${flags})
1383                 if [ $? -ne 0 ]; then
1384                         error_output "${TARGET_DEVTYPES[i]} From device" \
1385                         "${TARGET_DEVNAMES[i]} in host ${host_name}!"
1386                         return 1
1387                 fi
1388
1389                 # Get the lustre target mount options
1390                 TARGET_MNTOPTS[i]=$(get_mntopts "${mnt_opts}")
1391
1392                 # Get mgs nids of the lustre target
1393                 TARGET_MGSNIDS[i]=$(get_mgsnids "${params}")
1394
1395                 # Get failover nids of the lustre target
1396                 TARGET_FAILNIDS[i]=$(get_failnids "${params}")
1397                 if [ $? -ne 0 ]; then
1398                         error_output "${TARGET_FAILNIDS[i]} From device" \
1399                         "${TARGET_DEVNAMES[i]} in host ${host_name}!"
1400                         return 1
1401                 fi
1402
1403                 # Get other format options of the lustre target
1404                 TARGET_FMTOPTS[i]=$(get_fmtopts ${TARGET_DEVNAMES[i]} ${host_name} "${params}")
1405                 if [ $? -ne 0 ]; then
1406                         error_output "${TARGET_FMTOPTS[i]}"
1407                         return 1
1408                 fi
1409
1410                 if [ -n "${TARGET_DEVSIZES[i]}" ]; then
1411                         if [ -n "${TARGET_FMTOPTS[i]}" ]; then
1412                                 TARGET_FMTOPTS[i]="--device-size=${TARGET_DEVSIZES[i]} ""${TARGET_FMTOPTS[i]}"
1413                         else
1414                                 TARGET_FMTOPTS[i]="--device-size=${TARGET_DEVSIZES[i]}"
1415                         fi
1416                 fi
1417
1418                 if [ -n "${FAILOVER_FMTOPTS[i]}" ]; then
1419                         if [ -n "${TARGET_FMTOPTS[i]}" ]; then
1420                                 TARGET_FMTOPTS[i]=${TARGET_FMTOPTS[i]}" "${FAILOVER_FMTOPTS[i]}
1421                         else
1422                                 TARGET_FMTOPTS[i]=${FAILOVER_FMTOPTS[i]}
1423                         fi
1424                 fi
1425
1426                 if is_target "mdt" ${flags}; then
1427                         # Get the stripe count option
1428                         stripecount_opt=$(get_stripecount_opt ${host_name} ${TARGET_FSNAMES[i]})
1429                         if [ $? -ne 0 ]; then
1430                                 error_output "${stripecount_opt}"
1431                                 return 1
1432                         fi
1433
1434                         if [ -n "${stripecount_opt}" ]; then
1435                                 if [ -n "${TARGET_FMTOPTS[i]}" ]; then
1436                                         TARGET_FMTOPTS[i]=${TARGET_FMTOPTS[i]}" "${stripecount_opt}
1437                                 else
1438                                         TARGET_FMTOPTS[i]=${stripecount_opt}
1439                                 fi
1440                         fi
1441                 fi
1442
1443                 if [ "${TARGET_FMTOPTS[i]}" != "${TARGET_FMTOPTS[i]#*,*}" ]; then
1444                         TARGET_FMTOPTS[i]="\""${TARGET_FMTOPTS[i]}"\""
1445                 fi
1446         done
1447
1448         return 0
1449 }
1450
1451 # get_journalsize target_devname hostname
1452 # Get the journal size of lustre target @target_devname from @hostname
1453 get_journalsize() {
1454         local target_devname=$1
1455         local host_name=$2
1456         local journal_inode=
1457         local journal_size=
1458         local ret_str
1459
1460         # Execute remote command to get the journal inode number
1461         ret_str=`${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin \
1462 debugfs -R 'stats -h' ${target_devname} | grep 'Journal inode:'" 2>&1`
1463         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1464                 echo "`basename $0`: get_journalsize() error:" \
1465                      "remote command error: ${ret_str}"
1466                 return 1
1467         fi
1468
1469         ret_str=${ret_str#${ret_str%Journal inode:*}}
1470         journal_inode=`echo ${ret_str} | awk '{print $3}'`
1471         if [ -z "`echo ${journal_inode}|awk '/^[[:digit:]]/ {print $0}'`" ]
1472         then
1473                 echo "`basename $0`: get_journalsize() error: can't" \
1474                 "get journal inode of ${target_devname} in ${host_name}!"
1475                 return 1
1476         fi
1477
1478         # Execute remote command to get the journal size
1479         ret_str=`${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin \
1480 debugfs -R 'stat <${journal_inode}>' ${target_devname}|grep '^User:'" 2>&1`
1481         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1482                 echo "`basename $0`: get_journalsize() error:" \
1483                      "remote command error: ${ret_str}"
1484                 return 1
1485         fi
1486
1487         ret_str=${ret_str#${ret_str%User:*}}
1488         journal_size=`echo ${ret_str} | awk '{print $6}'`
1489         if [ -z "`echo ${journal_size}|awk '/^[[:digit:]]/ {print $0}'`" ]
1490         then
1491                 echo "`basename $0`: get_journalsize() error: can't" \
1492                 "get journal size of ${target_devname} in ${host_name}!"
1493                 return 1
1494         fi
1495
1496         let "journal_size /= 1024*1024" # MB
1497
1498         echo ${journal_size}
1499         return 0
1500 }
1501
1502 # get_defaultjournalsize target_devsize
1503 # Calculate the default journal size from target device size @target_devsize
1504 get_defaultjournalsize() {
1505         declare -i target_devsize=$1
1506         declare -i journal_size=0
1507         declare -i max_size base_size
1508
1509         let "base_size = 1024*1024"
1510         if [ ${target_devsize} -gt ${base_size} ]; then  # 1GB
1511                 let "journal_size = target_devsize / 102400"
1512                 let "journal_size *= 4"
1513         fi
1514
1515         let "max_size = 102400 * L_BLOCK_SIZE"
1516         let "max_size >>= 20" # 400MB
1517
1518         if [ ${journal_size} -gt ${max_size} ]; then
1519                 let "journal_size = max_size"
1520         fi
1521
1522         echo ${journal_size}
1523         return 0
1524 }
1525
1526 # figure_journal_size target_devname hostname
1527 # Find a reasonable journal file size given the number of blocks
1528 # in the filesystem. This algorithm is derived from figure_journal_size()
1529 # function in util.c of e2fsprogs-1.38.cfs2-1.src.rpm.
1530 figure_journal_size() {
1531         local target_devname=$1
1532         local host_name=$2
1533         local ret_str
1534         declare -i block_count
1535         declare -i journal_blocks
1536         declare -i journal_size
1537
1538         # Execute remote command to get the block count
1539         ret_str=`${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin \
1540 debugfs -R 'stats -h' ${target_devname} | grep 'Block count:'" 2>&1`
1541         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1542                 echo "`basename $0`: figure_journal_size() error:" \
1543                      "remote command error: ${ret_str}"
1544                 return 1
1545         fi
1546
1547         ret_str=${ret_str#${ret_str%Block count:*}}
1548         block_count=`echo ${ret_str} | awk '{print $3}'`
1549         if [ -z "`echo ${block_count}|awk '/^[[:digit:]]/ {print $0}'`" ]
1550         then
1551                 echo "`basename $0`: figure_journal_size() error: can't" \
1552                 "get block count of ${target_devname} in ${host_name}!"
1553                 return 1
1554         fi
1555
1556         if ((block_count < 32768)); then
1557                 let "journal_blocks = 1024"
1558         elif ((block_count < 256*1024)); then
1559                 let "journal_blocks = 4096"
1560         elif ((block_count < 512*1024)); then
1561                 let "journal_blocks = 8192"
1562         elif ((block_count < 1024*1024)); then
1563                 let "journal_blocks = 16384"
1564         else
1565                 let "journal_blocks = 32768"
1566         fi
1567
1568         let "journal_size = journal_blocks * L_BLOCK_SIZE / 1048576"
1569
1570         echo ${journal_size}
1571         return 0
1572 }
1573
1574 # get_J_opt hostname target_devname target_devsize
1575 # Get the mkfs -J option of lustre target @target_devname
1576 # from the node @hostname
1577 get_J_opt() {
1578         local host_name=$1
1579         local target_devname=$2
1580         local target_devsize=$3
1581         local journal_size=
1582         local default_journal_size=
1583         local journal_opt=
1584
1585         # Get the real journal size of lustre target
1586         journal_size=$(get_journalsize ${target_devname} ${host_name})
1587         if [ $? -ne 0 ]; then
1588                 echo "${journal_size}"
1589                 return 1
1590         fi
1591
1592         # Get the default journal size of lustre target
1593         default_journal_size=$(get_defaultjournalsize ${target_devsize})
1594         if [ "${default_journal_size}" = "0" ]; then
1595                 default_journal_size=$(figure_journal_size ${target_devname} \
1596                                        ${host_name})
1597                 if [ $? -ne 0 ]; then
1598                         echo "${default_journal_size}"
1599                         return 1
1600                 fi
1601         fi
1602
1603         if [ "${journal_size}" != "${default_journal_size}" ]; then
1604                 journal_opt="-J size=${journal_size}"
1605         fi
1606
1607         echo ${journal_opt}
1608         return 0
1609 }
1610
1611 # get_ratio target_devname hostname
1612 # Get the bytes/inode ratio of lustre target @target_devname from @hostname
1613 get_ratio() {
1614         local target_devname=$1
1615         local host_name=$2
1616         local inode_count=
1617         local block_count=
1618         local ratio=
1619         local ret_str
1620
1621         # Execute remote command to get the inode count
1622         ret_str=`${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin \
1623 debugfs -R 'stats -h' ${target_devname} | grep 'Inode count:'" 2>&1`
1624         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1625                 echo "`basename $0`: get_ratio() error:" \
1626                      "remote command error: ${ret_str}"
1627                 return 1
1628         fi
1629
1630         ret_str=${ret_str#${ret_str%Inode count:*}}
1631         inode_count=`echo ${ret_str} | awk '{print $3}'`
1632         if [ -z "`echo ${inode_count}|awk '/^[[:digit:]]/ {print $0}'`" ]
1633         then
1634                 echo "`basename $0`: get_ratio() error: can't" \
1635                 "get inode count of ${target_devname} in ${host_name}!"
1636                 return 1
1637         fi
1638
1639         # Execute remote command to get the block count
1640         ret_str=`${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin \
1641 debugfs -R 'stats -h' ${target_devname} | grep 'Block count:'" 2>&1`
1642         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1643                 echo "`basename $0`: get_ratio() error:" \
1644                      "remote command error: ${ret_str}"
1645                 return 1
1646         fi
1647
1648         ret_str=${ret_str#${ret_str%Block count:*}}
1649         block_count=`echo ${ret_str} | awk '{print $3}'`
1650         if [ -z "`echo ${block_count}|awk '/^[[:digit:]]/ {print $0}'`" ]
1651         then
1652                 echo "`basename $0`: get_ratio() error: can't" \
1653                 "get block count of ${target_devname} in ${host_name}!"
1654                 return 1
1655         fi
1656
1657         let "ratio = block_count*L_BLOCK_SIZE/inode_count"
1658
1659         echo ${ratio}
1660         return 0
1661 }
1662
1663 # get_default_ratio target_devtype target_devsize
1664 # Calculate the default bytes/inode ratio from target type @target_devtype
1665 get_default_ratio() {
1666         local target_devtype=$1
1667         declare -i target_devsize=$2
1668         local ratio=
1669
1670         case "${target_devtype}" in
1671         "mdt" | "mgs|mdt" | "mdt|mgs")
1672                 ratio=4096;;
1673         "ost")
1674                 [ ${target_devsize} -gt 1000000 ] && ratio=16384;;
1675         esac
1676
1677         [ -z "${ratio}" ] && ratio=${L_BLOCK_SIZE}
1678
1679         echo ${ratio}
1680         return 0
1681 }
1682
1683 # get_i_opt hostname target_devname target_devtype target_devsize
1684 # Get the mkfs -i option of lustre target @target_devname
1685 # from the node @hostname
1686 get_i_opt() {
1687         local host_name=$1
1688         local target_devname=$2
1689         local target_devtype=$3
1690         local target_devsize=$4
1691         local ratio=
1692         local default_ratio=
1693         local ratio_opt=
1694
1695         # Get the real bytes/inode ratio of lustre target
1696         ratio=$(get_ratio ${target_devname} ${host_name})
1697         if [ $? -ne 0 ]; then
1698                 echo "${ratio}"
1699                 return 1
1700         fi
1701
1702         # Get the default bytes/inode ratio of lustre target
1703         default_ratio=$(get_default_ratio ${target_devtype} ${target_devsize})
1704
1705         if [ "${ratio}" != "${default_ratio}" ]; then
1706                 ratio_opt="-i ${ratio}"
1707         fi
1708
1709         echo ${ratio_opt}
1710         return 0
1711 }
1712
1713 # get_isize target_devname hostname
1714 # Get the inode size of lustre target @target_devname from @hostname
1715 get_isize() {
1716         local target_devname=$1
1717         local host_name=$2
1718         local inode_size=
1719         local ret_str
1720
1721         # Execute remote command to get the inode size
1722         ret_str=`${REMOTE} ${host_name} "PATH=\$PATH:/sbin:/usr/sbin \
1723 debugfs -R 'stats -h' ${target_devname} | grep 'Inode size:'" 2>&1`
1724         if [ $? -ne 0 -a -n "${ret_str}" ]; then
1725                 echo "`basename $0`: get_isize() error:" \
1726                      "remote command error: ${ret_str}"
1727                 return 1
1728         fi
1729
1730         ret_str=${ret_str#${ret_str%Inode size:*}}
1731         inode_size=`echo ${ret_str} | awk '{print $3}'`
1732         if [ -z "`echo ${inode_size}|awk '/^[[:digit:]]/ {print $0}'`" ]
1733         then
1734                 echo "`basename $0`: get_isize() error: can't" \
1735                 "get inode size of ${target_devname} in ${host_name}!"
1736                 return 1
1737         fi
1738
1739         echo ${inode_size}
1740         return 0
1741 }
1742
1743 # get_mdt_default_isize host_name target_fsname
1744 # Calculate the default inode size of lustre mdt target
1745 get_mdt_default_isize() {
1746         local host_name=$1
1747         local target_fsname=$2
1748         declare -i stripe_count
1749         local inode_size=
1750
1751         # Get the stripe count
1752         stripe_count=$(get_stripecount ${host_name} ${target_fsname})
1753         if [ $? -ne 0 ]; then
1754                 echo "${stripe_count}"
1755                 return 1
1756         fi
1757
1758         if ((stripe_count > 77)); then
1759                 inode_size=512
1760         elif ((stripe_count > 34)); then
1761                 inode_size=2048
1762         elif ((stripe_count > 13)); then
1763                 inode_size=1024
1764         else
1765                 inode_size=512
1766         fi
1767
1768         echo ${inode_size}
1769         return 0
1770 }
1771
1772 # get_default_isize host_name target_devtype target_fsname
1773 # Calculate the default inode size of lustre target type @target_devtype
1774 get_default_isize() {
1775         local host_name=$1
1776         local target_devtype=$2
1777         local target_fsname=$3
1778         local inode_size=
1779
1780         case "${target_devtype}" in
1781         "mdt" | "mgs|mdt" | "mdt|mgs")
1782                 inode_size=$(get_mdt_default_isize ${host_name} ${target_fsname})
1783                 if [ $? -ne 0 ]; then
1784                         echo "${inode_size}"
1785                         return 1
1786                 fi
1787                 ;;
1788         "ost")
1789                 inode_size=256;;
1790         esac
1791
1792         [ -z "${inode_size}" ] && inode_size=128
1793
1794         echo ${inode_size}
1795         return 0
1796 }
1797
1798 # get_I_opt hostname target_devname target_devtype target_fsname
1799 # Get the mkfs -I option of lustre target @target_devname
1800 # from the node @hostname
1801 get_I_opt() {
1802         local host_name=$1
1803         local target_devname=$2
1804         local target_devtype=$3
1805         local target_fsname=$4
1806         local isize=
1807         local default_isize=
1808         local isize_opt=
1809
1810         # Get the real inode size of lustre target
1811         isize=$(get_isize ${target_devname} ${host_name})
1812         if [ $? -ne 0 ]; then
1813                 echo "${isize}"
1814                 return 1
1815         fi
1816
1817         # Get the default inode size of lustre target
1818         [ -z "${target_fsname}" ] && target_fsname="lustre"
1819         default_isize=$(get_default_isize ${host_name} ${target_devtype} \
1820                         ${target_fsname})
1821         if [ $? -ne 0 ]; then
1822                 echo "${default_isize}"
1823                 return 1
1824         fi
1825
1826         if [ "${isize}" != "${default_isize}" ]; then
1827                 isize_opt="-I ${isize}"
1828         fi
1829
1830         echo ${isize_opt}
1831         return 0
1832 }
1833
1834 # get_mkfsopts hostname
1835 # Get the mkfs options of lustre targets from the node @hostname
1836 get_mkfsopts(){
1837         declare -i i
1838         local host_name=$1
1839         local journal_opt
1840         local ratio_opt
1841         local inode_size_opt
1842
1843         # Initialize the arrays
1844         unset TARGET_MKFSOPTS
1845
1846         # FIXME: Get other mkfs options of ext3/ldiskfs besides -J, -i and -I
1847         for ((i = 0; i < ${#TARGET_DEVNAMES[@]}; i++)); do
1848                 journal_opt=
1849                 ratio_opt=
1850                 inode_size_opt=
1851
1852                 [ -z "${TARGET_DEVNAMES[i]}" ] && continue
1853
1854                 if [ -z "${TARGET_DEVSIZES[i]}" ]; then
1855                         # Get the device size
1856                         TARGET_DEVSIZES[i]=$(get_devsize ${host_name} \
1857                                          ${TARGET_DEVNAMES[i]})
1858                         if [ $? -ne 0 ]; then
1859                                 error_output "${TARGET_DEVSIZES[i]}"
1860                                 return 1
1861                         fi
1862                 fi
1863
1864                 # Get the journal option
1865                 journal_opt=$(get_J_opt ${host_name} ${TARGET_DEVNAMES[i]} \
1866                               ${TARGET_DEVSIZES[i]})
1867                 if [ $? -ne 0 ]; then
1868                         error_output "${journal_opt}"
1869                         return 1
1870                 fi
1871
1872                 if [ -n "${journal_opt}" ]; then
1873                         if [ -z "${TARGET_MKFSOPTS[i]}" ]; then
1874                                 TARGET_MKFSOPTS[i]="${journal_opt}"
1875                         else
1876                                 TARGET_MKFSOPTS[i]=${TARGET_MKFSOPTS[i]}" ${journal_opt}"
1877                         fi
1878                 fi
1879
1880                 # Get the bytes-per-inode ratio option
1881                 ratio_opt=$(get_i_opt ${host_name} ${TARGET_DEVNAMES[i]} \
1882                             ${TARGET_DEVTYPES[i]} ${TARGET_DEVSIZES[i]})
1883                 if [ $? -ne 0 ]; then
1884                         error_output "${ratio_opt}"
1885                         return 1
1886                 fi
1887
1888                 if [ -n "${ratio_opt}" ]; then
1889                         if [ -z "${TARGET_MKFSOPTS[i]}" ]; then
1890                                 TARGET_MKFSOPTS[i]="${ratio_opt}"
1891                         else
1892                                 TARGET_MKFSOPTS[i]=${TARGET_MKFSOPTS[i]}" ${ratio_opt}"
1893                         fi
1894                 fi
1895
1896                 # Get the inode size option
1897                 inode_size_opt=$(get_I_opt ${host_name} ${TARGET_DEVNAMES[i]} \
1898                                  ${TARGET_DEVTYPES[i]} ${TARGET_FSNAMES[i]})
1899                 if [ $? -ne 0 ]; then
1900                         error_output "${inode_size_opt}"
1901                         return 1
1902                 fi
1903
1904                 if [ -n "${inode_size_opt}" ]; then
1905                         if [ -z "${TARGET_MKFSOPTS[i]}" ]; then
1906                                 TARGET_MKFSOPTS[i]="${inode_size_opt}"
1907                         else
1908                                 TARGET_MKFSOPTS[i]=${TARGET_MKFSOPTS[i]}" ${inode_size_opt}"
1909                         fi
1910                 fi
1911
1912                 if [ "${TARGET_MKFSOPTS[i]}" != "${TARGET_MKFSOPTS[i]#*,*}" ]; then
1913                         TARGET_MKFSOPTS[i]="\""${TARGET_MKFSOPTS[i]}"\""
1914                 fi
1915         done
1916         return 0
1917 }
1918
1919 # get_target_configs hostname
1920 # Get the lustre target informations from the node @hostname
1921 get_target_configs() {
1922         declare -i i
1923         local host_name=$1
1924         local ret_line line
1925
1926         # Initialize the arrays
1927         unset TARGET_CONFIGS
1928
1929         # Get lustre target server names
1930         if ! get_svnames ${host_name}; then
1931                 return 1
1932         fi
1933
1934         # Get lustre target device names, mount points and loop device sizes
1935         if ! get_devnames ${host_name}; then
1936                 return 1
1937         fi
1938
1939         # Get lustre target device type, fsname, index, etc.
1940         if ! get_ldds ${host_name}; then
1941                 return 1
1942         fi
1943
1944         # Get mkfs options of lustre targets
1945         if ! get_mkfsopts ${host_name}; then
1946                 return 1
1947         fi
1948
1949         # Construct lustre target configs
1950         for ((i = 0; i < ${#TARGET_DEVNAMES[@]}; i++)); do
1951                 [ -z "${TARGET_DEVNAMES[i]}" ] && continue
1952                 TARGET_CONFIGS[i]=${TARGET_DEVNAMES[i]},${TARGET_MNTPNTS[i]},${TARGET_DEVTYPES[i]},${TARGET_FSNAMES[i]},${TARGET_MGSNIDS[i]},${TARGET_INDEXES[i]},${TARGET_FMTOPTS[i]},${TARGET_MKFSOPTS[i]},${TARGET_MNTOPTS[i]},${TARGET_FAILNIDS[i]}
1953         done
1954
1955         return 0
1956 }
1957
1958 # get_configs hostname
1959 # Get all the informations needed to generate a csv file from
1960 # the node @hostname
1961 get_configs() {
1962         # Check the hostname
1963         if [ -z "$1" ]; then
1964                 error_output "get_configs():" \
1965                          "Missing hostname!"
1966                 return 1
1967         fi
1968
1969         # Get network module options
1970         verbose_output ""
1971         verbose_output "Collecting network module options from host $1..."
1972         if ! get_module_opts $1; then
1973                 return 1
1974         fi
1975         verbose_output "OK"
1976
1977         # Get lustre target informations
1978         verbose_output "Collecting Lustre targets informations from host $1..."
1979         if ! get_target_configs $1; then
1980                 return 1
1981         fi
1982         verbose_output "OK"
1983
1984         # Get HA software configurations
1985         if ! get_ha_configs $1; then
1986                 return 1
1987         fi
1988
1989         return 0
1990 }
1991
1992 # Collect linux MD/LVM device informations from the lustre cluster and
1993 # append them to the csv file
1994 get_mdlvm_info() {
1995         declare -i idx
1996         declare -i i
1997         local line
1998
1999         # Collect and append linux MD/LVM informations to the csv file
2000         for ((idx = 0; idx < ${#HOST_NAMES[@]}; idx++)); do
2001                 [ -z "${HOST_NAMES[idx]}" ] && continue
2002
2003                 # Collect MD device informations
2004                 ! get_md_configs ${HOST_NAMES[idx]} && return 1
2005
2006                 # Append MD device informations to the csv file
2007                 for ((i = 0; i < ${#MD_NAME[@]}; i++)); do
2008                         line=${HOST_NAMES[idx]},${MD_MARKER},${MD_NAME[i]},,,${MD_LEVEL[i]},${MD_DEVS[i]}
2009                         verbose_output "Informations of MD device ${MD_NAME[i]}" \
2010                                        "in host ${HOST_NAMES[idx]} are as follows:"
2011                         verbose_output "${line}"
2012                         echo "${line}" >> ${LUSTRE_CSV_FILE}
2013                 done
2014
2015                 # Collect PV informations
2016                 ! get_pv_configs ${HOST_NAMES[idx]} && return 1
2017
2018                 # Append PV informations to the csv file
2019                 if [ -n "${PV_NAMES}" ]; then
2020                         line=${HOST_NAMES[idx]},${PV_MARKER},${PV_NAMES}
2021                         verbose_output "Informations of PVs" \
2022                                        "in host ${HOST_NAMES[idx]} are as follows:"
2023                         verbose_output "${line}"
2024                         echo "${line}" >> ${LUSTRE_CSV_FILE}
2025                 fi
2026
2027                 # Collect VG informations
2028                 ! get_vg_configs ${HOST_NAMES[idx]} && return 1
2029
2030                 # Append VG informations to the csv file
2031                 for ((i = 0; i < ${#VG_NAME[@]}; i++)); do
2032                         line=${HOST_NAMES[idx]},${VG_MARKER},${VG_NAME[i]},,,${VG_PVNAMES[i]}
2033                         verbose_output "Informations of VG ${VG_NAME[i]}" \
2034                                        "in host ${HOST_NAMES[idx]} are as follows:"
2035                         verbose_output "${line}"
2036                         echo "${line}" >> ${LUSTRE_CSV_FILE}
2037                 done
2038
2039                 # Collect LV informations
2040                 ! get_lv_configs ${HOST_NAMES[idx]} && return 1
2041
2042                 # Append LV informations to the csv file
2043                 for ((i = 0; i < ${#LV_NAME[@]}; i++)); do
2044                         line=${HOST_NAMES[idx]},${LV_MARKER},${LV_NAME[i]},,,${LV_SIZE[i]},${LV_VGNAME[i]}
2045                         verbose_output "Informations of LV /dev/${LV_VGNAME[i]}/${LV_NAME[i]}"\
2046                                        "in host ${HOST_NAMES[idx]} are as follows:"
2047                         verbose_output "${line}"
2048                         echo "${line}" >> ${LUSTRE_CSV_FILE}
2049                 done
2050         done
2051         return 0
2052 }
2053
2054 # Generate the csv file from the lustre cluster
2055 gen_csvfile() {
2056         declare -i idx
2057         declare -i i
2058         local line
2059
2060         # Get lustre cluster node names
2061         verbose_output "Collecting Lustre cluster node names..."
2062         if ! get_hostnames; then
2063                 return 1
2064         fi
2065         verbose_output "OK"
2066
2067         : > ${LUSTRE_CSV_FILE}
2068
2069         ${GET_MDLVM_INFO} && get_mdlvm_info
2070
2071         # Collect and append lustre target informations to the csv file
2072         for ((idx = 0; idx < ${#HOST_NAMES[@]}; idx++)); do
2073                 # Collect informations
2074                 if ! get_configs ${HOST_NAMES[idx]}; then
2075                         rm -f ${LUSTRE_CSV_FILE}
2076                         return 1
2077                 fi
2078
2079                 # Append informations to the csv file
2080                 for ((i = 0; i < ${#TARGET_DEVNAMES[@]}; i++)); do
2081                         [ -z "${TARGET_DEVNAMES[i]}" ] && continue
2082
2083                         if [ -z "${HA_CONFIGS[i]}" ]; then
2084                                 line=${HOST_NAMES[idx]},${MODULE_OPTS},${TARGET_CONFIGS[i]}
2085                         else
2086                                 line=${HOST_NAMES[idx]},${MODULE_OPTS},${TARGET_CONFIGS[i]},${HA_CONFIGS[i]}
2087                         fi
2088                         verbose_output "Informations of target ${TARGET_DEVNAMES[i]}" \
2089                                        "in host ${HOST_NAMES[idx]} are as follows:"
2090                         verbose_output "${line}"
2091                         echo "" >> ${LUSTRE_CSV_FILE}
2092                         echo "${line}" >> ${LUSTRE_CSV_FILE}
2093                 done
2094         done
2095
2096         return 0
2097 }
2098
2099 # Main flow
2100 echo "`basename $0`: ******** Generate csv file -- ${LUSTRE_CSV_FILE} START ********"
2101 if ! gen_csvfile; then
2102         exit 1
2103 fi
2104 echo "`basename $0`: ******** Generate csv file -- ${LUSTRE_CSV_FILE} OK **********"
2105
2106 exit 0