lustre/scripts/system-profile.sh

   1 #!/bin/sh
   2
   3 # System Profiling Script
   4
   5 TESTS="oprofile iostat vmstat proc_dump"
   6
   7 # common parameters
   8 export OUTPUTDIR=/home/op
   9 export KERNELDIR=/usr/src/linux
  10 export LUSTREDIR=/usr/src/lustre
  11 export PORTALSDIR=/usr/src/portals
  12 NAL=socknal
  13
  14 # Params for OPROFILE
  15 #CTR0_EVENT=CPU_CLK_UNHALTED
  16 CTR0_COUNT=10000
  17
  18 # for intel Petium 4 onwards... Also requires Unit Mask
  19 CTR0_EVENT=GLOBAL_POWER_EVENTS
  20 CTR0_UNIT_MASK=0x01
  21
  22 # Params for VMSTAT
  23 VM_SAMPLING=1
  24
  25 # Params for IOSTAT
  26 IO_SAMPLING=1
  27
  28 # Params for PROC DUMP
  29 export PROC_SAMPLING=2
  30
  31 oprofile_start() {
  32         echo "$(date +%T): oprofile started...." >>  $OUTPUTDIR/$HOSTNAME/summary
  33         op_start --ctr0-event=$CTR0_EVENT --ctr0-count=$CTR0_COUNT --ctr0-unit-mask=$CTR0_UNIT_MASK --vmlinux=${KERNELDIR}/vmlinux
  34         mkdir -p ${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/{obdclass,obdecho,osc,ptlrpc,extN,obdfilter,ost,mdc,mds4mds,mds4mds_extN,llite,portals,$NAL}
  35         mkdir -p ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling
  36 }
  37
  38 iostat_start() {
  39         echo "$(date +%T): iostat started...." >>  $OUTPUTDIR/$HOSTNAME/summary
  40         mkdir ${OUTPUTDIR}/${HOSTNAME}/iostat
  41         iostat $IO_SAMPLING > $OUTPUTDIR/$HOSTNAME/iostat/iostat.op &
  42         PID=$!
  43         echo $PID > $OUTPUTDIR/$HOSTNAME/tmp/iostat.pid
  44 }
  45
  46 vmstat_start() {
  47         echo $OUTPUTDIR;
  48         echo "$(date +%T): vmstat started...." >>  $OUTPUTDIR/$HOSTNAME/summary
  49         mkdir ${OUTPUTDIR}/${HOSTNAME}/vmstat
  50         vmstat $VM_SAMPLING > $OUTPUTDIR/$HOSTNAME/vmstat/vmstat.op &
  51         PID=$!
  52         echo $PID > $OUTPUTDIR/$HOSTNAME/tmp/vmstat.pid
  53 }
  54
  55 proc_dump_start() {
  56         echo "$(date +%T): proc dump started...." >> $OUTPUTDIR/$HOSTNAME/summary
  57         sh -c '
  58                 CTRFILE=/$OUTPUTDIR/$HOSTNAME/tmp/running.$$.pid;
  59                 echo $$ > ${CTRFILE};
  60                 while [ -f $CTRFILE ]; do
  61                         cat /proc/meminfo >> $OUTPUTDIR/$HOSTNAME/meminfo;
  62                         cat /proc/interrupts >> $OUTPUTDIR/$HOSTNAME/interrupts;
  63                         cat /proc/net/dev >> $OUTPUTDIR/$HOSTNAME/net-dev;
  64                         sleep ${PROC_SAMPLING};
  65                 done;
  66                 ' &
  67         CTRFILE=/tmp/running.$!.pid
  68         echo "proc_dump_ctrl $CTRFILE" >> /tmp/prof-ctrl
  69 }
  70
  71 oprofile_stop() {
  72         op_dump
  73         op_stop
  74
  75         echo "$(date +%T): oprofile stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
  76
  77         for i in obdclass obdecho osc ptlrpc extN obdfilter mds ost mdc llite
  78         do
  79                 oprofpp -l ${LUSTREDIR}/${i}/${i}.o >  ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/${i}.prof 2>/dev/null
  80         done
  81
  82         oprofpp -l ${LUSTREDIR}/mds/mds_extN.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/mds_extN.prof 2>/dev/null
  83         oprofpp -l ${PORTALSDIR}/linux/oslib/portals.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/portals.prof 2>/dev/null
  84         oprofpp -l ${PORTALSDIR}/linux/${NAL}/k${NAL}.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/k${NAL}.prof 2>/dev/null
  85
  86         for i in obdclass obdecho osc ptlrpc extN obdfilter ost mdc llite
  87         do
  88                 op_to_source --source-dir=${LUSTREDIR}/${i}/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/${i}/ ${LUSTREDIR}/${i}/${i}.o 2>/dev/null
  89         done
  90
  91         op_to_source --source-dir=${LUSTREDIR}/mds/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/mds4mds/ ${LUSTREDIR}/mds/mds.o 2>/dev/null
  92         op_to_source --source-dir=${LUSTREDIR}/mds/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/mds4mds_extN/  ${LUSTREDIR}/mds/mds_extN.o 2>/dev/null
  93
  94         op_to_source --source-dir=${PORTALSDIR}/linux/oslib/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/portals ${PORTALSDIR}/linux/oslib/portals.o 2>/dev/null
  95         op_to_source --source-dir=${PORTALSDIR}/linux/${NAL}/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/${NAL} ${PORTALSDIR}/linux/${NAL}/k${NAL}.o 2>/dev/null
  96         op_time -l > ${OUTPUTDIR}/${HOSTNAME}/oprofile/globalprofile 2>/dev/null
  97 }
  98
  99 iostat_stop() {
 100         echo "$(date +%T): iostat stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
 101         PID=$(cat $OUTPUTDIR/$HOSTNAME/tmp/iostat.pid)
 102         kill $PID
 103 }
 104
 105 vmstat_stop() {
 106         echo "$(date +%T): vmstat stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
 107         PID=$(cat $OUTPUTDIR/$HOSTNAME/tmp/vmstat.pid)
 108         kill $PID
 109 }
 110
 111 oprofile_dump() {
 112         op_dump;
 113 }
 114
 115 proc_dump_stop() {
 116         echo "$(date +%T): proc dump stopped...." >> $OUTPUTDIR/$HOSTNAME/summary
 117         CTRFILE=$(cat /tmp/prof-ctrl | awk '$1 == "prof_dump_ctrl" {print $2}')
 118         rm -f $CTRFILE
 119 }
 120
 121 case "$1" in
 122         start)
 123                 shift;
 124                 while [ ${#*} -gt 1 ]; do
 125                         case "$1" in
 126                                 -k)
 127                                         shift;
 128                                         KERNELDIR=$1;
 129                                         ;;
 130                                 -l)
 131                                         shift;
 132                                         LUSTREDIR=$1;
 133                                         ;;
 134                                 -p)
 135                                         shift;
 136                                         PORTALSDIR=$1;
 137                                         ;;
 138                                 -o)
 139                                         shift;
 140                                         OUTPUTDIR=$1;
 141                                         ;;
 142                                 *)
 143                                         echo unrecognized option $1
 144                                         break;
 145                                         ;;
 146                         esac
 147                         shift;
 148                 done
 149
 150                 echo "kerneldir $KERNELDIR" > /tmp/prof-ctrl
 151                 echo -e "\nlustredir $LUSTREDIR" >> /tmp/prof-ctrl
 152                 echo -e "\nportalsdir $PORTALSDIR" >> /tmp/prof-ctrl
 153                 echo -e "\noutputdir $OUTPUTDIR" >> /tmp/prof-ctrl
 154
 155                 if [ -d ${OUTPUTDIR}/${HOSTNAME} ]; then
 156                         echo "Output already exists"
 157                         echo "Please take backup and remove it"
 158                         exit 1
 159                 fi
 160
 161                 mkdir -p ${OUTPUTDIR}/${HOSTNAME}
 162                 echo -e "Profiling started on $HOSTNAME" >  ${OUTPUTDIR}/${HOSTNAME}/summary
 163                 echo -e "\n\nModules Listing on $HOSTNAME" >> ${OUTPUTDIR}/${HOSTNAME}/summary
 164                 /sbin/lsmod >> ${OUTPUTDIR}/${HOSTNAME}/summary
 165                 echo -e "\n\nKernel : " >> ${OUTPUTDIR}/${HOSTNAME}/summary
 166                 uname -a  >> ${OUTPUTDIR}/${HOSTNAME}/summary
 167                 echo -e "\n\nPCI Devices : "  >> ${OUTPUTDIR}/${HOSTNAME}/summary
 168                 lspci -t -v  >> ${OUTPUTDIR}/${HOSTNAME}/summary
 169                 echo -e "\n\nTests carried out " >> ${OUTPUTDIR}/${HOSTNAME}/summary
 170                 mkdir $OUTPUTDIR/$HOSTNAME/tmp
 171
 172                 for test in $TESTS; do
 173                         ${test}_start;
 174                 done
 175         ;;
 176         stop)
 177                 KERNELDIR=$(cat /tmp/prof-ctrl | awk '$1 == "kerneldir" {print $2}')
 178                 LUSTREDIR=$(cat /tmp/prof-ctrl | awk '$1 == "lustredir" {print $2}')
 179                 PORTALSDIR=$(cat /tmp/prof-ctrl | awk '$1 == "portalsdir" {print $2}')
 180                 OUTPUTDIR=$(cat /tmp/prof-ctrl | awk '$1 == "outputdir" {print $2}')
 181
 182                 for test in $TESTS; do
 183                         ${test}_stop;
 184                 done
 185
 186                 rm -rf ${OUTPUTDIR}/$HOSTNAME/tmp
 187                 tar -cf ${OUTPUTDIR}/${HOSTNAME}.tar  ${OUTPUTDIR}/${HOSTNAME}
 188                 echo "Dumped results in ${OUTPUTDIR}/${HOSTNAME}.tar"
 189         ;;
 190         dump)
 191                 if "oprofile" in $TESTS; then
 192                         oprofile_dump;
 193                 fi
 194         ;;
 195         clean)
 196                 OUTPUTDIR=$(cat /tmp/prof-ctrl | awk '$1 == "outputdir" {print $2}')
 197
 198                 echo Deleting directory $OUTPUTDIR/$HOSTNAME ...
 199                 rm -rf $OUTPUTDIR/$HOSTNAME
 200                 echo Deleting file $OUTPUTDIR/${HOSTNAME}.tar ...
 201                 rm -rf $OUTPUTDIR/${HOSTNAME}.tar
 202                 echo Deleting oprofile samples ...
 203                 rm -f /var/lib/oprofile/samples/*
 204                 rm -f /tmp/prof-ctrl
 205         ;;
 206         *)
 207                 echo $"Usage : $0 {start|stop|dump|clean} [OPTIONS]";
 208                 echo $"OPTIONS :"
 209                 echo $" -l lustre_dir"
 210                 echo $" -p portals_dir"
 211                 echo $" -k kernel_dir"
 212                 echo $" -o output_dir"
 213                 exit 1
 214 esac
 215
 216 exit 0