lustre/scripts/system-profile.sh

   1 #!/bin/sh
   2
   3 #       System Profiling Script
   4
   5 TESTS="oprofile iostat vmstat proc_dump"
   6
   7 # common parameters
   8 export OUTPUTDIR=/home/op
   9 export KERNELDIR=/usr/src/linux
  10 export LUSTREDIR=/usr/src/lustre
  11 export PORTALSDIR=/usr/src/portals
  12 NAL=socknal
  13
  14 # Params for OPROFILE
  15 #CTR0_EVENT=CPU_CLK_UNHALTED
  16 CTR0_COUNT=10000
  17
  18 # for intel Petium 4 onwards... Also requires Unit Mask
  19 CTR0_EVENT=GLOBAL_POWER_EVENTS
  20 CTR0_UNIT_MASK=0x01
  21
  22 # Params for VMSTAT
  23 VM_SAMPLING=1
  24
  25 # Params for IOSTAT
  26 IO_SAMPLING=1
  27
  28 # Params for PROC DUMP
  29 export PROC_SAMPLING=2
  30
  31 #-------------------------------------------------------------------
  32 oprofile_start() {
  33
  34         echo `date +%T`": oprofile started...." >>  $OUTPUTDIR/$HOSTNAME/summary
  35         op_start --ctr0-event=$CTR0_EVENT --ctr0-count=$CTR0_COUNT --ctr0-unit-mask=$CTR0_UNIT_MASK --vmlinux=${KERNELDIR}/vmlinux
  36         mkdir -p ${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/{obdclass,obdecho,osc,ptlrpc,extN,obdfilter,ost,mdc,mds4mds,mds4mds_extN,llite,portals,$NAL}
  37         mkdir -p ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling
  38
  39 }
  40
  41 iostat_start() {
  42         echo `date +%T`": iostat started...." >>  $OUTPUTDIR/$HOSTNAME/summary
  43
  44         mkdir ${OUTPUTDIR}/${HOSTNAME}/iostat
  45
  46         iostat $IO_SAMPLING > $OUTPUTDIR/$HOSTNAME/iostat/iostat.op &
  47         PID=$!
  48         echo $PID > $OUTPUTDIR/$HOSTNAME/tmp/iostat.pid
  49 }
  50
  51 vmstat_start() {
  52
  53         echo $OUTPUTDIR;
  54         echo `date +%T`": vmstat started...." >>  $OUTPUTDIR/$HOSTNAME/summary
  55
  56         mkdir ${OUTPUTDIR}/${HOSTNAME}/vmstat
  57
  58         vmstat $VM_SAMPLING > $OUTPUTDIR/$HOSTNAME/vmstat/vmstat.op &
  59         PID=$!
  60         echo $PID > $OUTPUTDIR/$HOSTNAME/tmp/vmstat.pid
  61 }
  62
  63 proc_dump_start() {
  64         echo `date +%T`": proc dump started...." >> $OUTPUTDIR/$HOSTNAME/summary
  65         sh -c '
  66                 CTRFILE=/$OUTPUTDIR/$HOSTNAME/tmp/running.$$.pid;
  67                 echo $$ > ${CTRFILE};
  68                 while [ -f $CTRFILE ]; do
  69                         cat /proc/meminfo >> $OUTPUTDIR/$HOSTNAME/meminfo;
  70                         cat /proc/interrupts >> $OUTPUTDIR/$HOSTNAME/interrupts;
  71                         cat /proc/net/dev >> $OUTPUTDIR/$HOSTNAME/net-dev;
  72                         sleep ${PROC_SAMPLING};
  73                 done;
  74                 ' &
  75         CTRFILE=/tmp/running.$!.pid
  76         echo "proc_dump_ctrl $CTRFILE" >> /tmp/prof-ctrl
  77
  78 }
  79
  80 oprofile_stop() {
  81         op_dump
  82         op_stop
  83
  84         echo `date +%T`": oprofile stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
  85
  86         for i in obdclass obdecho osc ptlrpc extN obdfilter mds ost mdc llite
  87         do
  88                 oprofpp -l ${LUSTREDIR}/${i}/${i}.o >  ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/${i}.prof 2>/dev/null
  89         done
  90         oprofpp -l ${LUSTREDIR}/mds/mds_extN.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/mds_extN.prof 2>/dev/null
  91         oprofpp -l ${PORTALSDIR}/linux/oslib/portals.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/portals.prof 2>/dev/null
  92         oprofpp -l ${PORTALSDIR}/linux/${NAL}/k${NAL}.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/k${NAL}.prof 2>/dev/null
  93
  94
  95         for i in obdclass obdecho osc ptlrpc extN obdfilter ost mdc llite
  96         do
  97                 op_to_source --source-dir=${LUSTREDIR}/${i}/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/${i}/ ${LUSTREDIR}/${i}/${i}.o 2>/dev/null
  98         done
  99
 100         op_to_source --source-dir=${LUSTREDIR}/mds/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/mds4mds/ ${LUSTREDIR}/mds/mds.o 2>/dev/null
 101         op_to_source --source-dir=${LUSTREDIR}/mds/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/mds4mds_extN/  ${LUSTREDIR}/mds/mds_extN.o 2>/dev/null
 102
 103         op_to_source --source-dir=${PORTALSDIR}/linux/oslib/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/portals ${PORTALSDIR}/linux/oslib/portals.o 2>/dev/null
 104         op_to_source --source-dir=${PORTALSDIR}/linux/${NAL}/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/${NAL} ${PORTALSDIR}/linux/${NAL}/k${NAL}.o 2>/dev/null
 105
 106         op_time -l > ${OUTPUTDIR}/${HOSTNAME}/oprofile/globalprofile 2>/dev/null
 107
 108 }
 109
 110 iostat_stop() {
 111         echo `date +%T`": iostat stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
 112
 113         PID=$(cat $OUTPUTDIR/$HOSTNAME/tmp/iostat.pid)
 114         kill $PID
 115 }
 116
 117 vmstat_stop() {
 118         echo `date +%T`": vmstat stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
 119
 120         PID=$(cat $OUTPUTDIR/$HOSTNAME/tmp/vmstat.pid)
 121         kill $PID
 122 }
 123
 124 oprofile_dump() {
 125         op_dump;
 126 }
 127
 128 proc_dump_stop() {
 129         echo `date +%T`": proc dump stopped...." >> $OUTPUTDIR/$HOSTNAME/summary
 130         CTRFILE=`cat /tmp/prof-ctrl | awk '$1 == "prof_dump_ctrl" {print $2}'`
 131         rm -f $CTRFILE
 132 }
 133 #-------------------------------------------------------------------
 134
 135 case "$1" in
 136
 137         start)
 138
 139                 shift;
 140                 while [ ${#*} -gt 1 ]; do
 141                         case "$1" in
 142                                 -k)
 143                                         shift;
 144                                         KERNELDIR=$1;
 145                                         ;;
 146
 147                                 -l)
 148                                         shift;
 149                                         LUSTREDIR=$1;
 150                                         ;;
 151                                 -p)
 152                                         shift;
 153                                         PORTALSDIR=$1;
 154                                         ;;
 155
 156                                 -o)
 157                                         shift;
 158                                         OUTPUTDIR=$1;
 159                                         ;;
 160                                  *)
 161                                         echo unrecognized option $1
 162                                         break;
 163                                         ;;
 164                         esac
 165                         shift;
 166                 done
 167                 echo "kerneldir $KERNELDIR" > /tmp/prof-ctrl
 168                 echo -e "\nlustredir $LUSTREDIR" >> /tmp/prof-ctrl
 169                 echo -e "\nportalsdir $PORTALSDIR" >> /tmp/prof-ctrl
 170                 echo -e "\noutputdir $OUTPUTDIR" >> /tmp/prof-ctrl
 171
 172                 if [ -d ${OUTPUTDIR}/${HOSTNAME} ]; then
 173                         echo "Output already exists"
 174                         echo "Please take backup and remove it"
 175                         exit 1
 176                 fi
 177
 178                 mkdir -p ${OUTPUTDIR}/${HOSTNAME}
 179                 echo -e "Profiling started on $HOSTNAME" >  ${OUTPUTDIR}/${HOSTNAME}/summary
 180                 echo -e "\n\nModules Listing on $HOSTNAME" >> ${OUTPUTDIR}/${HOSTNAME}/summary
 181                 /sbin/lsmod >> ${OUTPUTDIR}/${HOSTNAME}/summary
 182                 echo -e "\n\nKernel : " >> ${OUTPUTDIR}/${HOSTNAME}/summary
 183                 uname -a  >> ${OUTPUTDIR}/${HOSTNAME}/summary
 184                 echo -e "\n\nPCI Devices : "  >> ${OUTPUTDIR}/${HOSTNAME}/summary
 185                 lspci -t -v  >> ${OUTPUTDIR}/${HOSTNAME}/summary
 186                 echo -e "\n\nTests carried out " >> ${OUTPUTDIR}/${HOSTNAME}/summary
 187
 188                 mkdir $OUTPUTDIR/$HOSTNAME/tmp
 189                 for test in $TESTS; do
 190                         ${test}_start;
 191                 done
 192         ;;
 193         stop)
 194
 195                 KERNELDIR=`cat /tmp/prof-ctrl | awk '$1 == "kerneldir" {print $2}'`
 196                 LUSTREDIR=`cat /tmp/prof-ctrl | awk '$1 == "lustredir" {print $2}'`
 197                 PORTALSDIR=`cat /tmp/prof-ctrl | awk '$1 == "portalsdir" {print $2}'`
 198                 OUTPUTDIR=`cat /tmp/prof-ctrl | awk '$1 == "outputdir" {print $2}'`
 199                 for test in $TESTS; do
 200                         ${test}_stop;
 201                 done
 202
 203                 rm -rf ${OUTPUTDIR}/$HOSTNAME/tmp
 204                 tar -cf ${OUTPUTDIR}/${HOSTNAME}.tar  ${OUTPUTDIR}/${HOSTNAME}
 205                 echo "Dumped results in ${OUTPUTDIR}/${HOSTNAME}.tar"
 206         ;;
 207         dump)
 208                 if "oprofile" in $TESTS; then
 209                         oprofile_dump;
 210                 fi
 211         ;;
 212         clean)
 213
 214                 OUTPUTDIR=`cat /tmp/prof-ctrl | awk '$1 == "outputdir" {print $2}'`
 215                 echo Deleting directory $OUTPUTDIR/$HOSTNAME ...
 216                 rm -rf $OUTPUTDIR/$HOSTNAME
 217                 echo Deleting file $OUTPUTDIR/${HOSTNAME}.tar ...
 218                 rm -rf $OUTPUTDIR/${HOSTNAME}.tar
 219                 echo Deleting oprofile samples ...
 220                 rm -f /var/lib/oprofile/samples/*
 221                 rm -f /tmp/prof-ctrl
 222         ;;
 223         *)
 224                 echo $"Usage : $0 {start|stop|dump|clean} [OPTIONS]";
 225                 echo $"OPTIONS :"
 226                 echo $" -l lustre_dir"
 227                 echo $" -p portals_dir"
 228                 echo $" -k kernel_dir"
 229                 echo $" -o output_dir"
 230                 exit 1
 231 esac
 232
 233 exit 0