lustre/scripts/lustre_req_history

   1 #!/bin/sh
   2 # Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
   3
   4 # Purpose:
   5 # This script displays the history of requests from the local client
   6 # to the Lustre servers.
   7
   8 # Requires: tcp network (ksocklnd), ssh
   9 # In order to execute this script the user needs interaction-free scp access
  10 # to all the servers
  11
  12 # Limitations:
  13 # The req_history files only keep a limited amount of data, so the results
  14 # may not be a complete history.
  15
  16 REQ_FILES_OST="ost/OSS/ost_io/req_history \
  17 ost/OSS/ost_create/req_history \
  18 ost/OSS/ost/req_history"
  19
  20 REQ_FILES_MDT="ldlm/services/ldlm_canceld/req_history \
  21 ldlm/services/ldlm_cbd/req_history \
  22 mdt/MDS/mds_readpage/req_history \
  23 mdt/MDS/mds_setattr/req_history \
  24 mdt/MDS/mds/req_history"
  25
  26 REQ_FILES_CLIENT="ldlm/services/ldlm_canceld/req_history \
  27 ldlm/services/ldlm_cbd/req_history"
  28
  29 if [ $#  -gt  0 ]; then
  30         echo "Usage: $0"
  31         echo "  This script collects the history of ptlrpc requests on"
  32         echo "the servers from this client. The results are sorted by"
  33         echo "the transaction ID."
  34         exit 1
  35 fi
  36
  37 tempip=`lctl list_nids | cut -d "@" -f1`
  38
  39 echo $tempip | egrep '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'
  40 if [ $? -ne 0 ] ; then
  41         CLIENT=`host $tempip | cut -d" " -f4`
  42 else
  43         CLIENT=$tempip
  44 fi
  45
  46 TMP_DIR_CLIENT=`mktemp -d /tmp/src_req_history.XXXXX`
  47 chmod ugo+rwx $TMP_DIR_CLIENT
  48 MDS_NODE=`cat /proc/fs/lustre/mdc/*/mds_conn_uuid | cut -d @ -f1`
  49 TARGET="OSS"
  50 i=0
  51 for NODE in `cat /proc/fs/lustre/osc/*-osc-*/ost_conn_uuid | cut -d @ -f1` ; do
  52         NODE_ARRAY[$i]=$NODE
  53         i=`expr $i + 1`
  54 done
  55
  56 # Get the Histories from all the OSS's
  57
  58 k=0
  59 for NODE in `cat /proc/fs/lustre/osc/*-osc-*/ost_conn_uuid | cut -d @ -f1` ; do
  60
  61         SAME_OST=0
  62         j=0
  63         while [ $j -lt $k ]
  64         do
  65                 if [ ${NODE_ARRAY[$j]} = $NODE ]
  66                 then
  67                         SAME_OST=1
  68                fi
  69                 j=`expr $j + 1`
  70         done
  71         k=`expr $k + 1`
  72         if [ $SAME_OST -eq 1 ]
  73         then
  74                 continue
  75         fi
  76         TMPFILE1=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
  77         TMPFILE2=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
  78         rm -f $TMPFILE1
  79         rm -f $TMPFILE2
  80         ( echo cd /proc/fs/lustre
  81         echo touch $TMPFILE1
  82         echo touch $TMPFILE2
  83         echo chmod go+rw $TMPFILE1
  84         echo chmod go+rw $TMPFILE2
  85         for FILE in $REQ_FILES_OST; do
  86                 SERVICE=`echo $FILE | cut -d "/" -f3`
  87                 echo "cat $FILE |cut -d\" \" -f1 | sed s/$/:$TARGET/| sed s/$/:$SERVICE/ >> $TMPFILE1"
  88         done
  89         for FILE in $REQ_FILES_CLIENT; do
  90                 SERVICE=`echo $FILE | cut -d "/" -f3`
  91                 echo "cat $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1"
  92         done
  93         if [ $CLIENT = $NODE ]
  94         then
  95                 echo "grep -r 0@lo $TMPFILE1 >> $TMPFILE2"
  96         else
  97                 echo "grep -r $CLIENT $TMPFILE1 >> $TMPFILE2"
  98         fi
  99         ) | ssh $NODE
 100         scp $NODE:$TMPFILE2 $TMP_DIR_CLIENT/
 101         echo rm -f $TMPFILE1 $TMPFILE2 | ssh $NODE
 102 done
 103
 104 # Get the Histories from the MDS
 105
 106 TARGET="MDS"
 107 for NODE in `cat /proc/fs/lustre/mdc/*/mds_conn_uuid | cut -d @ -f1`; do
 108         TMPFILE1=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
 109         rm -f $TMPFILE1
 110         TMPFILE2=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
 111         rm -f $TMPFILE2
 112         ( echo cd /proc/fs/lustre
 113         echo touch $TMPFILE1
 114         echo touch $TMPFILE2
 115         echo chmod go+rw $TMPFILE1
 116         echo chmod go+rw $TMPFILE2
 117         for FILE in $REQ_FILES_MDT; do
 118                 SERVICE=`echo $FILE | cut -d "/" -f3`
 119                 echo "cat $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1"
 120         done
 121         if [ $CLIENT = $NODE ]
 122         then
 123                 echo "grep -r 0@lo $TMPFILE1 >> $TMPFILE2"
 124         else
 125                 echo "grep -r $CLIENT $TMPFILE1 >> $TMPFILE2"
 126         fi
 127         ) | ssh $NODE
 128         scp $NODE:$TMPFILE2 $TMP_DIR_CLIENT/
 129         echo rm -f $TMPFILE1 $TMPFILE2 | ssh $NODE
 130 done
 131
 132 # Get the Histories from the CLIENT
 133
 134 TMPFILE1=`mktemp $TMP_DIR_CLIENT/temp_histories_$CLIENT.XXXXXXXXXX`
 135 TMPFILE2=`mktemp $TMP_DIR_CLIENT/temp_histories_$CLIENT.XXXXXXXXXX`
 136 TARGET="CLIENT"
 137
 138 for FILE in $REQ_FILES_CLIENT; do
 139         SERVICE=`echo $FILE | cut -d "/" -f3`
 140         cat /proc/fs/lustre/$FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1
 141 done
 142
 143 grep -r 0@lo $TMPFILE1 >> $TMPFILE2
 144 rm -f $TMPFILE1
 145
 146 TMPFILE_DEST=`mktemp /tmp/req_histories.XXXXXXXXXX`
 147 TMPFILE_SORT=`mktemp /tmp/req_histories_sorted.XXXXXXXXXX`
 148
 149 # Merge Histories from OSS's, MDS and CLIENT
 150
 151 for NODE in `ls $TMP_DIR_CLIENT` ; do
 152         cat $TMP_DIR_CLIENT/$NODE >> $TMPFILE_DEST
 153 done
 154
 155 # Sort the histories w.r.t XID
 156
 157 sort -t : -k 4,4n $TMPFILE_DEST >> $TMPFILE_SORT
 158 rm -f $TMPFILE_DEST
 159 rm -rf $TMP_DIR_CLIENT
 160
 161 echo "XID          Target       Service        Seq                 Status"
 162 awk -F: '{ printf "%-10d %-10s %-15s   %-10d \t   %-15s \n", $4, $7, $8, $1, $6 }' $TMPFILE_SORT
 163 rm -f $TMPFILE_SORT