lustre/scripts/lustre_req_history

   1 #!/bin/sh
   2 # Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
   3
   4 # Purpose:
   5 # This script displays the history of requests from the local client
   6 # to the Lustre servers.
   7
   8 # Requires: tcp network (ksocklnd), ssh
   9 # In order to execute this script the user needs interaction-free scp access
  10 # to all the servers
  11
  12 # Limitations:
  13 # The req_history files only keep a limited amount of data, so the results
  14 # may not be a complete history.
  15
  16 REQ_FILES_OST="ost/OSS/ost_io/req_history \
  17 ost/OSS/ost_create/req_history \
  18 ost/OSS/ost/req_history"
  19
  20 REQ_FILES_MDT="ldlm/services/ldlm_canceld/req_history \
  21 ldlm/services/ldlm_cbd/req_history \
  22 mdt/MDS/mds_readpage/req_history \
  23 mdt/MDS/mds_setattr/req_history \
  24 mdt/MDS/mds/req_history"
  25
  26 REQ_FILES_CLIENT="ldlm/services/ldlm_canceld/req_history \
  27 ldlm/services/ldlm_cbd/req_history"
  28
  29 if [ $#  -gt  0 ]; then
  30         echo "Usage: $0"
  31         echo "  This script collects the history of ptlrpc requests on"
  32         echo "the servers from this client. The results are sorted by"
  33         echo "the transaction ID."
  34         exit 1
  35 fi
  36
  37 tempip=`lctl list_nids | cut -d "@" -f1`
  38
  39 echo $tempip | egrep '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'
  40 if [ $? -ne 0 ] ; then
  41         CLIENT=`host $tempip | cut -d" " -f4`
  42 else
  43         CLIENT=$tempip
  44 fi
  45
  46 TMP_DIR_CLIENT=`mktemp -d /tmp/src_req_history.XXXXX`
  47 chmod ugo+rwx $TMP_DIR_CLIENT
  48 MDS_NODE=$(lctl get_param -n mdc.*.mds_conn_uuid | cut -d @ -f1)
  49 TARGET="OSS"
  50 i=0
  51 for NODE in $(lctl get_param -n osc.*-osc-*.ost_conn_uuid | cut -d @ -f1) ; do
  52         NODE_ARRAY[$i]=$NODE
  53         i=`expr $i + 1`
  54 done
  55
  56 # Get the Histories from all the OSS's
  57
  58 k=0
  59 for NODE in $(lctl get_param -n osc.*-osc-*.ost_conn_uuid | cut -d @ -f1) ; do
  60
  61         SAME_OST=0
  62         j=0
  63         while [ $j -lt $k ]
  64         do
  65                 if [ ${NODE_ARRAY[$j]} = $NODE ]
  66                 then
  67                         SAME_OST=1
  68                fi
  69                 j=`expr $j + 1`
  70         done
  71         k=`expr $k + 1`
  72         if [ $SAME_OST -eq 1 ]
  73         then
  74                 continue
  75         fi
  76         TMPFILE1=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
  77         TMPFILE2=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
  78         rm -f $TMPFILE1
  79         rm -f $TMPFILE2
  80         ( echo touch $TMPFILE1
  81         echo touch $TMPFILE2
  82         echo chmod go+rw $TMPFILE1
  83         echo chmod go+rw $TMPFILE2
  84         for FILE in $REQ_FILES_OST; do
  85                 SERVICE=`echo $FILE | cut -d "/" -f3`
  86                 echo "lctl get_param -n $FILE |cut -d\" \" -f1 | sed s/$/:$TARGET/| sed s/$/:$SERVICE/ >> $TMPFILE1"
  87         done
  88         for FILE in $REQ_FILES_CLIENT; do
  89                 SERVICE=`echo $FILE | cut -d "/" -f3`
  90                 echo "lctl get_param -n $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1"
  91         done
  92         if [ $CLIENT = $NODE ]
  93         then
  94                 echo "grep -r 0@lo $TMPFILE1 >> $TMPFILE2"
  95         else
  96                 echo "grep -r $CLIENT $TMPFILE1 >> $TMPFILE2"
  97         fi
  98         ) | ssh $NODE
  99         scp $NODE:$TMPFILE2 $TMP_DIR_CLIENT/
 100         echo rm -f $TMPFILE1 $TMPFILE2 | ssh $NODE
 101 done
 102
 103 # Get the Histories from the MDS
 104
 105 TARGET="MDS"
 106 for NODE in $(lctl get_param -n mdc.*.mds_conn_uuid | cut -d @ -f1); do
 107         TMPFILE1=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
 108         rm -f $TMPFILE1
 109         TMPFILE2=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
 110         rm -f $TMPFILE2
 111         ( echo touch $TMPFILE1
 112         echo touch $TMPFILE2
 113         echo chmod go+rw $TMPFILE1
 114         echo chmod go+rw $TMPFILE2
 115         for FILE in $REQ_FILES_MDT; do
 116                 SERVICE=`echo $FILE | cut -d "/" -f3`
 117                 echo "lctl get_param -n $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1"
 118         done
 119         if [ $CLIENT = $NODE ]
 120         then
 121                 echo "grep -r 0@lo $TMPFILE1 >> $TMPFILE2"
 122         else
 123                 echo "grep -r $CLIENT $TMPFILE1 >> $TMPFILE2"
 124         fi
 125         ) | ssh $NODE
 126         scp $NODE:$TMPFILE2 $TMP_DIR_CLIENT/
 127         echo rm -f $TMPFILE1 $TMPFILE2 | ssh $NODE
 128 done
 129
 130 # Get the Histories from the CLIENT
 131
 132 TMPFILE1=`mktemp $TMP_DIR_CLIENT/temp_histories_$CLIENT.XXXXXXXXXX`
 133 TMPFILE2=`mktemp $TMP_DIR_CLIENT/temp_histories_$CLIENT.XXXXXXXXXX`
 134 TARGET="CLIENT"
 135
 136 for FILE in $REQ_FILES_CLIENT; do
 137         SERVICE=`echo $FILE | cut -d "/" -f3`
 138         lctl get_param -n $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1
 139 done
 140
 141 grep -r 0@lo $TMPFILE1 >> $TMPFILE2
 142 rm -f $TMPFILE1
 143
 144 TMPFILE_DEST=`mktemp /tmp/req_histories.XXXXXXXXXX`
 145 TMPFILE_SORT=`mktemp /tmp/req_histories_sorted.XXXXXXXXXX`
 146
 147 # Merge Histories from OSS's, MDS and CLIENT
 148
 149 for NODE in `ls $TMP_DIR_CLIENT` ; do
 150         cat $TMP_DIR_CLIENT/$NODE >> $TMPFILE_DEST
 151 done
 152
 153 # Sort the histories w.r.t XID
 154
 155 sort -t : -k 4,4n $TMPFILE_DEST >> $TMPFILE_SORT
 156 rm -f $TMPFILE_DEST
 157 rm -rf $TMP_DIR_CLIENT
 158
 159 echo "XID          Target       Service        Seq                 Status"
 160 awk -F: '{ printf "%-10d %-10s %-15s   %-10d \t   %-15s \n", $4, $7, $8, $1, $6 }' $TMPFILE_SORT
 161 rm -f $TMPFILE_SORT