#!/bin/sh
# Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.

# Purpose:
# This script displays the history of requests from the local client  
# to the Lustre servers.

# Requires: tcp network (ksocklnd), ssh
# In order to execute this script the user needs interaction-free scp access
# to all the servers

# Limitations:
# The req_history files only keep a limited amount of data, so the results
# may not be a complete history.

REQ_FILES_OST="ost/OSS/ost_io/req_history \
ost/OSS/ost_create/req_history \
ost/OSS/ost/req_history"

REQ_FILES_MDT="ldlm/services/ldlm_canceld/req_history \
ldlm/services/ldlm_cbd/req_history \
mdt/MDS/mds_readpage/req_history \
mdt/MDS/mds_setattr/req_history \
mdt/MDS/mds/req_history"

REQ_FILES_CLIENT="ldlm/services/ldlm_canceld/req_history \
ldlm/services/ldlm_cbd/req_history"

if [ $#  -gt  0 ]; then
	echo "Usage: $0"
	echo "  This script collects the history of ptlrpc requests on"
	echo "the servers from this client. The results are sorted by" 
	echo "the transaction ID."
	exit 1
fi

tempip=`lctl list_nids | cut -d "@" -f1`

echo $tempip | egrep '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'
if [ $? -ne 0 ] ; then
        CLIENT=`host $tempip | cut -d" " -f4`
else
        CLIENT=$tempip
fi

TMP_DIR_CLIENT=`mktemp -d /tmp/src_req_history.XXXXX`
chmod ugo+rwx $TMP_DIR_CLIENT
MDS_NODE=`cat /proc/fs/lustre/mdc/*/mds_conn_uuid | cut -d @ -f1`
TARGET="OSS"
i=0
for NODE in `cat /proc/fs/lustre/osc/*-osc-*/ost_conn_uuid | cut -d @ -f1` ; do
	NODE_ARRAY[$i]=$NODE
	i=`expr $i + 1`
done

# Get the Histories from all the OSS's

k=0
for NODE in `cat /proc/fs/lustre/osc/*-osc-*/ost_conn_uuid | cut -d @ -f1` ; do

	SAME_OST=0
	j=0
	while [ $j -lt $k ]
	do
		if [ ${NODE_ARRAY[$j]} = $NODE ]
		then
			SAME_OST=1
               fi
                j=`expr $j + 1`
	done
	k=`expr $k + 1` 
	if [ $SAME_OST -eq 1 ]
        then
                continue
        fi
	TMPFILE1=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
	TMPFILE2=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
	rm -f $TMPFILE1
	rm -f $TMPFILE2
	( echo cd /proc/fs/lustre
	echo touch $TMPFILE1
	echo touch $TMPFILE2
	echo chmod go+rw $TMPFILE1
	echo chmod go+rw $TMPFILE2
	for FILE in $REQ_FILES_OST; do
		SERVICE=`echo $FILE | cut -d "/" -f3`
		echo "cat $FILE |cut -d\" \" -f1 | sed s/$/:$TARGET/| sed s/$/:$SERVICE/ >> $TMPFILE1"
	done
	for FILE in $REQ_FILES_CLIENT; do
                SERVICE=`echo $FILE | cut -d "/" -f3`
                echo "cat $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1"
        done
	if [ $CLIENT = $NODE ]
	then
		echo "grep -r 0@lo $TMPFILE1 >> $TMPFILE2"
	else
		echo "grep -r $CLIENT $TMPFILE1 >> $TMPFILE2"
	fi
	) | ssh $NODE
	scp $NODE:$TMPFILE2 $TMP_DIR_CLIENT/
	echo rm -f $TMPFILE1 $TMPFILE2 | ssh $NODE
done

# Get the Histories from the MDS

TARGET="MDS"
for NODE in `cat /proc/fs/lustre/mdc/*/mds_conn_uuid | cut -d @ -f1`; do
	TMPFILE1=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
        rm -f $TMPFILE1
	TMPFILE2=`mktemp /tmp/temp_histories_$NODE.XXXXXXXXXX`
        rm -f $TMPFILE2
	( echo cd /proc/fs/lustre
	echo touch $TMPFILE1
	echo touch $TMPFILE2
	echo chmod go+rw $TMPFILE1
	echo chmod go+rw $TMPFILE2
	for FILE in $REQ_FILES_MDT; do
		SERVICE=`echo $FILE | cut -d "/" -f3`
		echo "cat $FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1"
	done
	if [ $CLIENT = $NODE ]
        then
		echo "grep -r 0@lo $TMPFILE1 >> $TMPFILE2"
	else
		echo "grep -r $CLIENT $TMPFILE1 >> $TMPFILE2"
	fi
	) | ssh $NODE
	scp $NODE:$TMPFILE2 $TMP_DIR_CLIENT/
        echo rm -f $TMPFILE1 $TMPFILE2 | ssh $NODE
done

# Get the Histories from the CLIENT

TMPFILE1=`mktemp $TMP_DIR_CLIENT/temp_histories_$CLIENT.XXXXXXXXXX`
TMPFILE2=`mktemp $TMP_DIR_CLIENT/temp_histories_$CLIENT.XXXXXXXXXX`
TARGET="CLIENT"

for FILE in $REQ_FILES_CLIENT; do
	SERVICE=`echo $FILE | cut -d "/" -f3`
	cat /proc/fs/lustre/$FILE | sed s/$/:$TARGET/ | sed s/$/:$SERVICE/ >> $TMPFILE1
done

grep -r 0@lo $TMPFILE1 >> $TMPFILE2
rm -f $TMPFILE1

TMPFILE_DEST=`mktemp /tmp/req_histories.XXXXXXXXXX`
TMPFILE_SORT=`mktemp /tmp/req_histories_sorted.XXXXXXXXXX`

# Merge Histories from OSS's, MDS and CLIENT

for NODE in `ls $TMP_DIR_CLIENT` ; do
	cat $TMP_DIR_CLIENT/$NODE >> $TMPFILE_DEST
done

# Sort the histories w.r.t XID

sort -t : -k 4,4n $TMPFILE_DEST >> $TMPFILE_SORT
rm -f $TMPFILE_DEST
rm -rf $TMP_DIR_CLIENT

echo "XID  	   Target	Service        Seq  	    	   Status"
awk -F: '{ printf "%-10d %-10s %-15s   %-10d \t   %-15s \n", $4, $7, $8, $1, $6 }' $TMPFILE_SORT
rm -f $TMPFILE_SORT