4 Copyright 2015-2019 Cray Inc. All Rights Reserved
5 Dumps the Lustre RPC queues for all ptlrpcd_XX threads.
8 from pykdump.API import *
13 import lustrelib as ll
14 from crashlib.input import toint
16 from traceback import print_exc
18 description_short = "Displays the RPC queues of the Lustre ptlrpcd daemons"
20 def print_separator(count):
22 for idx in xrange(count):
26 def print_title(title):
29 print "%-14s %-6s %-19s %-18s %-19s %-4s %-14s %-4s %-22s %-19s" \
30 % ("thread", "pid", "ptlrpc_request", "xid", "nid", "opc",
31 "phase:flags", "R:W", "sent/deadline", "ptlrpc_body")
35 return type('Enum', (), enums)
37 REQ_Q = enum(rq_list=1, replay_list=2, set_chain=3, ctx_chain=4,
38 unreplied_list=5, timed_list=5, exp_list=6, hist_list=7)
41 REQ_Q.rq_list: ['struct ptlrpc_request', 'rq_list', 'rq_type'],
42 REQ_Q.replay_list: ['struct ptlrpc_request', 'rq_replay_list', 'rq_type'],
43 REQ_Q.set_chain: ['struct ptlrpc_cli_req', 'cr_set_chain', 'rq_cli'],
44 REQ_Q.ctx_chain: ['struct ptlrpc_cli_req', 'cr_ctx_chain', 'rq_cli'],
45 REQ_Q.unreplied_list: ['struct ptlrpc_cli_req', 'cr_unreplied_list', 'rq_cli'],
46 REQ_Q.timed_list: ['struct ptlrpc_srv_req', 'sr_timed_list', 'rq_srv'],
47 REQ_Q.exp_list: ['struct ptlrpc_srv_req', 'sr_exp_list', 'rq_srv'],
48 REQ_Q.hist_list: ['struct ptlrpc_srv_req', 'sr_hist_list', 'rq_srv']
56 return ((val + 7) & (~0x7))
58 LUSTRE_MSG_MAGIC_V2 = 0x0BD00BD3
60 def get_ptlrpc_body(req):
63 if not msg or msg == None:
66 if msg.lm_magic != LUSTRE_MSG_MAGIC_V2:
69 bufcount = msg.lm_bufcount
73 buflen = msg.lm_buflens[0]
74 if buflen < getSizeOf('struct ptlrpc_body_v2'):
77 offset = member_offset('struct lustre_msg_v2', 'lm_buflens')
79 buflen_size = getSizeOf("unsigned int")
80 offset += buflen_size * bufcount
81 offset = size_round(offset)
82 addr = Addr(msg) + offset
86 return readSU('struct ptlrpc_body_v2', addr)
88 RQ_PHASE_NEW = 0xebc0de00
89 RQ_PHASE_RPC = 0xebc0de01
90 RQ_PHASE_BULK = 0xebc0de02
91 RQ_PHASE_INTERPRET = 0xebc0de03
92 RQ_PHASE_COMPLETE = 0xebc0de04
93 RQ_PHASE_UNREG_RPC = 0xebc0de05
94 RQ_PHASE_UNREG_BULK = 0xebc0de06
95 RQ_PHASE_UNDEFINED = 0xebc0de07
100 RQ_PHASE_BULK: "BULK",
101 RQ_PHASE_INTERPRET: "NtrPrt",
102 RQ_PHASE_COMPLETE: "COMP",
103 RQ_PHASE_UNREG_RPC: "UNREG",
104 RQ_PHASE_UNREG_BULK: "UNBULK",
105 RQ_PHASE_UNDEFINED: "UNDEF"
108 FLAG_LEGEND = "\nFlag Legend:\n\n" + \
109 "I - rq_intr\tR - rq_replied\t\tE - rq_err\te - rq_net_err\tX - rq_timedout\tS - rq_resend\t\tT - rq_restart\n" + \
110 "P - rq_replay\tN - rq_no_resend\tW - rq_waiting\tC - rq_wait\tH - rq_hp\tM - rq_committed\tq - rq_req_unlinked\tu - rq_reply_unlinked\n"
112 def get_phase_flags(req):
114 phasestr = PHASES.get(phase & 0xffffffff, "?%d" % phase)
115 return "%s:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s" % \
117 "I" if req.rq_intr else "",
118 "R" if req.rq_replied else "",
119 "E" if req.rq_err else "",
120 "e" if req.rq_net_err else "",
121 "X" if req.rq_timedout else "",
122 "S" if req.rq_resend else "",
123 "T" if req.rq_restart else "",
124 "P" if req.rq_replay else "",
125 "N" if req.rq_no_resend else "",
126 "W" if req.rq_waiting else "",
127 "C" if req.rq_wait_ctx else "",
128 "H" if req.rq_hp else "",
129 "M" if req.rq_committed else "",
130 "q" if req.rq_req_unlinked else "",
131 "u" if req.rq_reply_unlinked else "")
133 LP_POISON = 0x5a5a5a5a5a5a5a5a
135 def print_one_request(sthread, req):
136 pb = get_ptlrpc_body(req)
141 status = pb.pb_status
147 obd_name = "Invalid Import"
148 if req.rq_import and req.rq_import != 0xffffffffffffffff and \
149 req.rq_import != LP_POISON:
150 imp_invalid = req.rq_import.imp_invalid
151 obd_name = ll.obd2str(req.rq_import.imp_obd)
153 if not imp_invalid and req.rq_import.imp_connection:
154 nid = ll.nid2str(req.rq_import.imp_connection.c_peer.nid)
155 brw = "%1d:%1d" % (req.rq_bulk_read, req.rq_bulk_write)
156 rq_sent_dl = "%d/%d" % (req.rq_sent, req.rq_deadline)
157 print "%-14s %-6s 0x%-17x %-18d %-19s %-4d %-14s %-4s %-22s 0x%-17x" % \
164 get_phase_flags(req),
169 def print_request_list(sthread, lhdr, loffset):
171 for reqlnk in readStructNext(lhdr, 'next'):
172 if reqlnk.next == Addr(lhdr):
174 req = readSU('struct ptlrpc_request', reqlnk.next-loffset)
175 print_one_request(sthread, req)
182 # Find offset from start of ptlrpc_request struct of link field
183 # Adjusts for links that are contained in embedded union
184 def get_linkfld_offset(lfld):
185 container = RQ_LIST_LNKS[lfld][STRUCT_IDX]
186 linkfld = RQ_LIST_LNKS[lfld][MEMBER_IDX]
187 req_union = RQ_LIST_LNKS[lfld][UNION_IDX]
189 off1 = member_offset('struct ptlrpc_request', req_union)
190 off2 = member_offset(container, linkfld)
193 def foreach_ptlrpcd_ctl(callback, *args):
194 pinfo_rpcds = readSymbol('ptlrpcds')
195 pinfo_count = readSymbol('ptlrpcds_num')
197 for idx in xrange(pinfo_count):
198 ptlrpcd = pinfo_rpcds[idx]
199 for jdx in xrange(ptlrpcd.pd_nthreads):
200 pd = ptlrpcd.pd_threads[jdx]
202 pd = readSymbol('ptlrpcd_rcv')
205 def get_daemon_listhdrs(pd, sent_rpcs, pend_rpcs):
206 sent_rpcs.append([pd.pc_name, pd.pc_set.set_requests])
207 pend_rpcs.append([pd.pc_name, pd.pc_set.set_new_requests])
209 def dump_list_of_lists(rpc_list, loffset):
210 for qinfo in rpc_list:
211 sthread, lhdr = qinfo
212 print_request_list(sthread, lhdr, loffset)
214 def dump_daemon_rpclists():
218 foreach_ptlrpcd_ctl(get_daemon_listhdrs, sent_rpcs, pend_rpcs)
219 offset = get_linkfld_offset(REQ_Q.set_chain)
221 print_title("Sent RPCS: ptlrpc_request_set.set_requests->")
222 dump_list_of_lists(sent_rpcs, offset)
224 print_title("Pending RPCS: ptlrpc_request_set.set_new_requests->")
225 dump_list_of_lists(pend_rpcs, offset)
228 def print_overview_entry(pd):
229 s = "%s:" % pd.pc_name
230 print "%-14s ptlrpcd_ctl 0x%x ptlrpc_request_set 0x%x" % \
231 (s, Addr(pd), pd.pc_set)
234 foreach_ptlrpcd_ctl(print_overview_entry)
236 def print_pcset_stats(pd):
237 if pd.pc_set.set_new_count.counter != 0 or \
238 pd.pc_set.set_remaining.counter != 0:
239 s = "%s:" %pd.pc_name
240 print "%-13s 0x%-18x %-4d %-4d %-6d" % \
242 pd.pc_set.set_refcount.counter,
243 pd.pc_set.set_new_count.counter,
244 pd.pc_set.set_remaining.counter)
247 print '%-14s %-19s %-4s %-4s %-6s' % \
248 ("thread", "ptlrpc_request_set", "ref", "new", "remain")
250 foreach_ptlrpcd_ctl(print_pcset_stats)
252 def dump_one_rpc(addr):
253 print_title("Request")
254 req = readSU('struct ptlrpc_request', addr)
255 print_one_request('', req)
257 def dump_one_rpclist(addr, link_fld):
258 lhdr = readSU('struct list_head', addr)
260 loffset = get_linkfld_offset(d[link_fld])
262 print_title("Request list at %s" % lhdr)
263 print_request_list('', lhdr, loffset)
265 def dump_rpcs_cmd(args):
274 dump_one_rpclist(args.rpc_addr[0], args.link_fld)
276 # dump_one_rpc(args.rpc_addr[0])
277 dump_one_rpc(args.rpc_addr)
279 dump_daemon_rpclists()
281 if __name__ == "__main__":
282 # usage = "$(prog)s [-o] [-s] [-l link_field] [addr]\n" + \
284 "Displays lists of Lustre RPC requests. If no arguments are \n" +\
285 "specified, all rpcs in the sent and pending queues of the \n" +\
286 "ptlrpcd daemons are printed. If an address is specified, it \n" +\
287 "must be a pointer to either a ptlrpc_request or a list_head \n" +\
288 "struct. If the addr is a list_head, then a link_field must \n" +\
289 "also be provided. The link_field identifies the member of \n" +\
290 "the ptlrpc_request struct used to link the list together."
292 parser = argparse.ArgumentParser(
293 formatter_class=argparse.RawDescriptionHelpFormatter,
294 description=description, epilog=FLAG_LEGEND)
295 parser.add_argument("-o", dest="oflag", action="store_true",
296 help="print overview of ptlrpcd_XX threads with ptlrpcd_ctl " + \
297 "structs and the associated pc_set field")
298 parser.add_argument("-s", dest="sflag", action="store_true",
299 help="print rpc counts per ptlrpc_request_set")
300 parser.add_argument("-l", dest="link_fld", default="",
301 choices=['rq_list', 'replay_list', 'set_chain', 'ctx_chain',
302 'unreplied_list', 'timed_list', 'exp_list', 'hist_list'],
303 help="name of link field in ptlrpc_request for list headed by addr")
304 parser.add_argument("rpc_addr", nargs="?", default=[], type=toint,
305 help="address of either single ptlrpc_request or list_head; list_head requires a -l argument")
306 args = parser.parse_args()