From 620026ea62f08f2c10ed226974941e5e1175d71c Mon Sep 17 00:00:00 2001 From: rread Date: Thu, 29 Aug 2002 23:50:47 +0000 Subject: [PATCH] * send and recv tcp buffer size is set * router opens connections to gateway --- lustre/utils/lconf | 73 ++++++++++++++++++++++++++++++++++++++++++++++-------- lustre/utils/lmc | 17 ++++++++++--- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/lustre/utils/lconf b/lustre/utils/lconf index 1e34a8b..68f1bd4 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -32,7 +32,7 @@ import xml.dom.minidom # Global parameters TCP_ACCEPTOR = '' - +MAXTCPBUF = 1048576 # # Maximum number of devices to search for. # (the /dev/loop* nodes need to be created beforehand) @@ -283,6 +283,15 @@ class LCTLInterface: quit """ % (net, gw, lo, hi) self.run(cmds) + + # add a route to a range + def del_route(self, net, gw, lo, hi): + cmds = """ + network %s + del_route %s %s + quit """ % (net, lo, hi) + self.run(cmds) + # add a route to a host def add_route_host(self, net, uuid, gw, tgt): cmds = """ @@ -640,8 +649,8 @@ class Network(Module): self.net_type = get_attr(dom_node,'type') self.nid = get_text(dom_node, 'server', '*') self.port = get_text_int(dom_node, 'port', 0) - self.send_buf = get_text_int(dom_node, 'send_buf', 65536) - self.read_buf = get_text_int(dom_node, 'read_buf', 65536) + self.send_mem = get_text_int(dom_node, 'send_mem', 65536) + self.recv_mem = get_text_int(dom_node, 'recv_mem', 65536) if self.nid == '*': self.nid = get_local_address(self.net_type) if not self.nid: @@ -660,21 +669,40 @@ class Network(Module): def prepare(self): self.info(self.net_type, self.nid, self.port) if self.net_type == 'tcp': - ret = run_daemon(TCP_ACCEPTOR, self.port) + ret = run_daemon(TCP_ACCEPTOR, '-s', self.send_mem, '-r', self.recv_mem, self.port) if ret: raise CommandError(TCP_ACCEPTOR, 'failed', ret) - ret = self.dom_node.getElementsByTagName('route_tbl') for a in ret: for r in a.getElementsByTagName('route'): - lctl.add_route(self.net_type, self.nid, get_attr(r, 'lo'), - get_attr(r,'hi', '')) + lo = get_attr(r, 'lo') + hi = get_attr(r,'hi', '') + lctl.add_route(self.net_type, self.nid, lo, hi) + if self.net_type == 'tcp' and hi == '': + srv = nid2server(self.dom_node.parentNode.parentNode, lo) + if not srv: + panic("no server for nid", lo) + else: + lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem) + lctl.network(self.net_type, self.nid) lctl.newdev(attach = "ptlrpc RPCDEV") def cleanup(self): self.info(self.net_type, self.nid, self.port) + ret = self.dom_node.getElementsByTagName('route_tbl') + for a in ret: + for r in a.getElementsByTagName('route'): + lo = get_attr(r, 'lo') + hi = get_attr(r,'hi', '') + if self.net_type == 'tcp' and hi == '': + srv = nid2server(self.dom_node.parentNode.parentNode, lo) + if not srv: + panic("no server for nid", lo) + else: + lctl.disconnect(srv.net_type, srv.nid, srv.port, srv.uuid) + lctl.del_route(self.net_type, self.nid, lo, hi) try: lctl.cleanup("RPCDEV", "") except CommandError, e: @@ -750,7 +778,7 @@ class MDC(Module): def prepare(self): self.info(self.mds_uuid) srv = self.get_server() - lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_buf, srv.read_buf) + lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem) lctl.newdev(attach="mdc %s %s" % (self.name, self.uuid), setup ="%s %s" %(self.mds_uuid, srv.uuid)) @@ -804,7 +832,7 @@ class OSC(Module): self.info(self.obd_uuid, self.ost_uuid) srv = self.get_server() if local_net(srv): - lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_buf, srv.read_buf) + lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem) else: r = find_route(srv) if r: @@ -961,6 +989,13 @@ def get_ost_net(dom_node, uuid): if not uuid: return None return lookup(dom_node, uuid) + +def nid2server(dom_node, nid): + netlist = dom_node.getElementsByTagName('network') + for net_node in netlist: + if get_text(net_node, 'server') == nid: + return Network(net_node) + return None def lookup(dom_node, uuid): for n in dom_node.childNodes: @@ -1281,6 +1316,21 @@ def sys_set_debug_path(): except IOError, e: print e +#/proc/sys/net/core/rmem_max +#/proc/sys/net/core/wmem_max +def sys_set_netmem_max(path, max): + debug("setting", path, "to at least", max) + if config.noexec(): + return + fp = open(path) + str = fp.readline() + fp.close + cur = int(str) + if max > cur: + fp = open(path, 'w') + fp.write('%d\n' %(max)) + fp.close() + def sys_make_devices(): if not os.access('/dev/portals', os.R_OK): @@ -1294,7 +1344,7 @@ def sys_make_devices(): # Shutdown does steps in reverse # def main(): - global TCP_ACCEPTOR, lctl + global TCP_ACCEPTOR, lctl, MAXTCPBUF args = parse_cmdline(sys.argv[1:]) if len(args) > 0: if not os.access(args[0], os.R_OK | os.W_OK): @@ -1329,6 +1379,8 @@ def main(): setupModulePath(sys.argv[0]) sys_make_devices() + sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF) + sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF) doHost(dom.documentElement, node_list) if __name__ == "__main__": @@ -1339,4 +1391,3 @@ if __name__ == "__main__": except CommandError, e: e.dump() - diff --git a/lustre/utils/lmc b/lustre/utils/lmc index d280452..2ae47a2 100755 --- a/lustre/utils/lmc +++ b/lustre/utils/lmc @@ -198,13 +198,17 @@ class GenConfig: node.appendChild(new) return new - def network(self, name, uuid, hostname, net, port=0): + def network(self, name, uuid, hostname, net, port=0, tcpbuf=0): """create node""" network = self.newService("network", name, uuid) network.setAttribute("type", net); self.addElement(network, "server", hostname) if port: self.addElement(network, "port", "%d" %(port)) + if tcpbuf: + self.addElement(network, "send_mem", "%d" %(tcpbuf)) + self.addElement(network, "recv_mem", "%d" %(tcpbuf)) + return network def route(self, lo, hi): @@ -412,13 +416,16 @@ def add_net(gen, lustre, options, args): node_name = options['node'] nid = args[0] net_type = args[1] + port = 0 + tcpbuf = 0 if net_type == 'tcp': if len(args) > 2: port = int(args[2]) else: port = DEFAULT_PORT - # add send, recv buffer size here + if options.has_key('tcpbuf'): + tcpbuf = int(options['tcpbuf']) elif net_type in ('elan', 'gm'): port = 0 else: @@ -432,7 +439,7 @@ def add_net(gen, lustre, options, args): node = ret net_name = new_name('NET_'+ node_name +'_'+ net_type) net_uuid = new_uuid(net_name) - node.appendChild(gen.network(net_name, net_uuid, nid, net_type, port)) + node.appendChild(gen.network(net_name, net_uuid, nid, net_type, port, tcpbuf)) node_add_profile(gen, node, "network", net_uuid) @@ -657,7 +664,7 @@ def add_mtpt(gen, lustre, options, args): # def parse_cmdline(argv): short_opts = "ho:i:m:" - long_opts = ["ost", "osc", "mtpt", "lov=", "node=", "mds=", "net", + long_opts = ["ost", "osc", "mtpt", "lov=", "node=", "mds=", "net", "tcpbuf=", "mdc", "route", "router", "merge=", "format", "reformat", "output=", "obdtype=", "in=", "help"] opts = [] @@ -698,6 +705,8 @@ def parse_cmdline(argv): options['merge'] = a if o == "--obdtype": options['obdtype'] = a + if o == "--tcpbuf": + options['tcpbuf'] = a if o == "--format": options['format'] = 1 if o == "--reformat": -- 1.8.3.1