2 * Copyright (C) 2009-2012 Cray, Inc.
4 * Copyright (c) 2014, Intel Corporation.
6 * Author: Nic Henke <nic@cray.com>, James Shimek <jshimek@cray.com>
8 * This file is part of Lustre, http://www.lustre.org.
10 * Lustre is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU General Public
12 * License as published by the Free Software Foundation.
14 * Lustre is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with Lustre; if not, write to the Free Software
21 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 #ifndef _GNILND_ARIES_H
25 #define _GNILND_ARIES_H
27 /* for lnet_ipif_query */
28 #include <lnet/lib-lnet.h>
30 #ifndef _GNILND_HSS_OPS_H
31 # error "must include gnilnd_hss_ops.h first"
34 /* Set HW related values */
36 #include <aries/aries_timeouts_gpl.h>
38 /* from aries_timeouts_gpl.h when building for generic kernel */
39 #define TIMEOUT_SECS(x) ((uint64_t)(((x) / 1000.0) + 0.5))
40 #ifndef TO_GNILND_timeout
41 #define TO_GNILND_timeout (60000.000000)
42 #endif /* TO_GNILND_timeout */
43 #endif /* CONFIG_CRAY_XT */
45 #define GNILND_BASE_TIMEOUT TIMEOUT_SECS(TO_GNILND_timeout)
46 #define GNILND_CHECKSUM_DEFAULT 0 /* all off for Aries */
48 #if defined(CONFIG_CRAY_COMPUTE)
49 #define GNILND_REVERSE_RDMA GNILND_REVERSE_PUT
50 #define GNILND_RDMA_DLVR_OPTION GNI_DLVMODE_PERFORMANCE
52 #define GNILND_REVERSE_RDMA GNILND_REVERSE_GET
53 #define GNILND_RDMA_DLVR_OPTION GNI_DLVMODE_PERFORMANCE
54 #define GNILND_SCHED_THREADS 7 /* scheduler threads */
57 /* Thread-safe kgni implemented in minor ver 45, code rev 0xb9 */
58 #define GNILND_KGNI_TS_MINOR_VER 0x45
59 #define GNILND_TS_ENABLE 1
61 /* register some memory to allocate a shared mdd */
62 static inline gni_return_t
63 kgnilnd_register_smdd_buf(kgn_device_t *dev)
65 __u32 flags = GNI_MEM_READWRITE;
67 if (*kgnilnd_tunables.kgn_bte_relaxed_ordering) {
68 flags |= GNI_MEM_RELAXED_PI_ORDERING;
71 LIBCFS_ALLOC(dev->gnd_smdd_hold_buf, PAGE_SIZE);
72 if (!dev->gnd_smdd_hold_buf) {
73 CERROR("Can't allocate smdd hold buffer\n");
74 return GNI_RC_ERROR_RESOURCE;
77 return kgnilnd_mem_register(dev->gnd_handle,
78 (__u64)dev->gnd_smdd_hold_buf,
79 PAGE_SIZE, NULL, flags,
80 &dev->gnd_smdd_hold_hndl);
83 static inline gni_return_t
84 kgnilnd_deregister_smdd_buf(kgn_device_t *dev)
86 gni_return_t rc = kgnilnd_mem_deregister(dev->gnd_handle,
87 &dev->gnd_smdd_hold_hndl, 0);
88 LIBCFS_FREE(dev->gnd_smdd_hold_buf, PAGE_SIZE);
93 /* plug in our functions for use on the simulator */
94 #if !defined(GNILND_USE_RCA)
96 extern kgn_data_t kgnilnd_data;
98 #define kgnilnd_hw_hb() do {} while(0)
100 #ifdef CONFIG_CRAY_XT
102 /* Aries Sim doesn't have hardcoded tables, so we'll hijack the nic_pe
103 * and decode our address and nic addr from that - the rest are just offsets */
106 kgnilnd_nid_to_nicaddrs(__u32 nid, int numnic, __u32 *nicaddr)
109 CERROR("manual nid2nic translation doesn't support"
110 "multiple nic addrs (you asked for %d)\n",
114 if (nid < kgnilnd_data.kgn_nid_trans_private) {
115 CERROR("Request for invalid nid translation %u,"
117 nid, kgnilnd_data.kgn_nid_trans_private);
121 *nicaddr = nid - kgnilnd_data.kgn_nid_trans_private;
123 CDEBUG(D_NETTRACE, "Sim nid %d -> nic 0x%x\n", nid, *nicaddr);
129 kgnilnd_nicaddr_to_nid(__u32 nicaddr, __u32 *nid)
131 *nid = kgnilnd_data.kgn_nid_trans_private + nicaddr;
135 /* XXX Nic: This does not support multiple device!!!! */
137 kgnilnd_setup_nic_translation(__u32 device_id)
139 char *if_name = "ipogif0";
140 __u32 ipaddr, netmask, my_nid;
143 LCONSOLE_INFO("using Aries SIM IP info for RCA translation\n");
145 rc = lnet_ipif_query(if_name, &up, &ipaddr, &netmask);
147 CERROR ("can't get IP interface for %s: %d\n", if_name, rc);
151 CERROR ("IP interface %s is down\n", if_name);
155 my_nid = ((ipaddr >> 8) & 0xFF) + (ipaddr & 0xFF);
156 kgnilnd_data.kgn_nid_trans_private = my_nid - device_id;
161 #else /* CONFIG_CRAY_XT */
162 #include <net/inet_common.h>
163 #include <linux/if_arp.h>
166 kgnilnd_nid_to_nicaddrs(__u32 nid, int numnic, __u32 *nicaddrs)
170 #define NID_MASK ((1ULL << 18) - 1)
172 struct arpreq req = {
173 .arp_dev = "ipogif0",
176 req.arp_pa.sa_family = AF_INET;
177 ((struct sockaddr_in *)&req.arp_pa)->sin_addr.s_addr = htonl(nid);
182 rc = inet_ioctl(kgnilnd_data.kgn_sock, SIOCGARP, (unsigned long)&req);
186 CDEBUG(D_NETERROR, "inet_ioctl returned %d\n", rc);
190 /* use the lower 18 bits of the mac address to use as a nid value */
191 *nicaddrs = *(__u32 *)&req.arp_ha.sa_data[2];
192 *nicaddrs = ntohl(*nicaddrs) & NID_MASK;
194 CDEBUG(D_NETTRACE, "nid %s -> nic 0x%x\n", libcfs_nid2str(nid),
201 kgnilnd_nicaddr_to_nid(__u32 nicaddr, __u32 *nid)
206 .ifr_name = "ipogif0",
209 struct sockaddr_in* ipaddr = (struct sockaddr_in*)&ifr.ifr_addr;
213 rc = inet_ioctl(kgnilnd_data.kgn_sock, SIOCGIFADDR, (unsigned long)&ifr);
217 CDEBUG(D_NETERROR, "inet_ioctl returned %d\n", rc);
221 CDEBUG(D_NETTRACE, "ipaddr %08x\n", htonl(ipaddr->sin_addr.s_addr));
223 *nid = htonl(ipaddr->sin_addr.s_addr);
224 CDEBUG(D_NETTRACE, "nic 0x%x -> nid %s\n", nicaddr,
225 libcfs_nid2str(*nid));
230 kgnilnd_setup_nic_translation(__u32 device_id)
235 #endif /* CONFIG_CRAY_XT */
237 #endif /* GNILND_USE_RCA */
239 #endif /* _GNILND_ARIES_H */