lnet/lnet/api-ni.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  */
  31
  32 #define DEBUG_SUBSYSTEM S_LNET
  33
  34 #include <linux/ctype.h>
  35 #include <linux/generic-radix-tree.h>
  36 #include <linux/log2.h>
  37 #include <linux/ktime.h>
  38 #include <linux/moduleparam.h>
  39 #include <linux/uaccess.h>
  40 #ifdef HAVE_SCHED_HEADERS
  41 #include <linux/sched/signal.h>
  42 #endif
  43 #include <net/genetlink.h>
  44
  45 #include <libcfs/linux/linux-net.h>
  46 #include <lnet/udsp.h>
  47 #include <lnet/lib-lnet.h>
  48
  49 #define D_LNI D_CONSOLE
  50
  51 /*
  52  * initialize ln_api_mutex statically, since it needs to be used in
  53  * discovery_set callback. That module parameter callback can be called
  54  * before module init completes. The mutex needs to be ready for use then.
  55  */
  56 struct lnet the_lnet = {
  57         .ln_api_mutex = __MUTEX_INITIALIZER(the_lnet.ln_api_mutex),
  58 };              /* THE state of the network */
  59 EXPORT_SYMBOL(the_lnet);
  60
  61 static char *ip2nets = "";
  62 module_param(ip2nets, charp, 0444);
  63 MODULE_PARM_DESC(ip2nets, "LNET network <- IP table");
  64
  65 static char *networks = "";
  66 module_param(networks, charp, 0444);
  67 MODULE_PARM_DESC(networks, "local networks");
  68
  69 static char *routes = "";
  70 module_param(routes, charp, 0444);
  71 MODULE_PARM_DESC(routes, "routes to non-local networks");
  72
  73 static int rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT;
  74 module_param(rnet_htable_size, int, 0444);
  75 MODULE_PARM_DESC(rnet_htable_size, "size of remote network hash table");
  76
  77 static int use_tcp_bonding;
  78 module_param(use_tcp_bonding, int, 0444);
  79 MODULE_PARM_DESC(use_tcp_bonding,
  80                  "use_tcp_bonding parameter has been removed");
  81
  82 unsigned int lnet_numa_range = 0;
  83 module_param(lnet_numa_range, uint, 0444);
  84 MODULE_PARM_DESC(lnet_numa_range,
  85                 "NUMA range to consider during Multi-Rail selection");
  86
  87 /*
  88  * lnet_health_sensitivity determines by how much we decrement the health
  89  * value on sending error. The value defaults to 100, which means health
  90  * interface health is decremented by 100 points every failure.
  91  */
  92 unsigned int lnet_health_sensitivity = 100;
  93 static int sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp);
  94 #ifdef HAVE_KERNEL_PARAM_OPS
  95 static struct kernel_param_ops param_ops_health_sensitivity = {
  96         .set = sensitivity_set,
  97         .get = param_get_int,
  98 };
  99 #define param_check_health_sensitivity(name, p) \
 100                 __param_check(name, p, int)
 101 module_param(lnet_health_sensitivity, health_sensitivity, S_IRUGO|S_IWUSR);
 102 #else
 103 module_param_call(lnet_health_sensitivity, sensitivity_set, param_get_int,
 104                   &lnet_health_sensitivity, S_IRUGO|S_IWUSR);
 105 #endif
 106 MODULE_PARM_DESC(lnet_health_sensitivity,
 107                 "Value to decrement the health value by on error");
 108
 109 /*
 110  * lnet_recovery_interval determines how often we should perform recovery
 111  * on unhealthy interfaces.
 112  */
 113 unsigned int lnet_recovery_interval = 1;
 114 static int recovery_interval_set(const char *val, cfs_kernel_param_arg_t *kp);
 115 #ifdef HAVE_KERNEL_PARAM_OPS
 116 static struct kernel_param_ops param_ops_recovery_interval = {
 117         .set = recovery_interval_set,
 118         .get = param_get_int,
 119 };
 120 #define param_check_recovery_interval(name, p) \
 121                 __param_check(name, p, int)
 122 module_param(lnet_recovery_interval, recovery_interval, S_IRUGO|S_IWUSR);
 123 #else
 124 module_param_call(lnet_recovery_interval, recovery_interval_set, param_get_int,
 125                   &lnet_recovery_interval, S_IRUGO|S_IWUSR);
 126 #endif
 127 MODULE_PARM_DESC(lnet_recovery_interval,
 128                 "DEPRECATED - Interval to recover unhealthy interfaces in seconds");
 129
 130 unsigned int lnet_recovery_limit;
 131 module_param(lnet_recovery_limit, uint, 0644);
 132 MODULE_PARM_DESC(lnet_recovery_limit,
 133                  "How long to attempt recovery of unhealthy peer interfaces in seconds. Set to 0 to allow indefinite recovery");
 134
 135 unsigned int lnet_max_recovery_ping_interval = 900;
 136 unsigned int lnet_max_recovery_ping_count = 9;
 137 static int max_recovery_ping_interval_set(const char *val,
 138                                           cfs_kernel_param_arg_t *kp);
 139
 140 #define param_check_max_recovery_ping_interval(name, p) \
 141                 __param_check(name, p, int)
 142
 143 #ifdef HAVE_KERNEL_PARAM_OPS
 144 static struct kernel_param_ops param_ops_max_recovery_ping_interval = {
 145         .set = max_recovery_ping_interval_set,
 146         .get = param_get_int,
 147 };
 148 module_param(lnet_max_recovery_ping_interval, max_recovery_ping_interval, 0644);
 149 #else
 150 module_param_call(lnet_max_recovery_ping_interval, max_recovery_ping_interval,
 151                   param_get_int, &lnet_max_recovery_ping_interval, 0644);
 152 #endif
 153 MODULE_PARM_DESC(lnet_max_recovery_ping_interval,
 154                  "The max interval between LNet recovery pings, in seconds");
 155
 156 static int lnet_interfaces_max = LNET_INTERFACES_MAX_DEFAULT;
 157 static int intf_max_set(const char *val, cfs_kernel_param_arg_t *kp);
 158
 159 static struct kernel_param_ops param_ops_interfaces_max = {
 160         .set = intf_max_set,
 161         .get = param_get_int,
 162 };
 163
 164 #define param_check_interfaces_max(name, p) \
 165                 __param_check(name, p, int)
 166
 167 #ifdef HAVE_KERNEL_PARAM_OPS
 168 module_param(lnet_interfaces_max, interfaces_max, 0644);
 169 #else
 170 module_param_call(lnet_interfaces_max, intf_max_set, param_get_int,
 171                   &param_ops_interfaces_max, 0644);
 172 #endif
 173 MODULE_PARM_DESC(lnet_interfaces_max,
 174                 "Maximum number of interfaces in a node.");
 175
 176 unsigned lnet_peer_discovery_disabled = 0;
 177 static int discovery_set(const char *val, cfs_kernel_param_arg_t *kp);
 178
 179 static struct kernel_param_ops param_ops_discovery_disabled = {
 180         .set = discovery_set,
 181         .get = param_get_int,
 182 };
 183
 184 #define param_check_discovery_disabled(name, p) \
 185                 __param_check(name, p, int)
 186 #ifdef HAVE_KERNEL_PARAM_OPS
 187 module_param(lnet_peer_discovery_disabled, discovery_disabled, 0644);
 188 #else
 189 module_param_call(lnet_peer_discovery_disabled, discovery_set, param_get_int,
 190                   &param_ops_discovery_disabled, 0644);
 191 #endif
 192 MODULE_PARM_DESC(lnet_peer_discovery_disabled,
 193                 "Set to 1 to disable peer discovery on this node.");
 194
 195 unsigned int lnet_drop_asym_route;
 196 static int drop_asym_route_set(const char *val, cfs_kernel_param_arg_t *kp);
 197
 198 static struct kernel_param_ops param_ops_drop_asym_route = {
 199         .set = drop_asym_route_set,
 200         .get = param_get_int,
 201 };
 202
 203 #define param_check_drop_asym_route(name, p)    \
 204         __param_check(name, p, int)
 205 #ifdef HAVE_KERNEL_PARAM_OPS
 206 module_param(lnet_drop_asym_route, drop_asym_route, 0644);
 207 #else
 208 module_param_call(lnet_drop_asym_route, drop_asym_route_set, param_get_int,
 209                   &param_ops_drop_asym_route, 0644);
 210 #endif
 211 MODULE_PARM_DESC(lnet_drop_asym_route,
 212                  "Set to 1 to drop asymmetrical route messages.");
 213
 214 #define LNET_TRANSACTION_TIMEOUT_DEFAULT 150
 215 unsigned int lnet_transaction_timeout = LNET_TRANSACTION_TIMEOUT_DEFAULT;
 216 static int transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp);
 217 #ifdef HAVE_KERNEL_PARAM_OPS
 218 static struct kernel_param_ops param_ops_transaction_timeout = {
 219         .set = transaction_to_set,
 220         .get = param_get_int,
 221 };
 222
 223 #define param_check_transaction_timeout(name, p) \
 224                 __param_check(name, p, int)
 225 module_param(lnet_transaction_timeout, transaction_timeout, S_IRUGO|S_IWUSR);
 226 #else
 227 module_param_call(lnet_transaction_timeout, transaction_to_set, param_get_int,
 228                   &lnet_transaction_timeout, S_IRUGO|S_IWUSR);
 229 #endif
 230 MODULE_PARM_DESC(lnet_transaction_timeout,
 231                 "Maximum number of seconds to wait for a peer response.");
 232
 233 #define LNET_RETRY_COUNT_DEFAULT 2
 234 unsigned int lnet_retry_count = LNET_RETRY_COUNT_DEFAULT;
 235 static int retry_count_set(const char *val, cfs_kernel_param_arg_t *kp);
 236 #ifdef HAVE_KERNEL_PARAM_OPS
 237 static struct kernel_param_ops param_ops_retry_count = {
 238         .set = retry_count_set,
 239         .get = param_get_int,
 240 };
 241
 242 #define param_check_retry_count(name, p) \
 243                 __param_check(name, p, int)
 244 module_param(lnet_retry_count, retry_count, S_IRUGO|S_IWUSR);
 245 #else
 246 module_param_call(lnet_retry_count, retry_count_set, param_get_int,
 247                   &lnet_retry_count, S_IRUGO|S_IWUSR);
 248 #endif
 249 MODULE_PARM_DESC(lnet_retry_count,
 250                  "Maximum number of times to retry transmitting a message");
 251
 252 unsigned int lnet_response_tracking = 3;
 253 static int response_tracking_set(const char *val, cfs_kernel_param_arg_t *kp);
 254
 255 #ifdef HAVE_KERNEL_PARAM_OPS
 256 static struct kernel_param_ops param_ops_response_tracking = {
 257         .set = response_tracking_set,
 258         .get = param_get_int,
 259 };
 260
 261 #define param_check_response_tracking(name, p)  \
 262         __param_check(name, p, int)
 263 module_param(lnet_response_tracking, response_tracking, 0644);
 264 #else
 265 module_param_call(lnet_response_tracking, response_tracking_set, param_get_int,
 266                   &lnet_response_tracking, 0644);
 267 #endif
 268 MODULE_PARM_DESC(lnet_response_tracking,
 269                  "(0|1|2|3) LNet Internal Only|GET Reply only|PUT ACK only|Full Tracking (default)");
 270
 271 int lock_prim_nid = 1;
 272 module_param(lock_prim_nid, int, 0444);
 273 MODULE_PARM_DESC(lock_prim_nid,
 274                  "Whether nid passed down by Lustre is locked as primary");
 275
 276 #define LNET_LND_TIMEOUT_DEFAULT ((LNET_TRANSACTION_TIMEOUT_DEFAULT - 1) / \
 277                                   (LNET_RETRY_COUNT_DEFAULT + 1))
 278 unsigned int lnet_lnd_timeout = LNET_LND_TIMEOUT_DEFAULT;
 279 static void lnet_set_lnd_timeout(void)
 280 {
 281         lnet_lnd_timeout = max((lnet_transaction_timeout - 1) /
 282                                (lnet_retry_count + 1), 1U);
 283 }
 284
 285 /*
 286  * This sequence number keeps track of how many times DLC was used to
 287  * update the local NIs. It is incremented when a NI is added or
 288  * removed and checked when sending a message to determine if there is
 289  * a need to re-run the selection algorithm. See lnet_select_pathway()
 290  * for more details on its usage.
 291  */
 292 static atomic_t lnet_dlc_seq_no = ATOMIC_INIT(0);
 293
 294 struct lnet_fail_ping {
 295         struct lnet_processid           lfp_id;
 296         int                             lfp_errno;
 297         char                            lfp_msg[256];
 298 };
 299
 300 struct lnet_genl_ping_list {
 301         unsigned int                    lgpl_index;
 302         unsigned int                    lgpl_list_count;
 303         unsigned int                    lgpl_failed_count;
 304         signed long                     lgpl_timeout;
 305         struct lnet_nid                 lgpl_src_nid;
 306         GENRADIX(struct lnet_fail_ping) lgpl_failed;
 307         GENRADIX(struct lnet_processid) lgpl_list;
 308 };
 309
 310 static int lnet_ping(struct lnet_processid *id, struct lnet_nid *src_nid,
 311                      signed long timeout, struct lnet_genl_ping_list *plist,
 312                      int n_ids);
 313
 314 static int lnet_discover(struct lnet_processid *id, u32 force,
 315                          struct lnet_genl_ping_list *dlists);
 316
 317 static int
 318 sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp)
 319 {
 320         int rc;
 321         unsigned *sensitivity = (unsigned *)kp->arg;
 322         unsigned long value;
 323
 324         rc = kstrtoul(val, 0, &value);
 325         if (rc) {
 326                 CERROR("Invalid module parameter value for 'lnet_health_sensitivity'\n");
 327                 return rc;
 328         }
 329
 330         /*
 331          * The purpose of locking the api_mutex here is to ensure that
 332          * the correct value ends up stored properly.
 333          */
 334         mutex_lock(&the_lnet.ln_api_mutex);
 335
 336         if (value > LNET_MAX_HEALTH_VALUE) {
 337                 mutex_unlock(&the_lnet.ln_api_mutex);
 338                 CERROR("Invalid health value. Maximum: %d value = %lu\n",
 339                        LNET_MAX_HEALTH_VALUE, value);
 340                 return -EINVAL;
 341         }
 342
 343         if (*sensitivity != 0 && value == 0 && lnet_retry_count != 0) {
 344                 lnet_retry_count = 0;
 345                 lnet_set_lnd_timeout();
 346         }
 347
 348         *sensitivity = value;
 349
 350         mutex_unlock(&the_lnet.ln_api_mutex);
 351
 352         return 0;
 353 }
 354
 355 static int
 356 recovery_interval_set(const char *val, cfs_kernel_param_arg_t *kp)
 357 {
 358         CWARN("'lnet_recovery_interval' has been deprecated\n");
 359
 360         return 0;
 361 }
 362
 363 static int
 364 max_recovery_ping_interval_set(const char *val, cfs_kernel_param_arg_t *kp)
 365 {
 366         int rc;
 367         unsigned long value;
 368
 369         rc = kstrtoul(val, 0, &value);
 370         if (rc) {
 371                 CERROR("Invalid module parameter value for 'lnet_max_recovery_ping_interval'\n");
 372                 return rc;
 373         }
 374
 375         if (!value) {
 376                 CERROR("Invalid max ping timeout. Must be strictly positive\n");
 377                 return -EINVAL;
 378         }
 379
 380         /* The purpose of locking the api_mutex here is to ensure that
 381          * the correct value ends up stored properly.
 382          */
 383         mutex_lock(&the_lnet.ln_api_mutex);
 384         lnet_max_recovery_ping_interval = value;
 385         lnet_max_recovery_ping_count = 0;
 386         value >>= 1;
 387         while (value) {
 388                 lnet_max_recovery_ping_count++;
 389                 value >>= 1;
 390         }
 391         mutex_unlock(&the_lnet.ln_api_mutex);
 392
 393         return 0;
 394 }
 395
 396 static int
 397 discovery_set(const char *val, cfs_kernel_param_arg_t *kp)
 398 {
 399         int rc;
 400         unsigned *discovery_off = (unsigned *)kp->arg;
 401         unsigned long value;
 402         struct lnet_ping_buffer *pbuf;
 403
 404         rc = kstrtoul(val, 0, &value);
 405         if (rc) {
 406                 CERROR("Invalid module parameter value for 'lnet_peer_discovery_disabled'\n");
 407                 return rc;
 408         }
 409
 410         value = (value) ? 1 : 0;
 411
 412         /*
 413          * The purpose of locking the api_mutex here is to ensure that
 414          * the correct value ends up stored properly.
 415          */
 416         mutex_lock(&the_lnet.ln_api_mutex);
 417
 418         if (value == *discovery_off) {
 419                 mutex_unlock(&the_lnet.ln_api_mutex);
 420                 return 0;
 421         }
 422
 423         /*
 424          * We still want to set the discovery value even when LNet is not
 425          * running. This is the case when LNet is being loaded and we want
 426          * the module parameters to take effect. Otherwise if we're
 427          * changing the value dynamically, we want to set it after
 428          * updating the peers
 429          */
 430         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
 431                 *discovery_off = value;
 432                 mutex_unlock(&the_lnet.ln_api_mutex);
 433                 return 0;
 434         }
 435
 436         /* tell peers that discovery setting has changed */
 437         lnet_net_lock(LNET_LOCK_EX);
 438         pbuf = the_lnet.ln_ping_target;
 439         if (value)
 440                 pbuf->pb_info.pi_features &= ~LNET_PING_FEAT_DISCOVERY;
 441         else
 442                 pbuf->pb_info.pi_features |= LNET_PING_FEAT_DISCOVERY;
 443         lnet_net_unlock(LNET_LOCK_EX);
 444
 445         /* only send a push when we're turning off discovery */
 446         if (*discovery_off <= 0 && value > 0)
 447                 lnet_push_update_to_peers(1);
 448         *discovery_off = value;
 449
 450         mutex_unlock(&the_lnet.ln_api_mutex);
 451
 452         return 0;
 453 }
 454
 455 static int
 456 drop_asym_route_set(const char *val, cfs_kernel_param_arg_t *kp)
 457 {
 458         int rc;
 459         unsigned int *drop_asym_route = (unsigned int *)kp->arg;
 460         unsigned long value;
 461
 462         rc = kstrtoul(val, 0, &value);
 463         if (rc) {
 464                 CERROR("Invalid module parameter value for "
 465                        "'lnet_drop_asym_route'\n");
 466                 return rc;
 467         }
 468
 469         /*
 470          * The purpose of locking the api_mutex here is to ensure that
 471          * the correct value ends up stored properly.
 472          */
 473         mutex_lock(&the_lnet.ln_api_mutex);
 474
 475         if (value == *drop_asym_route) {
 476                 mutex_unlock(&the_lnet.ln_api_mutex);
 477                 return 0;
 478         }
 479
 480         *drop_asym_route = value;
 481
 482         mutex_unlock(&the_lnet.ln_api_mutex);
 483
 484         return 0;
 485 }
 486
 487 static int
 488 transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp)
 489 {
 490         int rc;
 491         unsigned *transaction_to = (unsigned *)kp->arg;
 492         unsigned long value;
 493
 494         rc = kstrtoul(val, 0, &value);
 495         if (rc) {
 496                 CERROR("Invalid module parameter value for 'lnet_transaction_timeout'\n");
 497                 return rc;
 498         }
 499
 500         /*
 501          * The purpose of locking the api_mutex here is to ensure that
 502          * the correct value ends up stored properly.
 503          */
 504         mutex_lock(&the_lnet.ln_api_mutex);
 505
 506         if (value <= lnet_retry_count || value == 0) {
 507                 mutex_unlock(&the_lnet.ln_api_mutex);
 508                 CERROR("Invalid value for lnet_transaction_timeout (%lu). "
 509                        "Has to be greater than lnet_retry_count (%u)\n",
 510                        value, lnet_retry_count);
 511                 return -EINVAL;
 512         }
 513
 514         if (value == *transaction_to) {
 515                 mutex_unlock(&the_lnet.ln_api_mutex);
 516                 return 0;
 517         }
 518
 519         *transaction_to = value;
 520         /* Update the lnet_lnd_timeout now that we've modified the
 521          * transaction timeout
 522          */
 523         lnet_set_lnd_timeout();
 524
 525         mutex_unlock(&the_lnet.ln_api_mutex);
 526
 527         return 0;
 528 }
 529
 530 static int
 531 retry_count_set(const char *val, cfs_kernel_param_arg_t *kp)
 532 {
 533         int rc;
 534         unsigned *retry_count = (unsigned *)kp->arg;
 535         unsigned long value;
 536
 537         rc = kstrtoul(val, 0, &value);
 538         if (rc) {
 539                 CERROR("Invalid module parameter value for 'lnet_retry_count'\n");
 540                 return rc;
 541         }
 542
 543         /*
 544          * The purpose of locking the api_mutex here is to ensure that
 545          * the correct value ends up stored properly.
 546          */
 547         mutex_lock(&the_lnet.ln_api_mutex);
 548
 549         if (lnet_health_sensitivity == 0 && value > 0) {
 550                 mutex_unlock(&the_lnet.ln_api_mutex);
 551                 CERROR("Can not set lnet_retry_count when health feature is turned off\n");
 552                 return -EINVAL;
 553         }
 554
 555         if (value > lnet_transaction_timeout) {
 556                 mutex_unlock(&the_lnet.ln_api_mutex);
 557                 CERROR("Invalid value for lnet_retry_count (%lu). "
 558                        "Has to be smaller than lnet_transaction_timeout (%u)\n",
 559                        value, lnet_transaction_timeout);
 560                 return -EINVAL;
 561         }
 562
 563         *retry_count = value;
 564
 565         /* Update the lnet_lnd_timeout now that we've modified the
 566          * retry count
 567          */
 568         lnet_set_lnd_timeout();
 569
 570         mutex_unlock(&the_lnet.ln_api_mutex);
 571
 572         return 0;
 573 }
 574
 575 static int
 576 intf_max_set(const char *val, cfs_kernel_param_arg_t *kp)
 577 {
 578         int value, rc;
 579
 580         rc = kstrtoint(val, 0, &value);
 581         if (rc) {
 582                 CERROR("Invalid module parameter value for 'lnet_interfaces_max'\n");
 583                 return rc;
 584         }
 585
 586         if (value < LNET_INTERFACES_MIN) {
 587                 CWARN("max interfaces provided are too small, setting to %d\n",
 588                       LNET_INTERFACES_MAX_DEFAULT);
 589                 value = LNET_INTERFACES_MAX_DEFAULT;
 590         }
 591
 592         *(int *)kp->arg = value;
 593
 594         return 0;
 595 }
 596
 597 static int
 598 response_tracking_set(const char *val, cfs_kernel_param_arg_t *kp)
 599 {
 600         int rc;
 601         unsigned long new_value;
 602
 603         rc = kstrtoul(val, 0, &new_value);
 604         if (rc) {
 605                 CERROR("Invalid value for 'lnet_response_tracking'\n");
 606                 return -EINVAL;
 607         }
 608
 609         if (new_value < 0 || new_value > 3) {
 610                 CWARN("Invalid value (%lu) for 'lnet_response_tracking'\n",
 611                       new_value);
 612                 return -EINVAL;
 613         }
 614
 615         lnet_response_tracking = new_value;
 616
 617         return 0;
 618 }
 619
 620 static const char *
 621 lnet_get_routes(void)
 622 {
 623         return routes;
 624 }
 625
 626 static const char *
 627 lnet_get_networks(void)
 628 {
 629         const char *nets;
 630         int rc;
 631
 632         if (*networks != 0 && *ip2nets != 0) {
 633                 LCONSOLE_ERROR_MSG(0x101, "Please specify EITHER 'networks' or "
 634                                    "'ip2nets' but not both at once\n");
 635                 return NULL;
 636         }
 637
 638         if (*ip2nets != 0) {
 639                 rc = lnet_parse_ip2nets(&nets, ip2nets);
 640                 return (rc == 0) ? nets : NULL;
 641         }
 642
 643         if (*networks != 0)
 644                 return networks;
 645
 646         return "tcp";
 647 }
 648
 649 static void
 650 lnet_init_locks(void)
 651 {
 652         spin_lock_init(&the_lnet.ln_eq_wait_lock);
 653         spin_lock_init(&the_lnet.ln_msg_resend_lock);
 654         init_completion(&the_lnet.ln_mt_wait_complete);
 655         mutex_init(&the_lnet.ln_lnd_mutex);
 656 }
 657
 658 struct kmem_cache *lnet_mes_cachep;        /* MEs kmem_cache */
 659 struct kmem_cache *lnet_small_mds_cachep;  /* <= LNET_SMALL_MD_SIZE bytes
 660                                             *  MDs kmem_cache */
 661 struct kmem_cache *lnet_udsp_cachep;       /* udsp cache */
 662 struct kmem_cache *lnet_rspt_cachep;       /* response tracker cache */
 663 struct kmem_cache *lnet_msg_cachep;
 664
 665 static int
 666 lnet_slab_setup(void)
 667 {
 668         /* create specific kmem_cache for MEs and small MDs (i.e., originally
 669          * allocated in <size-xxx> kmem_cache).
 670          */
 671         lnet_mes_cachep = kmem_cache_create("lnet_MEs", sizeof(struct lnet_me),
 672                                             0, 0, NULL);
 673         if (!lnet_mes_cachep)
 674                 return -ENOMEM;
 675
 676         lnet_small_mds_cachep = kmem_cache_create("lnet_small_MDs",
 677                                                   LNET_SMALL_MD_SIZE, 0, 0,
 678                                                   NULL);
 679         if (!lnet_small_mds_cachep)
 680                 return -ENOMEM;
 681
 682         lnet_udsp_cachep = kmem_cache_create("lnet_udsp",
 683                                              sizeof(struct lnet_udsp),
 684                                              0, 0, NULL);
 685         if (!lnet_udsp_cachep)
 686                 return -ENOMEM;
 687
 688         lnet_rspt_cachep = kmem_cache_create("lnet_rspt", sizeof(struct lnet_rsp_tracker),
 689                                             0, 0, NULL);
 690         if (!lnet_rspt_cachep)
 691                 return -ENOMEM;
 692
 693         lnet_msg_cachep = kmem_cache_create("lnet_msg", sizeof(struct lnet_msg),
 694                                             0, 0, NULL);
 695         if (!lnet_msg_cachep)
 696                 return -ENOMEM;
 697
 698         return 0;
 699 }
 700
 701 static void
 702 lnet_slab_cleanup(void)
 703 {
 704         if (lnet_msg_cachep) {
 705                 kmem_cache_destroy(lnet_msg_cachep);
 706                 lnet_msg_cachep = NULL;
 707         }
 708
 709         if (lnet_rspt_cachep) {
 710                 kmem_cache_destroy(lnet_rspt_cachep);
 711                 lnet_rspt_cachep = NULL;
 712         }
 713
 714         if (lnet_udsp_cachep) {
 715                 kmem_cache_destroy(lnet_udsp_cachep);
 716                 lnet_udsp_cachep = NULL;
 717         }
 718
 719         if (lnet_small_mds_cachep) {
 720                 kmem_cache_destroy(lnet_small_mds_cachep);
 721                 lnet_small_mds_cachep = NULL;
 722         }
 723
 724         if (lnet_mes_cachep) {
 725                 kmem_cache_destroy(lnet_mes_cachep);
 726                 lnet_mes_cachep = NULL;
 727         }
 728 }
 729
 730 static int
 731 lnet_create_remote_nets_table(void)
 732 {
 733         int               i;
 734         struct list_head *hash;
 735
 736         LASSERT(the_lnet.ln_remote_nets_hash == NULL);
 737         LASSERT(the_lnet.ln_remote_nets_hbits > 0);
 738         CFS_ALLOC_PTR_ARRAY(hash, LNET_REMOTE_NETS_HASH_SIZE);
 739         if (hash == NULL) {
 740                 CERROR("Failed to create remote nets hash table\n");
 741                 return -ENOMEM;
 742         }
 743
 744         for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++)
 745                 INIT_LIST_HEAD(&hash[i]);
 746         the_lnet.ln_remote_nets_hash = hash;
 747         return 0;
 748 }
 749
 750 static void
 751 lnet_destroy_remote_nets_table(void)
 752 {
 753         int i;
 754
 755         if (the_lnet.ln_remote_nets_hash == NULL)
 756                 return;
 757
 758         for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++)
 759                 LASSERT(list_empty(&the_lnet.ln_remote_nets_hash[i]));
 760
 761         CFS_FREE_PTR_ARRAY(the_lnet.ln_remote_nets_hash,
 762                            LNET_REMOTE_NETS_HASH_SIZE);
 763         the_lnet.ln_remote_nets_hash = NULL;
 764 }
 765
 766 static void
 767 lnet_destroy_locks(void)
 768 {
 769         if (the_lnet.ln_res_lock != NULL) {
 770                 cfs_percpt_lock_free(the_lnet.ln_res_lock);
 771                 the_lnet.ln_res_lock = NULL;
 772         }
 773
 774         if (the_lnet.ln_net_lock != NULL) {
 775                 cfs_percpt_lock_free(the_lnet.ln_net_lock);
 776                 the_lnet.ln_net_lock = NULL;
 777         }
 778 }
 779
 780 static int
 781 lnet_create_locks(void)
 782 {
 783         lnet_init_locks();
 784
 785         the_lnet.ln_res_lock = cfs_percpt_lock_alloc(lnet_cpt_table());
 786         if (the_lnet.ln_res_lock == NULL)
 787                 goto failed;
 788
 789         the_lnet.ln_net_lock = cfs_percpt_lock_alloc(lnet_cpt_table());
 790         if (the_lnet.ln_net_lock == NULL)
 791                 goto failed;
 792
 793         return 0;
 794
 795  failed:
 796         lnet_destroy_locks();
 797         return -ENOMEM;
 798 }
 799
 800 static void lnet_assert_wire_constants(void)
 801 {
 802         /* Wire protocol assertions generated by 'wirecheck'
 803          * running on Linux robert.bartonsoftware.com 2.6.8-1.521
 804          * #1 Mon Aug 16 09:01:18 EDT 2004 i686 athlon i386 GNU/Linux
 805          * with gcc version 3.3.3 20040412 (Red Hat Linux 3.3.3-7)
 806          */
 807
 808         /* Constants... */
 809         BUILD_BUG_ON(LNET_PROTO_TCP_MAGIC != 0xeebc0ded);
 810         BUILD_BUG_ON(LNET_PROTO_TCP_VERSION_MAJOR != 1);
 811         BUILD_BUG_ON(LNET_PROTO_TCP_VERSION_MINOR != 0);
 812         BUILD_BUG_ON(LNET_MSG_ACK != 0);
 813         BUILD_BUG_ON(LNET_MSG_PUT != 1);
 814         BUILD_BUG_ON(LNET_MSG_GET != 2);
 815         BUILD_BUG_ON(LNET_MSG_REPLY != 3);
 816         BUILD_BUG_ON(LNET_MSG_HELLO != 4);
 817
 818         BUILD_BUG_ON((int)sizeof(lnet_nid_t) != 8);
 819         BUILD_BUG_ON((int)sizeof(lnet_pid_t) != 4);
 820
 821         /* Checks for struct lnet_nid */
 822         BUILD_BUG_ON((int)sizeof(struct lnet_nid) != 20);
 823         BUILD_BUG_ON((int)offsetof(struct lnet_nid, nid_size) != 0);
 824         BUILD_BUG_ON((int)sizeof(((struct lnet_nid *)0)->nid_size) != 1);
 825         BUILD_BUG_ON((int)offsetof(struct lnet_nid, nid_type) != 1);
 826         BUILD_BUG_ON((int)sizeof(((struct lnet_nid *)0)->nid_type) != 1);
 827         BUILD_BUG_ON((int)offsetof(struct lnet_nid, nid_num) != 2);
 828         BUILD_BUG_ON((int)sizeof(((struct lnet_nid *)0)->nid_num) != 2);
 829         BUILD_BUG_ON((int)offsetof(struct lnet_nid, nid_addr) != 4);
 830         BUILD_BUG_ON((int)sizeof(((struct lnet_nid *)0)->nid_addr) != 16);
 831
 832         /* Checks for struct lnet_process_id_packed */
 833         BUILD_BUG_ON((int)sizeof(struct lnet_process_id_packed) != 12);
 834         BUILD_BUG_ON((int)offsetof(struct lnet_process_id_packed, nid) != 0);
 835         BUILD_BUG_ON((int)sizeof(((struct lnet_process_id_packed *)0)->nid) != 8);
 836         BUILD_BUG_ON((int)offsetof(struct lnet_process_id_packed, pid) != 8);
 837         BUILD_BUG_ON((int)sizeof(((struct lnet_process_id_packed *)0)->pid) != 4);
 838
 839         /* Checks for struct lnet_handle_wire */
 840         BUILD_BUG_ON((int)sizeof(struct lnet_handle_wire) != 16);
 841         BUILD_BUG_ON((int)offsetof(struct lnet_handle_wire,
 842                                    wh_interface_cookie) != 0);
 843         BUILD_BUG_ON((int)sizeof(((struct lnet_handle_wire *)0)->wh_interface_cookie) != 8);
 844         BUILD_BUG_ON((int)offsetof(struct lnet_handle_wire,
 845                                    wh_object_cookie) != 8);
 846         BUILD_BUG_ON((int)sizeof(((struct lnet_handle_wire *)0)->wh_object_cookie) != 8);
 847
 848         /* Checks for struct struct lnet_magicversion */
 849         BUILD_BUG_ON((int)sizeof(struct lnet_magicversion) != 8);
 850         BUILD_BUG_ON((int)offsetof(struct lnet_magicversion, magic) != 0);
 851         BUILD_BUG_ON((int)sizeof(((struct lnet_magicversion *)0)->magic) != 4);
 852         BUILD_BUG_ON((int)offsetof(struct lnet_magicversion, version_major) != 4);
 853         BUILD_BUG_ON((int)sizeof(((struct lnet_magicversion *)0)->version_major) != 2);
 854         BUILD_BUG_ON((int)offsetof(struct lnet_magicversion,
 855                                    version_minor) != 6);
 856         BUILD_BUG_ON((int)sizeof(((struct lnet_magicversion *)0)->version_minor) != 2);
 857
 858         /* Checks for struct _lnet_hdr_nid4 */
 859         BUILD_BUG_ON((int)sizeof(struct _lnet_hdr_nid4) != 72);
 860         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, dest_nid) != 0);
 861         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->dest_nid) != 8);
 862         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, src_nid) != 8);
 863         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->src_nid) != 8);
 864         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, dest_pid) != 16);
 865         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->dest_pid) != 4);
 866         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, src_pid) != 20);
 867         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->src_pid) != 4);
 868         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, type) != 24);
 869         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->type) != 4);
 870         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, payload_length) != 28);
 871         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->payload_length) != 4);
 872         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg) != 32);
 873         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg) != 40);
 874
 875         /* Ack */
 876         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.ack.dst_wmd) != 32);
 877         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.ack.dst_wmd) != 16);
 878         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.ack.match_bits) != 48);
 879         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.ack.match_bits) != 8);
 880         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.ack.mlength) != 56);
 881         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.ack.mlength) != 4);
 882
 883         /* Put */
 884         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.put.ack_wmd) != 32);
 885         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.put.ack_wmd) != 16);
 886         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.put.match_bits) != 48);
 887         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.put.match_bits) != 8);
 888         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.put.hdr_data) != 56);
 889         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.put.hdr_data) != 8);
 890         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.put.ptl_index) != 64);
 891         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.put.ptl_index) != 4);
 892         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.put.offset) != 68);
 893         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.put.offset) != 4);
 894
 895         /* Get */
 896         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.get.return_wmd) != 32);
 897         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.get.return_wmd) != 16);
 898         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.get.match_bits) != 48);
 899         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.get.match_bits) != 8);
 900         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.get.ptl_index) != 56);
 901         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.get.ptl_index) != 4);
 902         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.get.src_offset) != 60);
 903         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.get.src_offset) != 4);
 904         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.get.sink_length) != 64);
 905         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.get.sink_length) != 4);
 906
 907         /* Reply */
 908         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.reply.dst_wmd) != 32);
 909         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.reply.dst_wmd) != 16);
 910
 911         /* Hello */
 912         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.hello.incarnation) != 32);
 913         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.hello.incarnation) != 8);
 914         BUILD_BUG_ON((int)offsetof(struct _lnet_hdr_nid4, msg.hello.type) != 40);
 915         BUILD_BUG_ON((int)sizeof(((struct _lnet_hdr_nid4 *)0)->msg.hello.type) != 4);
 916
 917         /* Checks for struct lnet_ni_status and related constants */
 918         BUILD_BUG_ON(LNET_NI_STATUS_INVALID != 0x00000000);
 919         BUILD_BUG_ON(LNET_NI_STATUS_UP != 0x15aac0de);
 920         BUILD_BUG_ON(LNET_NI_STATUS_DOWN != 0xdeadface);
 921
 922         /* Checks for struct lnet_ni_status */
 923         BUILD_BUG_ON((int)sizeof(struct lnet_ni_status) != 16);
 924         BUILD_BUG_ON((int)offsetof(struct lnet_ni_status, ns_nid) != 0);
 925         BUILD_BUG_ON((int)sizeof(((struct lnet_ni_status *)0)->ns_nid) != 8);
 926         BUILD_BUG_ON((int)offsetof(struct lnet_ni_status, ns_status) != 8);
 927         BUILD_BUG_ON((int)sizeof(((struct lnet_ni_status *)0)->ns_status) != 4);
 928         BUILD_BUG_ON((int)offsetof(struct lnet_ni_status, ns_msg_size) != 12);
 929         BUILD_BUG_ON((int)sizeof(((struct lnet_ni_status *)0)->ns_msg_size) != 4);
 930
 931         /* Checks for struct lnet_ni_large_status */
 932         BUILD_BUG_ON((int)sizeof(struct lnet_ni_large_status) != 24);
 933         BUILD_BUG_ON((int)offsetof(struct lnet_ni_large_status, ns_status) != 0);
 934         BUILD_BUG_ON((int)sizeof(((struct lnet_ni_large_status *)0)->ns_status) != 4);
 935         BUILD_BUG_ON((int)offsetof(struct lnet_ni_large_status, ns_nid) != 4);
 936         BUILD_BUG_ON((int)sizeof(((struct lnet_ni_large_status *)0)->ns_nid) != 20);
 937
 938         /* Checks for struct lnet_ping_info and related constants */
 939         BUILD_BUG_ON(LNET_PROTO_PING_MAGIC != 0x70696E67);
 940         BUILD_BUG_ON(LNET_PING_FEAT_INVAL != 0);
 941         BUILD_BUG_ON(LNET_PING_FEAT_BASE != 1);
 942         BUILD_BUG_ON(LNET_PING_FEAT_NI_STATUS != 2);
 943         BUILD_BUG_ON(LNET_PING_FEAT_RTE_DISABLED != 4);
 944         BUILD_BUG_ON(LNET_PING_FEAT_MULTI_RAIL != 8);
 945         BUILD_BUG_ON(LNET_PING_FEAT_DISCOVERY != 16);
 946         BUILD_BUG_ON(LNET_PING_FEAT_LARGE_ADDR != 32);
 947         BUILD_BUG_ON(LNET_PING_FEAT_PRIMARY_LARGE != 64);
 948         BUILD_BUG_ON(LNET_PING_FEAT_BITS != 127);
 949
 950         /* Checks for struct lnet_ping_info */
 951         BUILD_BUG_ON((int)sizeof(struct lnet_ping_info) != 16);
 952         BUILD_BUG_ON((int)offsetof(struct lnet_ping_info, pi_magic) != 0);
 953         BUILD_BUG_ON((int)sizeof(((struct lnet_ping_info *)0)->pi_magic) != 4);
 954         BUILD_BUG_ON((int)offsetof(struct lnet_ping_info, pi_features) != 4);
 955         BUILD_BUG_ON((int)sizeof(((struct lnet_ping_info *)0)->pi_features) != 4);
 956         BUILD_BUG_ON((int)offsetof(struct lnet_ping_info, pi_pid) != 8);
 957         BUILD_BUG_ON((int)sizeof(((struct lnet_ping_info *)0)->pi_pid) != 4);
 958         BUILD_BUG_ON((int)offsetof(struct lnet_ping_info, pi_nnis) != 12);
 959         BUILD_BUG_ON((int)sizeof(((struct lnet_ping_info *)0)->pi_nnis) != 4);
 960         BUILD_BUG_ON((int)offsetof(struct lnet_ping_info, pi_ni) != 16);
 961         BUILD_BUG_ON(offsetof(struct lnet_ping_info, pi_ni) != sizeof(struct lnet_ping_info));
 962
 963         /* Acceptor connection request */
 964         BUILD_BUG_ON(LNET_PROTO_ACCEPTOR_VERSION != 1);
 965
 966         /* Checks for struct lnet_acceptor_connreq */
 967         BUILD_BUG_ON((int)sizeof(struct lnet_acceptor_connreq) != 16);
 968         BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq, acr_magic) != 0);
 969         BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq *)0)->acr_magic) != 4);
 970         BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq, acr_version) != 4);
 971         BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq *)0)->acr_version) != 4);
 972         BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq, acr_nid) != 8);
 973         BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq *)0)->acr_nid) != 8);
 974
 975         /* Checks for struct lnet_acceptor_connreq_v2 */
 976         BUILD_BUG_ON((int)sizeof(struct lnet_acceptor_connreq_v2) != 28);
 977         BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_magic) != 0);
 978         BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_magic) != 4);
 979         BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_version) != 4);
 980         BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_version) != 4);
 981         BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_nid) != 8);
 982         BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_nid) != 20);
 983
 984         /* Checks for struct lnet_counters_common */
 985         BUILD_BUG_ON((int)sizeof(struct lnet_counters_common) != 60);
 986         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_msgs_alloc) != 0);
 987         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_msgs_alloc) != 4);
 988         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_msgs_max) != 4);
 989         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_msgs_max) != 4);
 990         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_errors) != 8);
 991         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_errors) != 4);
 992         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_send_count) != 12);
 993         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_send_count) != 4);
 994         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_recv_count) != 16);
 995         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_recv_count) != 4);
 996         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_route_count) != 20);
 997         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_route_count) != 4);
 998         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_drop_count) != 24);
 999         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_drop_count) != 4);
1000         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_send_length) != 28);
1001         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_send_length) != 8);
1002         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_recv_length) != 36);
1003         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_recv_length) != 8);
1004         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_route_length) != 44);
1005         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_route_length) != 8);
1006         BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_drop_length) != 52);
1007         BUILD_BUG_ON((int)sizeof(((struct lnet_counters_common *)0)->lcc_drop_length) != 8);
1008 }
1009
1010 static const struct lnet_lnd *lnet_find_lnd_by_type(__u32 type)
1011 {
1012         const struct lnet_lnd *lnd;
1013
1014         /* holding lnd mutex */
1015         if (type >= NUM_LNDS)
1016                 return NULL;
1017         lnd = the_lnet.ln_lnds[type];
1018         LASSERT(!lnd || lnd->lnd_type == type);
1019
1020         return lnd;
1021 }
1022
1023 unsigned int
1024 lnet_get_lnd_timeout(void)
1025 {
1026         return lnet_lnd_timeout;
1027 }
1028 EXPORT_SYMBOL(lnet_get_lnd_timeout);
1029
1030 void
1031 lnet_register_lnd(const struct lnet_lnd *lnd)
1032 {
1033         mutex_lock(&the_lnet.ln_lnd_mutex);
1034
1035         LASSERT(libcfs_isknown_lnd(lnd->lnd_type));
1036         LASSERT(lnet_find_lnd_by_type(lnd->lnd_type) == NULL);
1037
1038         the_lnet.ln_lnds[lnd->lnd_type] = lnd;
1039
1040         CDEBUG(D_NET, "%s LND registered\n", libcfs_lnd2str(lnd->lnd_type));
1041
1042         mutex_unlock(&the_lnet.ln_lnd_mutex);
1043 }
1044 EXPORT_SYMBOL(lnet_register_lnd);
1045
1046 void
1047 lnet_unregister_lnd(const struct lnet_lnd *lnd)
1048 {
1049         mutex_lock(&the_lnet.ln_lnd_mutex);
1050
1051         LASSERT(lnet_find_lnd_by_type(lnd->lnd_type) == lnd);
1052
1053         the_lnet.ln_lnds[lnd->lnd_type] = NULL;
1054         CDEBUG(D_NET, "%s LND unregistered\n", libcfs_lnd2str(lnd->lnd_type));
1055
1056         mutex_unlock(&the_lnet.ln_lnd_mutex);
1057 }
1058 EXPORT_SYMBOL(lnet_unregister_lnd);
1059
1060 static void
1061 lnet_counters_get_common_locked(struct lnet_counters_common *common)
1062 {
1063         struct lnet_counters *ctr;
1064         int i;
1065
1066         /* FIXME !!! Their is no assert_lnet_net_locked() to ensure this
1067          * actually called under the protection of the lnet_net_lock.
1068          */
1069         memset(common, 0, sizeof(*common));
1070
1071         cfs_percpt_for_each(ctr, i, the_lnet.ln_counters) {
1072                 common->lcc_msgs_max     += ctr->lct_common.lcc_msgs_max;
1073                 common->lcc_msgs_alloc   += ctr->lct_common.lcc_msgs_alloc;
1074                 common->lcc_errors       += ctr->lct_common.lcc_errors;
1075                 common->lcc_send_count   += ctr->lct_common.lcc_send_count;
1076                 common->lcc_recv_count   += ctr->lct_common.lcc_recv_count;
1077                 common->lcc_route_count  += ctr->lct_common.lcc_route_count;
1078                 common->lcc_drop_count   += ctr->lct_common.lcc_drop_count;
1079                 common->lcc_send_length  += ctr->lct_common.lcc_send_length;
1080                 common->lcc_recv_length  += ctr->lct_common.lcc_recv_length;
1081                 common->lcc_route_length += ctr->lct_common.lcc_route_length;
1082                 common->lcc_drop_length  += ctr->lct_common.lcc_drop_length;
1083         }
1084 }
1085
1086 void
1087 lnet_counters_get_common(struct lnet_counters_common *common)
1088 {
1089         lnet_net_lock(LNET_LOCK_EX);
1090         lnet_counters_get_common_locked(common);
1091         lnet_net_unlock(LNET_LOCK_EX);
1092 }
1093 EXPORT_SYMBOL(lnet_counters_get_common);
1094
1095 int
1096 lnet_counters_get(struct lnet_counters *counters)
1097 {
1098         struct lnet_counters *ctr;
1099         struct lnet_counters_health *health = &counters->lct_health;
1100         int i, rc = 0;
1101
1102         memset(counters, 0, sizeof(*counters));
1103
1104         lnet_net_lock(LNET_LOCK_EX);
1105
1106         if (the_lnet.ln_state != LNET_STATE_RUNNING)
1107                 GOTO(out_unlock, rc = -ENODEV);
1108
1109         lnet_counters_get_common_locked(&counters->lct_common);
1110
1111         cfs_percpt_for_each(ctr, i, the_lnet.ln_counters) {
1112                 health->lch_rst_alloc    += ctr->lct_health.lch_rst_alloc;
1113                 health->lch_resend_count += ctr->lct_health.lch_resend_count;
1114                 health->lch_response_timeout_count +=
1115                                 ctr->lct_health.lch_response_timeout_count;
1116                 health->lch_local_interrupt_count +=
1117                                 ctr->lct_health.lch_local_interrupt_count;
1118                 health->lch_local_dropped_count +=
1119                                 ctr->lct_health.lch_local_dropped_count;
1120                 health->lch_local_aborted_count +=
1121                                 ctr->lct_health.lch_local_aborted_count;
1122                 health->lch_local_no_route_count +=
1123                                 ctr->lct_health.lch_local_no_route_count;
1124                 health->lch_local_timeout_count +=
1125                                 ctr->lct_health.lch_local_timeout_count;
1126                 health->lch_local_error_count +=
1127                                 ctr->lct_health.lch_local_error_count;
1128                 health->lch_remote_dropped_count +=
1129                                 ctr->lct_health.lch_remote_dropped_count;
1130                 health->lch_remote_error_count +=
1131                                 ctr->lct_health.lch_remote_error_count;
1132                 health->lch_remote_timeout_count +=
1133                                 ctr->lct_health.lch_remote_timeout_count;
1134                 health->lch_network_timeout_count +=
1135                                 ctr->lct_health.lch_network_timeout_count;
1136         }
1137 out_unlock:
1138         lnet_net_unlock(LNET_LOCK_EX);
1139         return rc;
1140 }
1141 EXPORT_SYMBOL(lnet_counters_get);
1142
1143 void
1144 lnet_counters_reset(void)
1145 {
1146         struct lnet_counters *counters;
1147         int             i;
1148
1149         lnet_net_lock(LNET_LOCK_EX);
1150
1151         if (the_lnet.ln_state != LNET_STATE_RUNNING)
1152                 goto avoid_reset;
1153
1154         cfs_percpt_for_each(counters, i, the_lnet.ln_counters)
1155                 memset(counters, 0, sizeof(struct lnet_counters));
1156 avoid_reset:
1157         lnet_net_unlock(LNET_LOCK_EX);
1158 }
1159
1160 static char *
1161 lnet_res_type2str(int type)
1162 {
1163         switch (type) {
1164         default:
1165                 LBUG();
1166         case LNET_COOKIE_TYPE_MD:
1167                 return "MD";
1168         case LNET_COOKIE_TYPE_ME:
1169                 return "ME";
1170         case LNET_COOKIE_TYPE_EQ:
1171                 return "EQ";
1172         }
1173 }
1174
1175 static void
1176 lnet_res_container_cleanup(struct lnet_res_container *rec)
1177 {
1178         int     count = 0;
1179
1180         if (rec->rec_type == 0) /* not set yet, it's uninitialized */
1181                 return;
1182
1183         while (!list_empty(&rec->rec_active)) {
1184                 struct list_head *e = rec->rec_active.next;
1185
1186                 list_del_init(e);
1187                 if (rec->rec_type == LNET_COOKIE_TYPE_MD) {
1188                         lnet_md_free(list_entry(e, struct lnet_libmd, md_list));
1189
1190                 } else { /* NB: Active MEs should be attached on portals */
1191                         LBUG();
1192                 }
1193                 count++;
1194         }
1195
1196         if (count > 0) {
1197                 /* Found alive MD/ME/EQ, user really should unlink/free
1198                  * all of them before finalize LNet, but if someone didn't,
1199                  * we have to recycle garbage for him */
1200                 CERROR("%d active elements on exit of %s container\n",
1201                        count, lnet_res_type2str(rec->rec_type));
1202         }
1203
1204         if (rec->rec_lh_hash != NULL) {
1205                 CFS_FREE_PTR_ARRAY(rec->rec_lh_hash, LNET_LH_HASH_SIZE);
1206                 rec->rec_lh_hash = NULL;
1207         }
1208
1209         rec->rec_type = 0; /* mark it as finalized */
1210 }
1211
1212 static int
1213 lnet_res_container_setup(struct lnet_res_container *rec, int cpt, int type)
1214 {
1215         int     rc = 0;
1216         int     i;
1217
1218         LASSERT(rec->rec_type == 0);
1219
1220         rec->rec_type = type;
1221         INIT_LIST_HEAD(&rec->rec_active);
1222
1223         rec->rec_lh_cookie = (cpt << LNET_COOKIE_TYPE_BITS) | type;
1224
1225         /* Arbitrary choice of hash table size */
1226         LIBCFS_CPT_ALLOC(rec->rec_lh_hash, lnet_cpt_table(), cpt,
1227                          LNET_LH_HASH_SIZE * sizeof(rec->rec_lh_hash[0]));
1228         if (rec->rec_lh_hash == NULL) {
1229                 rc = -ENOMEM;
1230                 goto out;
1231         }
1232
1233         for (i = 0; i < LNET_LH_HASH_SIZE; i++)
1234                 INIT_LIST_HEAD(&rec->rec_lh_hash[i]);
1235
1236         return 0;
1237
1238 out:
1239         CERROR("Failed to setup %s resource container\n",
1240                lnet_res_type2str(type));
1241         lnet_res_container_cleanup(rec);
1242         return rc;
1243 }
1244
1245 static void
1246 lnet_res_containers_destroy(struct lnet_res_container **recs)
1247 {
1248         struct lnet_res_container       *rec;
1249         int                             i;
1250
1251         cfs_percpt_for_each(rec, i, recs)
1252                 lnet_res_container_cleanup(rec);
1253
1254         cfs_percpt_free(recs);
1255 }
1256
1257 static struct lnet_res_container **
1258 lnet_res_containers_create(int type)
1259 {
1260         struct lnet_res_container       **recs;
1261         struct lnet_res_container       *rec;
1262         int                             rc;
1263         int                             i;
1264
1265         recs = cfs_percpt_alloc(lnet_cpt_table(), sizeof(*rec));
1266         if (recs == NULL) {
1267                 CERROR("Failed to allocate %s resource containers\n",
1268                        lnet_res_type2str(type));
1269                 return NULL;
1270         }
1271
1272         cfs_percpt_for_each(rec, i, recs) {
1273                 rc = lnet_res_container_setup(rec, i, type);
1274                 if (rc != 0) {
1275                         lnet_res_containers_destroy(recs);
1276                         return NULL;
1277                 }
1278         }
1279
1280         return recs;
1281 }
1282
1283 struct lnet_libhandle *
1284 lnet_res_lh_lookup(struct lnet_res_container *rec, __u64 cookie)
1285 {
1286         /* ALWAYS called with lnet_res_lock held */
1287         struct list_head        *head;
1288         struct lnet_libhandle   *lh;
1289         unsigned int            hash;
1290
1291         if ((cookie & LNET_COOKIE_MASK) != rec->rec_type)
1292                 return NULL;
1293
1294         hash = cookie >> (LNET_COOKIE_TYPE_BITS + LNET_CPT_BITS);
1295         head = &rec->rec_lh_hash[hash & LNET_LH_HASH_MASK];
1296
1297         list_for_each_entry(lh, head, lh_hash_chain) {
1298                 if (lh->lh_cookie == cookie)
1299                         return lh;
1300         }
1301
1302         return NULL;
1303 }
1304
1305 void
1306 lnet_res_lh_initialize(struct lnet_res_container *rec,
1307                        struct lnet_libhandle *lh)
1308 {
1309         /* ALWAYS called with lnet_res_lock held */
1310         unsigned int    ibits = LNET_COOKIE_TYPE_BITS + LNET_CPT_BITS;
1311         unsigned int    hash;
1312
1313         lh->lh_cookie = rec->rec_lh_cookie;
1314         rec->rec_lh_cookie += 1 << ibits;
1315
1316         hash = (lh->lh_cookie >> ibits) & LNET_LH_HASH_MASK;
1317
1318         list_add(&lh->lh_hash_chain, &rec->rec_lh_hash[hash]);
1319 }
1320
1321 struct list_head **
1322 lnet_create_array_of_queues(void)
1323 {
1324         struct list_head **qs;
1325         struct list_head *q;
1326         int i;
1327
1328         qs = cfs_percpt_alloc(lnet_cpt_table(),
1329                               sizeof(struct list_head));
1330         if (!qs) {
1331                 CERROR("Failed to allocate queues\n");
1332                 return NULL;
1333         }
1334
1335         cfs_percpt_for_each(q, i, qs)
1336                 INIT_LIST_HEAD(q);
1337
1338         return qs;
1339 }
1340
1341 static int lnet_unprepare(void);
1342
1343 static int
1344 lnet_prepare(lnet_pid_t requested_pid)
1345 {
1346         /* Prepare to bring up the network */
1347         struct lnet_res_container **recs;
1348         int                       rc = 0;
1349
1350         if (requested_pid == LNET_PID_ANY) {
1351                 /* Don't instantiate LNET just for me */
1352                 return -ENETDOWN;
1353         }
1354
1355         LASSERT(the_lnet.ln_refcount == 0);
1356
1357         the_lnet.ln_routing = 0;
1358
1359         LASSERT((requested_pid & LNET_PID_USERFLAG) == 0);
1360         the_lnet.ln_pid = requested_pid;
1361
1362         INIT_LIST_HEAD(&the_lnet.ln_test_peers);
1363         INIT_LIST_HEAD(&the_lnet.ln_remote_peer_ni_list);
1364         INIT_LIST_HEAD(&the_lnet.ln_nets);
1365         INIT_LIST_HEAD(&the_lnet.ln_routers);
1366         INIT_LIST_HEAD(&the_lnet.ln_drop_rules);
1367         INIT_LIST_HEAD(&the_lnet.ln_delay_rules);
1368         INIT_LIST_HEAD(&the_lnet.ln_dc_request);
1369         INIT_LIST_HEAD(&the_lnet.ln_dc_working);
1370         INIT_LIST_HEAD(&the_lnet.ln_dc_expired);
1371         INIT_LIST_HEAD(&the_lnet.ln_mt_localNIRecovq);
1372         INIT_LIST_HEAD(&the_lnet.ln_mt_peerNIRecovq);
1373         INIT_LIST_HEAD(&the_lnet.ln_udsp_list);
1374         init_waitqueue_head(&the_lnet.ln_dc_waitq);
1375         the_lnet.ln_mt_handler = NULL;
1376         init_completion(&the_lnet.ln_started);
1377         atomic_set(&the_lnet.ln_late_msg_count, 0);
1378         atomic64_set(&the_lnet.ln_late_msg_nsecs, 0);
1379
1380         rc = lnet_slab_setup();
1381         if (rc != 0)
1382                 goto failed;
1383
1384         rc = lnet_create_remote_nets_table();
1385         if (rc != 0)
1386                 goto failed;
1387
1388         /*
1389          * NB the interface cookie in wire handles guards against delayed
1390          * replies and ACKs appearing valid after reboot.
1391          */
1392         the_lnet.ln_interface_cookie = ktime_get_real_ns();
1393
1394         the_lnet.ln_counters = cfs_percpt_alloc(lnet_cpt_table(),
1395                                                 sizeof(struct lnet_counters));
1396         if (the_lnet.ln_counters == NULL) {
1397                 CERROR("Failed to allocate counters for LNet\n");
1398                 rc = -ENOMEM;
1399                 goto failed;
1400         }
1401
1402         rc = lnet_peer_tables_create();
1403         if (rc != 0)
1404                 goto failed;
1405
1406         rc = lnet_msg_containers_create();
1407         if (rc != 0)
1408                 goto failed;
1409
1410         rc = lnet_res_container_setup(&the_lnet.ln_eq_container, 0,
1411                                       LNET_COOKIE_TYPE_EQ);
1412         if (rc != 0)
1413                 goto failed;
1414
1415         recs = lnet_res_containers_create(LNET_COOKIE_TYPE_MD);
1416         if (recs == NULL) {
1417                 rc = -ENOMEM;
1418                 goto failed;
1419         }
1420
1421         the_lnet.ln_md_containers = recs;
1422
1423         rc = lnet_portals_create();
1424         if (rc != 0) {
1425                 CERROR("Failed to create portals for LNet: %d\n", rc);
1426                 goto failed;
1427         }
1428
1429         the_lnet.ln_mt_zombie_rstqs = lnet_create_array_of_queues();
1430         if (!the_lnet.ln_mt_zombie_rstqs) {
1431                 rc = -ENOMEM;
1432                 goto failed;
1433         }
1434
1435         return 0;
1436
1437  failed:
1438         lnet_unprepare();
1439         return rc;
1440 }
1441
1442 static int
1443 lnet_unprepare(void)
1444 {
1445         /* NB no LNET_LOCK since this is the last reference.  All LND instances
1446          * have shut down already, so it is safe to unlink and free all
1447          * descriptors, even those that appear committed to a network op (eg MD
1448          * with non-zero pending count) */
1449
1450         lnet_fail_nid(LNET_NID_ANY, 0);
1451
1452         LASSERT(the_lnet.ln_refcount == 0);
1453         LASSERT(list_empty(&the_lnet.ln_test_peers));
1454         LASSERT(list_empty(&the_lnet.ln_nets));
1455
1456         if (the_lnet.ln_mt_zombie_rstqs) {
1457                 lnet_clean_zombie_rstqs();
1458                 the_lnet.ln_mt_zombie_rstqs = NULL;
1459         }
1460
1461         lnet_assert_handler_unused(the_lnet.ln_mt_handler, true);
1462         the_lnet.ln_mt_handler = NULL;
1463
1464         lnet_portals_destroy();
1465
1466         if (the_lnet.ln_md_containers != NULL) {
1467                 lnet_res_containers_destroy(the_lnet.ln_md_containers);
1468                 the_lnet.ln_md_containers = NULL;
1469         }
1470
1471         lnet_res_container_cleanup(&the_lnet.ln_eq_container);
1472
1473         lnet_msg_containers_destroy();
1474         lnet_peer_uninit();
1475         lnet_rtrpools_free(0);
1476
1477         if (the_lnet.ln_counters != NULL) {
1478                 cfs_percpt_free(the_lnet.ln_counters);
1479                 the_lnet.ln_counters = NULL;
1480         }
1481         lnet_destroy_remote_nets_table();
1482         lnet_udsp_destroy(true);
1483         lnet_slab_cleanup();
1484
1485         return 0;
1486 }
1487
1488 struct lnet_ni  *
1489 lnet_net2ni_locked(__u32 net_id, int cpt)
1490 {
1491         struct lnet_ni   *ni;
1492         struct lnet_net  *net;
1493
1494         LASSERT(cpt != LNET_LOCK_EX);
1495
1496         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
1497                 if (net->net_id == net_id) {
1498                         ni = list_first_entry(&net->net_ni_list, struct lnet_ni,
1499                                               ni_netlist);
1500                         return ni;
1501                 }
1502         }
1503
1504         return NULL;
1505 }
1506
1507 struct lnet_ni *
1508 lnet_net2ni_addref(__u32 net)
1509 {
1510         struct lnet_ni *ni;
1511
1512         lnet_net_lock(0);
1513         ni = lnet_net2ni_locked(net, 0);
1514         if (ni)
1515                 lnet_ni_addref_locked(ni, 0);
1516         lnet_net_unlock(0);
1517
1518         return ni;
1519 }
1520 EXPORT_SYMBOL(lnet_net2ni_addref);
1521
1522 struct lnet_net *
1523 lnet_get_net_locked(__u32 net_id)
1524 {
1525         struct lnet_net  *net;
1526
1527         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
1528                 if (net->net_id == net_id)
1529                         return net;
1530         }
1531
1532         return NULL;
1533 }
1534
1535 void
1536 lnet_net_clr_pref_rtrs(struct lnet_net *net)
1537 {
1538         struct list_head zombies;
1539         struct lnet_nid_list *ne;
1540         struct lnet_nid_list *tmp;
1541
1542         INIT_LIST_HEAD(&zombies);
1543
1544         lnet_net_lock(LNET_LOCK_EX);
1545         list_splice_init(&net->net_rtr_pref_nids, &zombies);
1546         lnet_net_unlock(LNET_LOCK_EX);
1547
1548         list_for_each_entry_safe(ne, tmp, &zombies, nl_list) {
1549                 list_del_init(&ne->nl_list);
1550                 LIBCFS_FREE(ne, sizeof(*ne));
1551         }
1552 }
1553
1554 int
1555 lnet_net_add_pref_rtr(struct lnet_net *net,
1556                       struct lnet_nid *gw_nid)
1557 __must_hold(&the_lnet.ln_api_mutex)
1558 {
1559         struct lnet_nid_list *ne;
1560
1561         /* This function is called with api_mutex held. When the api_mutex
1562          * is held the list can not be modified, as it is only modified as
1563          * a result of applying a UDSP and that happens under api_mutex
1564          * lock.
1565          */
1566         list_for_each_entry(ne, &net->net_rtr_pref_nids, nl_list) {
1567                 if (nid_same(&ne->nl_nid, gw_nid))
1568                         return -EEXIST;
1569         }
1570
1571         LIBCFS_ALLOC(ne, sizeof(*ne));
1572         if (!ne)
1573                 return -ENOMEM;
1574
1575         ne->nl_nid = *gw_nid;
1576
1577         /* Lock the cpt to protect against addition and checks in the
1578          * selection algorithm
1579          */
1580         lnet_net_lock(LNET_LOCK_EX);
1581         list_add(&ne->nl_list, &net->net_rtr_pref_nids);
1582         lnet_net_unlock(LNET_LOCK_EX);
1583
1584         return 0;
1585 }
1586
1587 static unsigned int
1588 lnet_nid4_cpt_hash(lnet_nid_t nid, unsigned int number)
1589 {
1590         __u64 key = nid;
1591         __u16 lnd = LNET_NETTYP(LNET_NIDNET(nid));
1592         unsigned int cpt;
1593
1594         if (lnd == KFILND || lnd == GNILND) {
1595                 cpt = hash_long(key, LNET_CPT_BITS);
1596
1597                 /* NB: The number of CPTs needn't be a power of 2 */
1598                 if (cpt >= number)
1599                         cpt = (key + cpt + (cpt >> 1)) % number;
1600         } else {
1601                 __u64 pair_bits = 0x0001000100010001LLU;
1602                 __u64 mask = pair_bits * 0xFF;
1603                 __u64 pair_sum;
1604                 /* For ipv4 NIDs, use (sum-by-multiplication of nid bytes) mod
1605                  * (number of CPTs) to match nid to a CPT.
1606                  */
1607                 pair_sum = (key & mask) + ((key >> 8) & mask);
1608                 pair_sum = (pair_sum * pair_bits) >> 48;
1609                 cpt = (unsigned int)(pair_sum) % number;
1610         }
1611
1612         CDEBUG(D_NET, "Match nid %s to cpt %u\n",
1613                libcfs_nid2str(nid), cpt);
1614
1615         return cpt;
1616 }
1617
1618 unsigned int
1619 lnet_nid_cpt_hash(struct lnet_nid *nid, unsigned int number)
1620 {
1621         unsigned int val;
1622         u32 h = 0;
1623         int i;
1624
1625         LASSERT(number >= 1 && number <= LNET_CPT_NUMBER);
1626
1627         if (number == 1)
1628                 return 0;
1629
1630         if (nid_is_nid4(nid))
1631                 return lnet_nid4_cpt_hash(lnet_nid_to_nid4(nid), number);
1632
1633         for (i = 0; i < 4; i++)
1634                 h = cfs_hash_32(nid->nid_addr[i]^h, 32);
1635         val = cfs_hash_32(LNET_NID_NET(nid) ^ h, LNET_CPT_BITS);
1636         if (val < number)
1637                 return val;
1638         return (unsigned int)(h + val + (val >> 1)) % number;
1639 }
1640
1641 int
1642 lnet_cpt_of_nid_locked(struct lnet_nid *nid, struct lnet_ni *ni)
1643 {
1644         struct lnet_net *net;
1645
1646         /* must called with hold of lnet_net_lock */
1647         if (LNET_CPT_NUMBER == 1)
1648                 return 0; /* the only one */
1649
1650         /*
1651          * If NI is provided then use the CPT identified in the NI cpt
1652          * list if one exists. If one doesn't exist, then that NI is
1653          * associated with all CPTs and it follows that the net it belongs
1654          * to is implicitly associated with all CPTs, so just hash the nid
1655          * and return that.
1656          */
1657         if (ni != NULL) {
1658                 if (ni->ni_cpts != NULL)
1659                         return ni->ni_cpts[lnet_nid_cpt_hash(nid,
1660                                                              ni->ni_ncpts)];
1661                 else
1662                         return lnet_nid_cpt_hash(nid, LNET_CPT_NUMBER);
1663         }
1664
1665         /* no NI provided so look at the net */
1666         net = lnet_get_net_locked(LNET_NID_NET(nid));
1667
1668         if (net != NULL && net->net_cpts != NULL) {
1669                 return net->net_cpts[lnet_nid_cpt_hash(nid, net->net_ncpts)];
1670         }
1671
1672         return lnet_nid_cpt_hash(nid, LNET_CPT_NUMBER);
1673 }
1674
1675 int
1676 lnet_nid2cpt(struct lnet_nid *nid, struct lnet_ni *ni)
1677 {
1678         int     cpt;
1679         int     cpt2;
1680
1681         if (LNET_CPT_NUMBER == 1)
1682                 return 0; /* the only one */
1683
1684         cpt = lnet_net_lock_current();
1685
1686         cpt2 = lnet_cpt_of_nid_locked(nid, ni);
1687
1688         lnet_net_unlock(cpt);
1689
1690         return cpt2;
1691 }
1692 EXPORT_SYMBOL(lnet_nid2cpt);
1693
1694 int
1695 lnet_cpt_of_nid(lnet_nid_t nid4, struct lnet_ni *ni)
1696 {
1697         struct lnet_nid nid;
1698
1699         if (LNET_CPT_NUMBER == 1)
1700                 return 0; /* the only one */
1701
1702         lnet_nid4_to_nid(nid4, &nid);
1703         return lnet_nid2cpt(&nid, ni);
1704 }
1705 EXPORT_SYMBOL(lnet_cpt_of_nid);
1706
1707 int
1708 lnet_islocalnet_locked(__u32 net_id)
1709 {
1710         struct lnet_net *net;
1711         bool local;
1712
1713         net = lnet_get_net_locked(net_id);
1714
1715         local = net != NULL;
1716
1717         return local;
1718 }
1719
1720 int
1721 lnet_islocalnet(__u32 net_id)
1722 {
1723         int cpt;
1724         bool local;
1725
1726         cpt = lnet_net_lock_current();
1727
1728         local = lnet_islocalnet_locked(net_id);
1729
1730         lnet_net_unlock(cpt);
1731
1732         return local;
1733 }
1734
1735 struct lnet_ni  *
1736 lnet_nid_to_ni_locked(struct lnet_nid *nid, int cpt)
1737 {
1738         struct lnet_net  *net;
1739         struct lnet_ni *ni;
1740
1741         LASSERT(cpt != LNET_LOCK_EX);
1742
1743         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
1744                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
1745                         if (nid_same(&ni->ni_nid, nid))
1746                                 return ni;
1747                 }
1748         }
1749
1750         return NULL;
1751 }
1752
1753 struct lnet_ni *
1754 lnet_nid_to_ni_addref(struct lnet_nid *nid)
1755 {
1756         struct lnet_ni *ni;
1757
1758         lnet_net_lock(0);
1759         ni = lnet_nid_to_ni_locked(nid, 0);
1760         if (ni)
1761                 lnet_ni_addref_locked(ni, 0);
1762         lnet_net_unlock(0);
1763
1764         return ni;
1765 }
1766 EXPORT_SYMBOL(lnet_nid_to_ni_addref);
1767
1768 int
1769 lnet_islocalnid(struct lnet_nid *nid)
1770 {
1771         struct lnet_ni  *ni;
1772         int             cpt;
1773
1774         cpt = lnet_net_lock_current();
1775         ni = lnet_nid_to_ni_locked(nid, cpt);
1776         lnet_net_unlock(cpt);
1777
1778         return ni != NULL;
1779 }
1780
1781 int
1782 lnet_count_acceptor_nets(void)
1783 {
1784         /* Return the # of NIs that need the acceptor. */
1785         int              count = 0;
1786         struct lnet_net  *net;
1787         int              cpt;
1788
1789         cpt = lnet_net_lock_current();
1790         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
1791                 /* all socklnd type networks should have the acceptor
1792                  * thread started */
1793                 if (net->net_lnd->lnd_accept != NULL)
1794                         count++;
1795         }
1796
1797         lnet_net_unlock(cpt);
1798
1799         return count;
1800 }
1801
1802 struct lnet_ping_buffer *
1803 lnet_ping_buffer_alloc(int nbytes, gfp_t gfp)
1804 {
1805         struct lnet_ping_buffer *pbuf;
1806
1807         LIBCFS_ALLOC_GFP(pbuf, LNET_PING_BUFFER_SIZE(nbytes), gfp);
1808         if (pbuf) {
1809                 pbuf->pb_nbytes = nbytes;       /* sizeof of pb_info */
1810                 pbuf->pb_needs_post = false;
1811                 atomic_set(&pbuf->pb_refcnt, 1);
1812         }
1813
1814         return pbuf;
1815 }
1816
1817 void
1818 lnet_ping_buffer_free(struct lnet_ping_buffer *pbuf)
1819 {
1820         LASSERT(atomic_read(&pbuf->pb_refcnt) == 0);
1821         LIBCFS_FREE(pbuf, LNET_PING_BUFFER_SIZE(pbuf->pb_nbytes));
1822 }
1823
1824 static struct lnet_ping_buffer *
1825 lnet_ping_target_create(int nbytes)
1826 {
1827         struct lnet_ping_buffer *pbuf;
1828
1829         pbuf = lnet_ping_buffer_alloc(nbytes, GFP_NOFS);
1830         if (pbuf == NULL) {
1831                 CERROR("Can't allocate ping source [%d]\n", nbytes);
1832                 return NULL;
1833         }
1834
1835         pbuf->pb_info.pi_nnis = 0;
1836         pbuf->pb_info.pi_pid = the_lnet.ln_pid;
1837         pbuf->pb_info.pi_magic = LNET_PROTO_PING_MAGIC;
1838         pbuf->pb_info.pi_features =
1839                 LNET_PING_FEAT_NI_STATUS | LNET_PING_FEAT_MULTI_RAIL;
1840
1841         return pbuf;
1842 }
1843
1844 static inline int
1845 lnet_get_net_ni_bytes_locked(struct lnet_net *net)
1846 {
1847         struct lnet_ni *ni;
1848         int bytes = 0;
1849
1850         list_for_each_entry(ni, &net->net_ni_list, ni_netlist)
1851                 bytes += lnet_ping_sts_size(&ni->ni_nid);
1852
1853         return bytes;
1854 }
1855
1856 static inline int
1857 lnet_get_ni_bytes(void)
1858 {
1859         struct lnet_ni *ni;
1860         struct lnet_net *net;
1861         int bytes = 0;
1862
1863         lnet_net_lock(0);
1864
1865         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
1866                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist)
1867                         bytes += lnet_ping_sts_size(&ni->ni_nid);
1868         }
1869
1870         lnet_net_unlock(0);
1871
1872         return bytes;
1873 }
1874
1875 void
1876 lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf)
1877 {
1878         struct lnet_ni_large_status *lstat, *lend;
1879         struct lnet_ni_status *stat, *end;
1880         int nnis;
1881         int i;
1882
1883         __swab32s(&pbuf->pb_info.pi_magic);
1884         __swab32s(&pbuf->pb_info.pi_features);
1885         __swab32s(&pbuf->pb_info.pi_pid);
1886         __swab32s(&pbuf->pb_info.pi_nnis);
1887         nnis = pbuf->pb_info.pi_nnis;
1888         stat = &pbuf->pb_info.pi_ni[0];
1889         end = (void *)&pbuf->pb_info + pbuf->pb_nbytes;
1890         for (i = 0; i < nnis && stat + 1 <= end; i++, stat++) {
1891                 __swab64s(&stat->ns_nid);
1892                 __swab32s(&stat->ns_status);
1893                 if (i == 0)
1894                         /* Might be total size */
1895                         __swab32s(&stat->ns_msg_size);
1896         }
1897         if (!(pbuf->pb_info.pi_features & LNET_PING_FEAT_LARGE_ADDR))
1898                 return;
1899
1900         lstat = (struct lnet_ni_large_status *)stat;
1901         lend = (void *)end;
1902         while (lstat + 1 <= lend) {
1903                 __swab32s(&lstat->ns_status);
1904                 /* struct lnet_nid never needs to be swabed */
1905                 lstat = lnet_ping_sts_next(lstat);
1906         }
1907 }
1908
1909 int
1910 lnet_ping_info_validate(struct lnet_ping_info *pinfo)
1911 {
1912         if (!pinfo)
1913                 return -EINVAL;
1914         if (pinfo->pi_magic != LNET_PROTO_PING_MAGIC)
1915                 return -EPROTO;
1916         if (!(pinfo->pi_features & LNET_PING_FEAT_NI_STATUS))
1917                 return -EPROTO;
1918         /* Loopback is guaranteed to be present */
1919         if (pinfo->pi_nnis < 1 || pinfo->pi_nnis > lnet_interfaces_max)
1920                 return -ERANGE;
1921         if (LNET_PING_INFO_LONI(pinfo) != LNET_NID_LO_0)
1922                 return -EPROTO;
1923         return 0;
1924 }
1925
1926 static void
1927 lnet_ping_target_destroy(void)
1928 {
1929         struct lnet_net *net;
1930         struct lnet_ni  *ni;
1931
1932         lnet_net_lock(LNET_LOCK_EX);
1933
1934         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
1935                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
1936                         lnet_ni_lock(ni);
1937                         ni->ni_status = NULL;
1938                         lnet_ni_unlock(ni);
1939                 }
1940         }
1941
1942         lnet_ping_buffer_decref(the_lnet.ln_ping_target);
1943         the_lnet.ln_ping_target = NULL;
1944
1945         lnet_net_unlock(LNET_LOCK_EX);
1946 }
1947
1948 static void
1949 lnet_ping_target_event_handler(struct lnet_event *event)
1950 {
1951         struct lnet_ping_buffer *pbuf = event->md_user_ptr;
1952
1953         if (event->unlinked)
1954                 lnet_ping_buffer_decref(pbuf);
1955 }
1956
1957 static int
1958 lnet_ping_target_setup(struct lnet_ping_buffer **ppbuf,
1959                        struct lnet_handle_md *ping_mdh,
1960                        int ni_bytes, bool set_eq)
1961 {
1962         struct lnet_processid id = {
1963                 .nid = LNET_ANY_NID,
1964                 .pid = LNET_PID_ANY
1965         };
1966         struct lnet_me *me;
1967         struct lnet_md md = { NULL };
1968         int rc;
1969
1970         if (set_eq)
1971                 the_lnet.ln_ping_target_handler =
1972                         lnet_ping_target_event_handler;
1973
1974         *ppbuf = lnet_ping_target_create(ni_bytes);
1975         if (*ppbuf == NULL) {
1976                 rc = -ENOMEM;
1977                 goto fail_free_eq;
1978         }
1979
1980         /* Ping target ME/MD */
1981         me = LNetMEAttach(LNET_RESERVED_PORTAL, &id,
1982                           LNET_PROTO_PING_MATCHBITS, 0,
1983                           LNET_UNLINK, LNET_INS_AFTER);
1984         if (IS_ERR(me)) {
1985                 rc = PTR_ERR(me);
1986                 CERROR("Can't create ping target ME: %d\n", rc);
1987                 goto fail_decref_ping_buffer;
1988         }
1989
1990         /* initialize md content */
1991         md.start     = &(*ppbuf)->pb_info;
1992         md.length    = (*ppbuf)->pb_nbytes;
1993         md.threshold = LNET_MD_THRESH_INF;
1994         md.max_size  = 0;
1995         md.options   = LNET_MD_OP_GET | LNET_MD_TRUNCATE |
1996                        LNET_MD_MANAGE_REMOTE;
1997         md.handler   = the_lnet.ln_ping_target_handler;
1998         md.user_ptr  = *ppbuf;
1999
2000         rc = LNetMDAttach(me, &md, LNET_RETAIN, ping_mdh);
2001         if (rc != 0) {
2002                 CERROR("Can't attach ping target MD: %d\n", rc);
2003                 goto fail_decref_ping_buffer;
2004         }
2005         lnet_ping_buffer_addref(*ppbuf);
2006
2007         return 0;
2008
2009 fail_decref_ping_buffer:
2010         LASSERT(atomic_read(&(*ppbuf)->pb_refcnt) == 1);
2011         lnet_ping_buffer_decref(*ppbuf);
2012         *ppbuf = NULL;
2013 fail_free_eq:
2014         return rc;
2015 }
2016
2017 static void
2018 lnet_ping_md_unlink(struct lnet_ping_buffer *pbuf,
2019                     struct lnet_handle_md *ping_mdh)
2020 {
2021         LNetMDUnlink(*ping_mdh);
2022         LNetInvalidateMDHandle(ping_mdh);
2023
2024         /* NB the MD could be busy; this just starts the unlink */
2025         wait_var_event_warning(&pbuf->pb_refcnt,
2026                                atomic_read(&pbuf->pb_refcnt) <= 1,
2027                                "Still waiting for ping data MD to unlink\n");
2028 }
2029
2030 static void
2031 lnet_ping_target_install_locked(struct lnet_ping_buffer *pbuf)
2032 {
2033         struct lnet_ni *ni;
2034         struct lnet_net *net;
2035         struct lnet_ni_status *ns, *end;
2036         struct lnet_ni_large_status *lns, *lend;
2037         int rc;
2038
2039         pbuf->pb_info.pi_nnis = 0;
2040         ns = &pbuf->pb_info.pi_ni[0];
2041         end = (void *)&pbuf->pb_info + pbuf->pb_nbytes;
2042         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
2043                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
2044                         if (!nid_is_nid4(&ni->ni_nid)) {
2045                                 if (ns == &pbuf->pb_info.pi_ni[1]) {
2046                                         /* This is primary, and it is long */
2047                                         pbuf->pb_info.pi_features |=
2048                                                 LNET_PING_FEAT_PRIMARY_LARGE;
2049                                 }
2050                                 continue;
2051                         }
2052                         LASSERT(ns + 1 <= end);
2053                         ns->ns_nid = lnet_nid_to_nid4(&ni->ni_nid);
2054
2055                         lnet_ni_lock(ni);
2056                         ns->ns_status = lnet_ni_get_status_locked(ni);
2057                         ni->ni_status = &ns->ns_status;
2058                         lnet_ni_unlock(ni);
2059
2060                         pbuf->pb_info.pi_nnis++;
2061                         ns++;
2062                 }
2063         }
2064
2065         lns = (void *)ns;
2066         lend = (void *)end;
2067         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
2068                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
2069                         if (nid_is_nid4(&ni->ni_nid))
2070                                 continue;
2071                         LASSERT(lns + 1 <= lend);
2072
2073                         lns->ns_nid = ni->ni_nid;
2074
2075                         lnet_ni_lock(ni);
2076                         lns->ns_status = lnet_ni_get_status_locked(ni);
2077                         ni->ni_status = &lns->ns_status;
2078                         lnet_ni_unlock(ni);
2079
2080                         lns = lnet_ping_sts_next(lns);
2081                 }
2082         }
2083         if ((void *)lns > (void *)ns) {
2084                 /* Record total info size */
2085                 pbuf->pb_info.pi_ni[0].ns_msg_size =
2086                         (void *)lns - (void *)&pbuf->pb_info;
2087                 pbuf->pb_info.pi_features |= LNET_PING_FEAT_LARGE_ADDR;
2088         }
2089
2090         /* We (ab)use the ns_status of the loopback interface to
2091          * transmit the sequence number. The first interface listed
2092          * must be the loopback interface.
2093          */
2094         rc = lnet_ping_info_validate(&pbuf->pb_info);
2095         if (rc) {
2096                 LCONSOLE_EMERG("Invalid ping target: %d\n", rc);
2097                 LBUG();
2098         }
2099         LNET_PING_BUFFER_SEQNO(pbuf) =
2100                 atomic_inc_return(&the_lnet.ln_ping_target_seqno);
2101 }
2102
2103 static void
2104 lnet_ping_target_update(struct lnet_ping_buffer *pbuf,
2105                         struct lnet_handle_md ping_mdh)
2106 __must_hold(&the_lnet.ln_api_mutex)
2107 {
2108         struct lnet_ping_buffer *old_pbuf = NULL;
2109         struct lnet_handle_md old_ping_md;
2110
2111         /* switch the NIs to point to the new ping info created */
2112         lnet_net_lock(LNET_LOCK_EX);
2113
2114         if (!the_lnet.ln_routing)
2115                 pbuf->pb_info.pi_features |= LNET_PING_FEAT_RTE_DISABLED;
2116         if (!lnet_peer_discovery_disabled)
2117                 pbuf->pb_info.pi_features |= LNET_PING_FEAT_DISCOVERY;
2118
2119         /* Ensure only known feature bits have been set. */
2120         LASSERT(pbuf->pb_info.pi_features & LNET_PING_FEAT_BITS);
2121         LASSERT(!(pbuf->pb_info.pi_features & ~LNET_PING_FEAT_BITS));
2122
2123         lnet_ping_target_install_locked(pbuf);
2124
2125         if (the_lnet.ln_ping_target) {
2126                 old_pbuf = the_lnet.ln_ping_target;
2127                 old_ping_md = the_lnet.ln_ping_target_md;
2128         }
2129         the_lnet.ln_ping_target_md = ping_mdh;
2130         the_lnet.ln_ping_target = pbuf;
2131
2132         lnet_net_unlock(LNET_LOCK_EX);
2133
2134         if (old_pbuf) {
2135                 /* unlink and free the old ping info.
2136                  * There may be outstanding traffic on this MD, and
2137                  * ln_api_mutex may be required to finalize that
2138                  * traffic. Release ln_api_mutex while we wait for
2139                  * refs on this ping buffer to drop
2140                  */
2141                 mutex_unlock(&the_lnet.ln_api_mutex);
2142                 lnet_ping_md_unlink(old_pbuf, &old_ping_md);
2143                 mutex_lock(&the_lnet.ln_api_mutex);
2144                 lnet_ping_buffer_decref(old_pbuf);
2145         }
2146
2147         lnet_push_update_to_peers(0);
2148 }
2149
2150 static void
2151 lnet_ping_target_fini(void)
2152 {
2153         lnet_ping_md_unlink(the_lnet.ln_ping_target,
2154                             &the_lnet.ln_ping_target_md);
2155
2156         lnet_assert_handler_unused(the_lnet.ln_ping_target_handler, true);
2157         lnet_ping_target_destroy();
2158 }
2159
2160 /* Resize the push target. */
2161 int lnet_push_target_resize(void)
2162 {
2163         struct lnet_handle_md mdh;
2164         struct lnet_handle_md old_mdh;
2165         struct lnet_ping_buffer *pbuf;
2166         struct lnet_ping_buffer *old_pbuf;
2167         int nbytes;
2168         int rc;
2169
2170 again:
2171         nbytes = the_lnet.ln_push_target_nbytes;
2172         if (nbytes <= 0) {
2173                 CDEBUG(D_NET, "Invalid nbytes %d\n", nbytes);
2174                 return -EINVAL;
2175         }
2176
2177         /* NB: lnet_ping_buffer_alloc() sets pbuf refcount to 1. That ref is
2178          * dropped when we need to resize again (see "old_pbuf" below) or when
2179          * LNet is shutdown (see lnet_push_target_fini())
2180          */
2181         pbuf = lnet_ping_buffer_alloc(nbytes, GFP_NOFS);
2182         if (!pbuf) {
2183                 CDEBUG(D_NET, "Can't allocate pbuf for nbytes %d\n", nbytes);
2184                 return -ENOMEM;
2185         }
2186
2187         rc = lnet_push_target_post(pbuf, &mdh);
2188         if (rc) {
2189                 CDEBUG(D_NET, "Failed to post push target: %d\n", rc);
2190                 lnet_ping_buffer_decref(pbuf);
2191                 return rc;
2192         }
2193
2194         lnet_net_lock(LNET_LOCK_EX);
2195         old_pbuf = the_lnet.ln_push_target;
2196         old_mdh = the_lnet.ln_push_target_md;
2197         the_lnet.ln_push_target = pbuf;
2198         the_lnet.ln_push_target_md = mdh;
2199         lnet_net_unlock(LNET_LOCK_EX);
2200
2201         if (old_pbuf) {
2202                 LNetMDUnlink(old_mdh);
2203                 /* Drop ref set by lnet_ping_buffer_alloc() */
2204                 lnet_ping_buffer_decref(old_pbuf);
2205         }
2206
2207         /* Received another push or reply that requires a larger buffer */
2208         if (nbytes < the_lnet.ln_push_target_nbytes)
2209                 goto again;
2210
2211         CDEBUG(D_NET, "nbytes %d success\n", nbytes);
2212         return 0;
2213 }
2214
2215 int lnet_push_target_post(struct lnet_ping_buffer *pbuf,
2216                           struct lnet_handle_md *mdhp)
2217 {
2218         struct lnet_processid id = { LNET_ANY_NID, LNET_PID_ANY };
2219         struct lnet_md md = { NULL };
2220         struct lnet_me *me;
2221         int rc;
2222
2223         me = LNetMEAttach(LNET_RESERVED_PORTAL, &id,
2224                           LNET_PROTO_PING_MATCHBITS, 0,
2225                           LNET_UNLINK, LNET_INS_AFTER);
2226         if (IS_ERR(me)) {
2227                 rc = PTR_ERR(me);
2228                 CERROR("Can't create push target ME: %d\n", rc);
2229                 return rc;
2230         }
2231
2232         pbuf->pb_needs_post = false;
2233
2234         /* This reference is dropped by lnet_push_target_event_handler() */
2235         lnet_ping_buffer_addref(pbuf);
2236
2237         /* initialize md content */
2238         md.start     = &pbuf->pb_info;
2239         md.length    = pbuf->pb_nbytes;
2240         md.threshold = 1;
2241         md.max_size  = 0;
2242         md.options   = LNET_MD_OP_PUT | LNET_MD_TRUNCATE;
2243         md.user_ptr  = pbuf;
2244         md.handler   = the_lnet.ln_push_target_handler;
2245
2246         rc = LNetMDAttach(me, &md, LNET_UNLINK, mdhp);
2247         if (rc) {
2248                 CERROR("Can't attach push MD: %d\n", rc);
2249                 lnet_ping_buffer_decref(pbuf);
2250                 pbuf->pb_needs_post = true;
2251                 return rc;
2252         }
2253
2254         CDEBUG(D_NET, "posted push target %p\n", pbuf);
2255
2256         return 0;
2257 }
2258
2259 static void lnet_push_target_event_handler(struct lnet_event *ev)
2260 {
2261         struct lnet_ping_buffer *pbuf = ev->md_user_ptr;
2262
2263         CDEBUG(D_NET, "type %d status %d unlinked %d\n", ev->type, ev->status,
2264                ev->unlinked);
2265
2266         if (pbuf->pb_info.pi_magic == __swab32(LNET_PROTO_PING_MAGIC))
2267                 lnet_swap_pinginfo(pbuf);
2268
2269         if (ev->type == LNET_EVENT_UNLINK) {
2270                 /* Drop ref added by lnet_push_target_post() */
2271                 lnet_ping_buffer_decref(pbuf);
2272                 return;
2273         }
2274
2275         lnet_peer_push_event(ev);
2276         if (ev->unlinked)
2277                 /* Drop ref added by lnet_push_target_post */
2278                 lnet_ping_buffer_decref(pbuf);
2279 }
2280
2281 /* Initialize the push target. */
2282 static int lnet_push_target_init(void)
2283 {
2284         int rc;
2285
2286         if (the_lnet.ln_push_target)
2287                 return -EALREADY;
2288
2289         the_lnet.ln_push_target_handler =
2290                 lnet_push_target_event_handler;
2291
2292         rc = LNetSetLazyPortal(LNET_RESERVED_PORTAL);
2293         LASSERT(rc == 0);
2294
2295         /* Start at the required minimum, we'll enlarge if required. */
2296         the_lnet.ln_push_target_nbytes = LNET_PING_INFO_MIN_SIZE;
2297
2298         rc = lnet_push_target_resize();
2299         if (rc) {
2300                 LNetClearLazyPortal(LNET_RESERVED_PORTAL);
2301                 the_lnet.ln_push_target_handler = NULL;
2302         }
2303
2304         return rc;
2305 }
2306
2307 /* Clean up the push target. */
2308 static void lnet_push_target_fini(void)
2309 {
2310         if (!the_lnet.ln_push_target)
2311                 return;
2312
2313         /* Unlink and invalidate to prevent new references. */
2314         LNetMDUnlink(the_lnet.ln_push_target_md);
2315         LNetInvalidateMDHandle(&the_lnet.ln_push_target_md);
2316
2317         /* Wait for the unlink to complete. */
2318         wait_var_event_warning(&the_lnet.ln_push_target->pb_refcnt,
2319                                atomic_read(&the_lnet.ln_push_target->pb_refcnt) <= 1,
2320                                "Still waiting for ping data MD to unlink\n");
2321
2322         /* Drop ref set by lnet_ping_buffer_alloc() */
2323         lnet_ping_buffer_decref(the_lnet.ln_push_target);
2324         the_lnet.ln_push_target = NULL;
2325         the_lnet.ln_push_target_nbytes = 0;
2326
2327         LNetClearLazyPortal(LNET_RESERVED_PORTAL);
2328         lnet_assert_handler_unused(the_lnet.ln_push_target_handler, true);
2329         the_lnet.ln_push_target_handler = NULL;
2330 }
2331
2332 static int
2333 lnet_ni_tq_credits(struct lnet_ni *ni)
2334 {
2335         int     credits;
2336
2337         LASSERT(ni->ni_ncpts >= 1);
2338
2339         if (ni->ni_ncpts == 1)
2340                 return ni->ni_net->net_tunables.lct_max_tx_credits;
2341
2342         credits = ni->ni_net->net_tunables.lct_max_tx_credits / ni->ni_ncpts;
2343         credits = max(credits, 8 * ni->ni_net->net_tunables.lct_peer_tx_credits);
2344         credits = min(credits, ni->ni_net->net_tunables.lct_max_tx_credits);
2345
2346         return credits;
2347 }
2348
2349 static void
2350 lnet_ni_unlink_locked(struct lnet_ni *ni)
2351 {
2352         /* move it to zombie list and nobody can find it anymore */
2353         LASSERT(!list_empty(&ni->ni_netlist));
2354         list_move(&ni->ni_netlist, &ni->ni_net->net_ni_zombie);
2355         lnet_ni_decref_locked(ni, 0);
2356 }
2357
2358 static void
2359 lnet_clear_zombies_nis_locked(struct lnet_net *net)
2360 {
2361         int             i;
2362         int             islo;
2363         struct lnet_ni  *ni;
2364         struct list_head *zombie_list = &net->net_ni_zombie;
2365
2366         /*
2367          * Now wait for the NIs I just nuked to show up on the zombie
2368          * list and shut them down in guaranteed thread context
2369          */
2370         i = 2;
2371         while ((ni = list_first_entry_or_null(zombie_list,
2372                                               struct lnet_ni,
2373                                               ni_netlist)) != NULL) {
2374                 int *ref;
2375                 int j;
2376
2377                 list_del_init(&ni->ni_netlist);
2378                 /* the ni should be in deleting state. If it's not it's
2379                  * a bug */
2380                 LASSERT(ni->ni_state == LNET_NI_STATE_DELETING);
2381                 cfs_percpt_for_each(ref, j, ni->ni_refs) {
2382                         if (*ref == 0)
2383                                 continue;
2384                         /* still busy, add it back to zombie list */
2385                         list_add(&ni->ni_netlist, zombie_list);
2386                         break;
2387                 }
2388
2389                 if (!list_empty(&ni->ni_netlist)) {
2390                         /* Unlock mutex while waiting to allow other
2391                          * threads to read the LNet state and fall through
2392                          * to avoid deadlock
2393                          */
2394                         lnet_net_unlock(LNET_LOCK_EX);
2395                         mutex_unlock(&the_lnet.ln_api_mutex);
2396
2397                         ++i;
2398                         if ((i & (-i)) == i) {
2399                                 CDEBUG(D_WARNING,
2400                                        "Waiting for zombie LNI %s\n",
2401                                        libcfs_nidstr(&ni->ni_nid));
2402                         }
2403                         schedule_timeout_uninterruptible(cfs_time_seconds(1));
2404
2405                         mutex_lock(&the_lnet.ln_api_mutex);
2406                         lnet_net_lock(LNET_LOCK_EX);
2407                         continue;
2408                 }
2409
2410                 lnet_net_unlock(LNET_LOCK_EX);
2411
2412                 islo = ni->ni_net->net_lnd->lnd_type == LOLND;
2413
2414                 LASSERT(!in_interrupt());
2415                 /* Holding the LND mutex makes it safe for lnd_shutdown
2416                  * to call module_put(). Module unload cannot finish
2417                  * until lnet_unregister_lnd() completes, and that
2418                  * requires the LND mutex.
2419                  */
2420                 mutex_unlock(&the_lnet.ln_api_mutex);
2421                 mutex_lock(&the_lnet.ln_lnd_mutex);
2422                 (net->net_lnd->lnd_shutdown)(ni);
2423                 mutex_unlock(&the_lnet.ln_lnd_mutex);
2424                 mutex_lock(&the_lnet.ln_api_mutex);
2425
2426                 if (!islo)
2427                         CDEBUG(D_LNI, "Removed LNI %s\n",
2428                               libcfs_nidstr(&ni->ni_nid));
2429
2430                 lnet_ni_free(ni);
2431                 i = 2;
2432                 lnet_net_lock(LNET_LOCK_EX);
2433         }
2434 }
2435
2436 /* shutdown down the NI and release refcount */
2437 static void
2438 lnet_shutdown_lndni(struct lnet_ni *ni)
2439 {
2440         int i;
2441         struct lnet_net *net = ni->ni_net;
2442
2443         lnet_net_lock(LNET_LOCK_EX);
2444         lnet_ni_lock(ni);
2445         ni->ni_state = LNET_NI_STATE_DELETING;
2446         lnet_ni_unlock(ni);
2447         lnet_ni_unlink_locked(ni);
2448         lnet_incr_dlc_seq();
2449         lnet_net_unlock(LNET_LOCK_EX);
2450
2451         /* clear messages for this NI on the lazy portal */
2452         for (i = 0; i < the_lnet.ln_nportals; i++)
2453                 lnet_clear_lazy_portal(ni, i, "Shutting down NI");
2454
2455         lnet_net_lock(LNET_LOCK_EX);
2456         lnet_clear_zombies_nis_locked(net);
2457         lnet_net_unlock(LNET_LOCK_EX);
2458 }
2459
2460 static void
2461 lnet_shutdown_lndnet(struct lnet_net *net)
2462 {
2463         struct lnet_ni *ni;
2464
2465         lnet_net_lock(LNET_LOCK_EX);
2466
2467         list_del_init(&net->net_list);
2468
2469         while ((ni = list_first_entry_or_null(&net->net_ni_list,
2470                                               struct lnet_ni,
2471                                               ni_netlist)) != NULL) {
2472                 lnet_net_unlock(LNET_LOCK_EX);
2473                 lnet_shutdown_lndni(ni);
2474                 lnet_net_lock(LNET_LOCK_EX);
2475         }
2476
2477         lnet_net_unlock(LNET_LOCK_EX);
2478
2479         /* Do peer table cleanup for this net */
2480         lnet_peer_tables_cleanup(net);
2481
2482         lnet_net_free(net);
2483 }
2484
2485 static void
2486 lnet_shutdown_lndnets(void)
2487 {
2488         struct lnet_net *net;
2489         LIST_HEAD(resend);
2490         struct lnet_msg *msg, *tmp;
2491
2492         /* NB called holding the global mutex */
2493
2494         /* All quiet on the API front */
2495         LASSERT(the_lnet.ln_state == LNET_STATE_RUNNING ||
2496                 the_lnet.ln_state == LNET_STATE_STOPPING);
2497         LASSERT(the_lnet.ln_refcount == 0);
2498
2499         lnet_net_lock(LNET_LOCK_EX);
2500         the_lnet.ln_state = LNET_STATE_STOPPING;
2501
2502         /*
2503          * move the nets to the zombie list to avoid them being
2504          * picked up for new work. LONET is also included in the
2505          * Nets that will be moved to the zombie list
2506          */
2507         list_splice_init(&the_lnet.ln_nets, &the_lnet.ln_net_zombie);
2508
2509         /* Drop the cached loopback Net. */
2510         if (the_lnet.ln_loni != NULL) {
2511                 lnet_ni_decref_locked(the_lnet.ln_loni, 0);
2512                 the_lnet.ln_loni = NULL;
2513         }
2514         lnet_net_unlock(LNET_LOCK_EX);
2515
2516         /* iterate through the net zombie list and delete each net */
2517         while ((net = list_first_entry_or_null(&the_lnet.ln_net_zombie,
2518                                                struct lnet_net,
2519                                                net_list)) != NULL)
2520                 lnet_shutdown_lndnet(net);
2521
2522         spin_lock(&the_lnet.ln_msg_resend_lock);
2523         list_splice(&the_lnet.ln_msg_resend, &resend);
2524         spin_unlock(&the_lnet.ln_msg_resend_lock);
2525
2526         list_for_each_entry_safe(msg, tmp, &resend, msg_list) {
2527                 list_del_init(&msg->msg_list);
2528                 msg->msg_no_resend = true;
2529                 lnet_finalize(msg, -ECANCELED);
2530         }
2531
2532         lnet_net_lock(LNET_LOCK_EX);
2533         the_lnet.ln_state = LNET_STATE_SHUTDOWN;
2534         lnet_net_unlock(LNET_LOCK_EX);
2535 }
2536
2537 static int
2538 lnet_startup_lndni(struct lnet_ni *ni, struct lnet_lnd_tunables *tun)
2539 {
2540         int                     rc = -EINVAL;
2541         struct lnet_tx_queue    *tq;
2542         int                     i;
2543         struct lnet_net         *net = ni->ni_net;
2544
2545         mutex_lock(&the_lnet.ln_lnd_mutex);
2546
2547         if (tun) {
2548                 memcpy(&ni->ni_lnd_tunables, tun, sizeof(*tun));
2549                 ni->ni_lnd_tunables_set = true;
2550         }
2551
2552         rc = (net->net_lnd->lnd_startup)(ni);
2553
2554         mutex_unlock(&the_lnet.ln_lnd_mutex);
2555
2556         if (rc != 0) {
2557                 LCONSOLE_ERROR_MSG(0x105, "Error %d starting up LNI %s\n",
2558                                    rc, libcfs_lnd2str(net->net_lnd->lnd_type));
2559                 goto failed0;
2560         }
2561
2562         /* We keep a reference on the loopback net through the loopback NI */
2563         if (net->net_lnd->lnd_type == LOLND) {
2564                 lnet_ni_addref(ni);
2565                 LASSERT(the_lnet.ln_loni == NULL);
2566                 the_lnet.ln_loni = ni;
2567                 ni->ni_net->net_tunables.lct_peer_tx_credits = 0;
2568                 ni->ni_net->net_tunables.lct_peer_rtr_credits = 0;
2569                 ni->ni_net->net_tunables.lct_max_tx_credits = 0;
2570                 ni->ni_net->net_tunables.lct_peer_timeout = 0;
2571                 return 0;
2572         }
2573
2574         if (ni->ni_net->net_tunables.lct_peer_tx_credits == 0 ||
2575             ni->ni_net->net_tunables.lct_max_tx_credits == 0) {
2576                 LCONSOLE_ERROR_MSG(0x107, "LNI %s has no %scredits\n",
2577                                    libcfs_lnd2str(net->net_lnd->lnd_type),
2578                                    ni->ni_net->net_tunables.lct_peer_tx_credits == 0 ?
2579                                         "" : "per-peer ");
2580                 /* shutdown the NI since if we get here then it must've already
2581                  * been started
2582                  */
2583                 lnet_shutdown_lndni(ni);
2584                 return -EINVAL;
2585         }
2586
2587         cfs_percpt_for_each(tq, i, ni->ni_tx_queues) {
2588                 tq->tq_credits_min =
2589                 tq->tq_credits_max =
2590                 tq->tq_credits = lnet_ni_tq_credits(ni);
2591         }
2592
2593         atomic_set(&ni->ni_tx_credits,
2594                    lnet_ni_tq_credits(ni) * ni->ni_ncpts);
2595         atomic_set(&ni->ni_healthv, LNET_MAX_HEALTH_VALUE);
2596
2597         /* Nodes with small feet have little entropy. The NID for this
2598          * node gives the most entropy in the low bits.
2599          */
2600         add_device_randomness(&ni->ni_nid, sizeof(ni->ni_nid));
2601
2602         CDEBUG(D_LNI, "Added LNI %s [%d/%d/%d/%d]\n",
2603                 libcfs_nidstr(&ni->ni_nid),
2604                 ni->ni_net->net_tunables.lct_peer_tx_credits,
2605                 lnet_ni_tq_credits(ni) * LNET_CPT_NUMBER,
2606                 ni->ni_net->net_tunables.lct_peer_rtr_credits,
2607                 ni->ni_net->net_tunables.lct_peer_timeout);
2608
2609         return 0;
2610 failed0:
2611         lnet_ni_free(ni);
2612         return rc;
2613 }
2614
2615 static const struct lnet_lnd *lnet_load_lnd(u32 lnd_type)
2616 {
2617         const struct lnet_lnd *lnd;
2618         int rc = 0;
2619
2620         mutex_lock(&the_lnet.ln_lnd_mutex);
2621         lnd = lnet_find_lnd_by_type(lnd_type);
2622         if (!lnd) {
2623                 mutex_unlock(&the_lnet.ln_lnd_mutex);
2624                 rc = request_module("%s", libcfs_lnd2modname(lnd_type));
2625                 mutex_lock(&the_lnet.ln_lnd_mutex);
2626
2627                 lnd = lnet_find_lnd_by_type(lnd_type);
2628                 if (!lnd) {
2629                         mutex_unlock(&the_lnet.ln_lnd_mutex);
2630                         CERROR("Can't load LND %s, module %s, rc=%d\n",
2631                         libcfs_lnd2str(lnd_type),
2632                         libcfs_lnd2modname(lnd_type), rc);
2633 #ifndef HAVE_MODULE_LOADING_SUPPORT
2634                         LCONSOLE_ERROR_MSG(0x104,
2635                                            "Your kernel must be compiled with kernel module loading support.");
2636 #endif
2637                         return ERR_PTR(-EINVAL);
2638                 }
2639         }
2640         mutex_unlock(&the_lnet.ln_lnd_mutex);
2641
2642         return lnd;
2643 }
2644
2645 static int
2646 lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
2647 {
2648         struct lnet_ni *ni;
2649         struct lnet_net *net_l = NULL;
2650         LIST_HEAD(local_ni_list);
2651         int rc;
2652         int ni_count = 0;
2653         __u32 lnd_type;
2654         const struct lnet_lnd  *lnd;
2655         int peer_timeout =
2656                 net->net_tunables.lct_peer_timeout;
2657         int maxtxcredits =
2658                 net->net_tunables.lct_max_tx_credits;
2659         int peerrtrcredits =
2660                 net->net_tunables.lct_peer_rtr_credits;
2661
2662         /*
2663          * make sure that this net is unique. If it isn't then
2664          * we are adding interfaces to an already existing network, and
2665          * 'net' is just a convenient way to pass in the list.
2666          * if it is unique we need to find the LND and load it if
2667          * necessary.
2668          */
2669         if (lnet_net_unique(net->net_id, &the_lnet.ln_nets, &net_l)) {
2670                 lnd_type = LNET_NETTYP(net->net_id);
2671
2672                 lnd = lnet_load_lnd(lnd_type);
2673                 if (IS_ERR(lnd)) {
2674                         rc = PTR_ERR(lnd);
2675                         goto failed0;
2676                 }
2677
2678                 mutex_lock(&the_lnet.ln_lnd_mutex);
2679                 net->net_lnd = lnd;
2680                 mutex_unlock(&the_lnet.ln_lnd_mutex);
2681
2682                 net_l = net;
2683         }
2684
2685         /*
2686          * net_l: if the network being added is unique then net_l
2687          *        will point to that network
2688          *        if the network being added is not unique then
2689          *        net_l points to the existing network.
2690          *
2691          * When we enter the loop below, we'll pick NIs off he
2692          * network beign added and start them up, then add them to
2693          * a local ni list. Once we've successfully started all
2694          * the NIs then we join the local NI list (of started up
2695          * networks) with the net_l->net_ni_list, which should
2696          * point to the correct network to add the new ni list to
2697          *
2698          * If any of the new NIs fail to start up, then we want to
2699          * iterate through the local ni list, which should include
2700          * any NIs which were successfully started up, and shut
2701          * them down.
2702          *
2703          * After than we want to delete the network being added,
2704          * to avoid a memory leak.
2705          */
2706         while ((ni = list_first_entry_or_null(&net->net_ni_added,
2707                                               struct lnet_ni,
2708                                               ni_netlist)) != NULL) {
2709                 list_del_init(&ni->ni_netlist);
2710
2711                 /* make sure that the the NI we're about to start
2712                  * up is actually unique. if it's not fail. */
2713                 if (!lnet_ni_unique_net(&net_l->net_ni_list,
2714                                         ni->ni_interface)) {
2715                         rc = -EEXIST;
2716                         goto failed1;
2717                 }
2718
2719                 /* adjust the pointer the parent network, just in case it
2720                  * the net is a duplicate */
2721                 ni->ni_net = net_l;
2722
2723                 rc = lnet_startup_lndni(ni, tun);
2724
2725                 if (rc != 0)
2726                         goto failed1;
2727
2728                 lnet_ni_addref(ni);
2729                 list_add_tail(&ni->ni_netlist, &local_ni_list);
2730
2731                 ni_count++;
2732         }
2733
2734         lnet_net_lock(LNET_LOCK_EX);
2735         list_splice_tail(&local_ni_list, &net_l->net_ni_list);
2736         lnet_incr_dlc_seq();
2737
2738         list_for_each_entry(ni, &net_l->net_ni_list, ni_netlist) {
2739                 if (!ni)
2740                         break;
2741                 lnet_ni_lock(ni);
2742                 ni->ni_state = LNET_NI_STATE_ACTIVE;
2743                 lnet_ni_unlock(ni);
2744         }
2745         lnet_net_unlock(LNET_LOCK_EX);
2746
2747         /* if the network is not unique then we don't want to keep
2748          * it around after we're done. Free it. Otherwise add that
2749          * net to the global the_lnet.ln_nets */
2750         if (net_l != net && net_l != NULL) {
2751                 /*
2752                  * TODO - note. currently the tunables can not be updated
2753                  * once added
2754                  */
2755                 lnet_net_free(net);
2756         } else {
2757                 /*
2758                  * restore tunables after it has been overwitten by the
2759                  * lnd
2760                  */
2761                 if (peer_timeout != -1)
2762                         net->net_tunables.lct_peer_timeout = peer_timeout;
2763                 if (maxtxcredits != -1)
2764                         net->net_tunables.lct_max_tx_credits = maxtxcredits;
2765                 if (peerrtrcredits != -1)
2766                         net->net_tunables.lct_peer_rtr_credits = peerrtrcredits;
2767
2768                 lnet_net_lock(LNET_LOCK_EX);
2769                 list_add_tail(&net->net_list, &the_lnet.ln_nets);
2770                 lnet_net_unlock(LNET_LOCK_EX);
2771         }
2772
2773         return ni_count;
2774
2775 failed1:
2776         /*
2777          * shutdown the new NIs that are being started up
2778          * free the NET being started
2779          */
2780         while ((ni = list_first_entry_or_null(&local_ni_list,
2781                                               struct lnet_ni,
2782                                               ni_netlist)) != NULL)
2783                 lnet_shutdown_lndni(ni);
2784
2785 failed0:
2786         lnet_net_free(net);
2787
2788         return rc;
2789 }
2790
2791 static int
2792 lnet_startup_lndnets(struct list_head *netlist)
2793 {
2794         struct lnet_net         *net;
2795         int                     rc;
2796         int                     ni_count = 0;
2797
2798         /*
2799          * Change to running state before bringing up the LNDs. This
2800          * allows lnet_shutdown_lndnets() to assert that we've passed
2801          * through here.
2802          */
2803         lnet_net_lock(LNET_LOCK_EX);
2804         the_lnet.ln_state = LNET_STATE_RUNNING;
2805         lnet_net_unlock(LNET_LOCK_EX);
2806
2807         while ((net = list_first_entry_or_null(netlist,
2808                                                struct lnet_net,
2809                                                net_list)) != NULL) {
2810                 list_del_init(&net->net_list);
2811
2812                 rc = lnet_startup_lndnet(net, NULL);
2813
2814                 if (rc < 0)
2815                         goto failed;
2816
2817                 ni_count += rc;
2818         }
2819
2820         return ni_count;
2821 failed:
2822         lnet_shutdown_lndnets();
2823
2824         return rc;
2825 }
2826
2827 static int lnet_genl_parse_list(struct sk_buff *msg,
2828                                 const struct ln_key_list *data[], u16 idx)
2829 {
2830         const struct ln_key_list *list = data[idx];
2831         const struct ln_key_props *props;
2832         struct nlattr *node;
2833         u16 count;
2834
2835         if (!list)
2836                 return 0;
2837
2838         if (!list->lkl_maxattr)
2839                 return -ERANGE;
2840
2841         props = list->lkl_list;
2842         if (!props)
2843                 return -EINVAL;
2844
2845         node = nla_nest_start(msg, LN_SCALAR_ATTR_LIST);
2846         if (!node)
2847                 return -ENOBUFS;
2848
2849         for (count = 1; count <= list->lkl_maxattr; count++) {
2850                 struct nlattr *key = nla_nest_start(msg, count);
2851
2852                 if (!key)
2853                         return -EMSGSIZE;
2854
2855                 if (count == 1)
2856                         nla_put_u16(msg, LN_SCALAR_ATTR_LIST_SIZE,
2857                                     list->lkl_maxattr);
2858
2859                 nla_put_u16(msg, LN_SCALAR_ATTR_INDEX, count);
2860                 if (props[count].lkp_value)
2861                         nla_put_string(msg, LN_SCALAR_ATTR_VALUE,
2862                                        props[count].lkp_value);
2863                 if (props[count].lkp_key_format)
2864                         nla_put_u16(msg, LN_SCALAR_ATTR_KEY_FORMAT,
2865                                     props[count].lkp_key_format);
2866                 nla_put_u16(msg, LN_SCALAR_ATTR_NLA_TYPE,
2867                             props[count].lkp_data_type);
2868                 if (props[count].lkp_data_type == NLA_NESTED) {
2869                         int rc;
2870
2871                         rc = lnet_genl_parse_list(msg, data, ++idx);
2872                         if (rc < 0)
2873                                 return rc;
2874                         idx = rc;
2875                 }
2876
2877                 nla_nest_end(msg, key);
2878         }
2879
2880         nla_nest_end(msg, node);
2881         return idx;
2882 }
2883
2884 int lnet_genl_send_scalar_list(struct sk_buff *msg, u32 portid, u32 seq,
2885                                const struct genl_family *family, int flags,
2886                                u8 cmd, const struct ln_key_list *data[])
2887 {
2888         int rc = 0;
2889         void *hdr;
2890
2891         if (!data[0])
2892                 return -EINVAL;
2893
2894         hdr = genlmsg_put(msg, portid, seq, family, flags, cmd);
2895         if (!hdr)
2896                 GOTO(canceled, rc = -EMSGSIZE);
2897
2898         rc = lnet_genl_parse_list(msg, data, 0);
2899         if (rc < 0)
2900                 GOTO(canceled, rc);
2901
2902         genlmsg_end(msg, hdr);
2903 canceled:
2904         if (rc < 0)
2905                 genlmsg_cancel(msg, hdr);
2906         return rc > 0 ? 0 : rc;
2907 }
2908 EXPORT_SYMBOL(lnet_genl_send_scalar_list);
2909
2910 static struct genl_family lnet_family;
2911
2912 /**
2913  * Initialize LNet library.
2914  *
2915  * Automatically called at module loading time. Caller has to call
2916  * lnet_lib_exit() after a call to lnet_lib_init(), if and only if the
2917  * latter returned 0. It must be called exactly once.
2918  *
2919  * \retval 0 on success
2920  * \retval -ve on failures.
2921  */
2922 int lnet_lib_init(void)
2923 {
2924         int rc;
2925
2926         lnet_assert_wire_constants();
2927
2928         /* refer to global cfs_cpt_table for now */
2929         the_lnet.ln_cpt_table = cfs_cpt_tab;
2930         the_lnet.ln_cpt_number = cfs_cpt_number(cfs_cpt_tab);
2931
2932         LASSERT(the_lnet.ln_cpt_number > 0);
2933         if (the_lnet.ln_cpt_number > LNET_CPT_MAX) {
2934                 /* we are under risk of consuming all lh_cookie */
2935                 CERROR("Can't have %d CPTs for LNet (max allowed is %d), "
2936                        "please change setting of CPT-table and retry\n",
2937                        the_lnet.ln_cpt_number, LNET_CPT_MAX);
2938                 return -E2BIG;
2939         }
2940
2941         while ((1 << the_lnet.ln_cpt_bits) < the_lnet.ln_cpt_number)
2942                 the_lnet.ln_cpt_bits++;
2943
2944         rc = lnet_create_locks();
2945         if (rc != 0) {
2946                 CERROR("Can't create LNet global locks: %d\n", rc);
2947                 return rc;
2948         }
2949
2950         rc = genl_register_family(&lnet_family);
2951         if (rc != 0) {
2952                 lnet_destroy_locks();
2953                 CERROR("Can't register LNet netlink family: %d\n", rc);
2954                 return rc;
2955         }
2956
2957         the_lnet.ln_refcount = 0;
2958         INIT_LIST_HEAD(&the_lnet.ln_net_zombie);
2959         INIT_LIST_HEAD(&the_lnet.ln_msg_resend);
2960
2961         /* The hash table size is the number of bits it takes to express the set
2962          * ln_num_routes, minus 1 (better to under estimate than over so we
2963          * don't waste memory). */
2964         if (rnet_htable_size <= 0)
2965                 rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT;
2966         else if (rnet_htable_size > LNET_REMOTE_NETS_HASH_MAX)
2967                 rnet_htable_size = LNET_REMOTE_NETS_HASH_MAX;
2968         the_lnet.ln_remote_nets_hbits = max_t(int, 1,
2969                                            order_base_2(rnet_htable_size) - 1);
2970
2971         /* All LNDs apart from the LOLND are in separate modules.  They
2972          * register themselves when their module loads, and unregister
2973          * themselves when their module is unloaded. */
2974         lnet_register_lnd(&the_lolnd);
2975         return 0;
2976 }
2977
2978 /**
2979  * Finalize LNet library.
2980  *
2981  * \pre lnet_lib_init() called with success.
2982  * \pre All LNet users called LNetNIFini() for matching LNetNIInit() calls.
2983  *
2984  * As this happens at module-unload, all lnds must already be unloaded,
2985  * so they must already be unregistered.
2986  */
2987 void lnet_lib_exit(void)
2988 {
2989         int i;
2990
2991         LASSERT(the_lnet.ln_refcount == 0);
2992         lnet_unregister_lnd(&the_lolnd);
2993         for (i = 0; i < NUM_LNDS; i++)
2994                 LASSERT(!the_lnet.ln_lnds[i]);
2995         lnet_destroy_locks();
2996         genl_unregister_family(&lnet_family);
2997 }
2998
2999 /**
3000  * Set LNet PID and start LNet interfaces, routing, and forwarding.
3001  *
3002  * Users must call this function at least once before any other functions.
3003  * For each successful call there must be a corresponding call to
3004  * LNetNIFini(). For subsequent calls to LNetNIInit(), \a requested_pid is
3005  * ignored.
3006  *
3007  * The PID used by LNet may be different from the one requested.
3008  * See LNetGetId().
3009  *
3010  * \param requested_pid PID requested by the caller.
3011  *
3012  * \return >= 0 on success, and < 0 error code on failures.
3013  */
3014 int
3015 LNetNIInit(lnet_pid_t requested_pid)
3016 {
3017         int im_a_router = 0;
3018         int rc;
3019         int ni_bytes;
3020         struct lnet_ping_buffer *pbuf;
3021         struct lnet_handle_md ping_mdh;
3022         LIST_HEAD(net_head);
3023         struct lnet_net *net;
3024
3025         mutex_lock(&the_lnet.ln_api_mutex);
3026
3027         CDEBUG(D_OTHER, "refs %d\n", the_lnet.ln_refcount);
3028
3029         if (the_lnet.ln_state == LNET_STATE_STOPPING) {
3030                 mutex_unlock(&the_lnet.ln_api_mutex);
3031                 return -ESHUTDOWN;
3032         }
3033
3034         if (the_lnet.ln_refcount > 0) {
3035                 rc = the_lnet.ln_refcount++;
3036                 mutex_unlock(&the_lnet.ln_api_mutex);
3037                 return rc;
3038         }
3039
3040         rc = lnet_prepare(requested_pid);
3041         if (rc != 0) {
3042                 mutex_unlock(&the_lnet.ln_api_mutex);
3043                 return rc;
3044         }
3045
3046         /* create a network for Loopback network */
3047         net = lnet_net_alloc(LNET_MKNET(LOLND, 0), &net_head);
3048         if (net == NULL) {
3049                 rc = -ENOMEM;
3050                 goto err_empty_list;
3051         }
3052
3053         /* Add in the loopback NI */
3054         if (lnet_ni_alloc(net, NULL, NULL) == NULL) {
3055                 rc = -ENOMEM;
3056                 goto err_empty_list;
3057         }
3058
3059         if (use_tcp_bonding)
3060                 CWARN("use_tcp_bonding has been removed. Use Multi-Rail and Dynamic Discovery instead, see LU-13641\n");
3061
3062         /* If LNet is being initialized via DLC it is possible
3063          * that the user requests not to load module parameters (ones which
3064          * are supported by DLC) on initialization.  Therefore, make sure not
3065          * to load networks, routes and forwarding from module parameters
3066          * in this case.  On cleanup in case of failure only clean up
3067          * routes if it has been loaded */
3068         if (!the_lnet.ln_nis_from_mod_params) {
3069                 rc = lnet_parse_networks(&net_head, lnet_get_networks());
3070                 if (rc < 0)
3071                         goto err_empty_list;
3072         }
3073
3074         rc = lnet_startup_lndnets(&net_head);
3075         if (rc < 0)
3076                 goto err_empty_list;
3077
3078         if (!the_lnet.ln_nis_from_mod_params) {
3079                 rc = lnet_parse_routes(lnet_get_routes(), &im_a_router);
3080                 if (rc != 0)
3081                         goto err_shutdown_lndnis;
3082
3083                 rc = lnet_rtrpools_alloc(im_a_router);
3084                 if (rc != 0)
3085                         goto err_destroy_routes;
3086         }
3087
3088         rc = lnet_acceptor_start();
3089         if (rc != 0)
3090                 goto err_destroy_routes;
3091
3092         the_lnet.ln_refcount = 1;
3093         /* Now I may use my own API functions... */
3094
3095         ni_bytes = LNET_PING_INFO_HDR_SIZE;
3096         list_for_each_entry(net, &the_lnet.ln_nets, net_list)
3097                 ni_bytes += lnet_get_net_ni_bytes_locked(net);
3098
3099         rc = lnet_ping_target_setup(&pbuf, &ping_mdh, ni_bytes, true);
3100         if (rc != 0)
3101                 goto err_acceptor_stop;
3102
3103         lnet_ping_target_update(pbuf, ping_mdh);
3104
3105         the_lnet.ln_mt_handler = lnet_mt_event_handler;
3106
3107         rc = lnet_push_target_init();
3108         if (rc != 0)
3109                 goto err_stop_ping;
3110
3111         rc = lnet_monitor_thr_start();
3112         if (rc != 0)
3113                 goto err_destroy_push_target;
3114
3115         rc = lnet_peer_discovery_start();
3116         if (rc != 0)
3117                 goto err_stop_monitor_thr;
3118
3119         lnet_fault_init();
3120         lnet_router_debugfs_init();
3121
3122         mutex_unlock(&the_lnet.ln_api_mutex);
3123
3124         complete_all(&the_lnet.ln_started);
3125
3126         /* wait for all routers to start */
3127         lnet_wait_router_start();
3128
3129         return 0;
3130
3131 err_stop_monitor_thr:
3132         lnet_monitor_thr_stop();
3133 err_destroy_push_target:
3134         lnet_push_target_fini();
3135 err_stop_ping:
3136         lnet_ping_target_fini();
3137 err_acceptor_stop:
3138         the_lnet.ln_refcount = 0;
3139         lnet_acceptor_stop();
3140 err_destroy_routes:
3141         if (!the_lnet.ln_nis_from_mod_params)
3142                 lnet_destroy_routes();
3143 err_shutdown_lndnis:
3144         lnet_shutdown_lndnets();
3145 err_empty_list:
3146         lnet_unprepare();
3147         LASSERT(rc < 0);
3148         mutex_unlock(&the_lnet.ln_api_mutex);
3149         while ((net = list_first_entry_or_null(&net_head,
3150                                                struct lnet_net,
3151                                                net_list)) != NULL) {
3152                 list_del_init(&net->net_list);
3153                 lnet_net_free(net);
3154         }
3155         return rc;
3156 }
3157 EXPORT_SYMBOL(LNetNIInit);
3158
3159 /**
3160  * Stop LNet interfaces, routing, and forwarding.
3161  *
3162  * Users must call this function once for each successful call to LNetNIInit().
3163  * Once the LNetNIFini() operation has been started, the results of pending
3164  * API operations are undefined.
3165  *
3166  * \return always 0 for current implementation.
3167  */
3168 int
3169 LNetNIFini(void)
3170 {
3171         mutex_lock(&the_lnet.ln_api_mutex);
3172
3173         LASSERT(the_lnet.ln_refcount > 0);
3174
3175         if (the_lnet.ln_refcount != 1) {
3176                 the_lnet.ln_refcount--;
3177         } else {
3178                 LASSERT(!the_lnet.ln_niinit_self);
3179
3180                 lnet_net_lock(LNET_LOCK_EX);
3181                 the_lnet.ln_state = LNET_STATE_STOPPING;
3182                 lnet_net_unlock(LNET_LOCK_EX);
3183
3184                 lnet_fault_fini();
3185
3186                 lnet_router_debugfs_fini();
3187                 lnet_peer_discovery_stop();
3188                 lnet_monitor_thr_stop();
3189                 lnet_push_target_fini();
3190                 lnet_ping_target_fini();
3191
3192                 /* Teardown fns that use my own API functions BEFORE here */
3193                 the_lnet.ln_refcount = 0;
3194
3195                 lnet_acceptor_stop();
3196                 lnet_destroy_routes();
3197                 lnet_shutdown_lndnets();
3198                 lnet_unprepare();
3199         }
3200
3201         mutex_unlock(&the_lnet.ln_api_mutex);
3202         return 0;
3203 }
3204 EXPORT_SYMBOL(LNetNIFini);
3205
3206 /**
3207  * Grabs the ni data from the ni structure and fills the out
3208  * parameters
3209  *
3210  * \param[in] ni network        interface structure
3211  * \param[out] cfg_ni           NI config information
3212  * \param[out] tun              network and LND tunables
3213  */
3214 static void
3215 lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_ni *cfg_ni,
3216                    struct lnet_ioctl_config_lnd_tunables *tun,
3217                    struct lnet_ioctl_element_stats *stats,
3218                    __u32 tun_size)
3219 {
3220         size_t min_size = 0;
3221         int i;
3222
3223         if (!ni || !cfg_ni || !tun || !nid_is_nid4(&ni->ni_nid))
3224                 return;
3225
3226         if (ni->ni_interface != NULL) {
3227                 strncpy(cfg_ni->lic_ni_intf,
3228                         ni->ni_interface,
3229                         sizeof(cfg_ni->lic_ni_intf));
3230         }
3231
3232         cfg_ni->lic_nid = lnet_nid_to_nid4(&ni->ni_nid);
3233         cfg_ni->lic_status = lnet_ni_get_status_locked(ni);
3234         cfg_ni->lic_dev_cpt = ni->ni_dev_cpt;
3235
3236         memcpy(&tun->lt_cmn, &ni->ni_net->net_tunables, sizeof(tun->lt_cmn));
3237
3238         if (stats) {
3239                 stats->iel_send_count = lnet_sum_stats(&ni->ni_stats,
3240                                                        LNET_STATS_TYPE_SEND);
3241                 stats->iel_recv_count = lnet_sum_stats(&ni->ni_stats,
3242                                                        LNET_STATS_TYPE_RECV);
3243                 stats->iel_drop_count = lnet_sum_stats(&ni->ni_stats,
3244                                                        LNET_STATS_TYPE_DROP);
3245         }
3246
3247         /*
3248          * tun->lt_tun will always be present, but in order to be
3249          * backwards compatible, we need to deal with the cases when
3250          * tun->lt_tun is smaller than what the kernel has, because it
3251          * comes from an older version of a userspace program, then we'll
3252          * need to copy as much information as we have available space.
3253          */
3254         min_size = tun_size - sizeof(tun->lt_cmn);
3255         memcpy(&tun->lt_tun, &ni->ni_lnd_tunables, min_size);
3256
3257         /* copy over the cpts */
3258         if (ni->ni_ncpts == LNET_CPT_NUMBER &&
3259             ni->ni_cpts == NULL)  {
3260                 for (i = 0; i < ni->ni_ncpts; i++)
3261                         cfg_ni->lic_cpts[i] = i;
3262         } else {
3263                 for (i = 0;
3264                      ni->ni_cpts != NULL && i < ni->ni_ncpts &&
3265                      i < LNET_MAX_SHOW_NUM_CPT;
3266                      i++)
3267                         cfg_ni->lic_cpts[i] = ni->ni_cpts[i];
3268         }
3269         cfg_ni->lic_ncpts = ni->ni_ncpts;
3270 }
3271
3272 /**
3273  * NOTE: This is a legacy function left in the code to be backwards
3274  * compatible with older userspace programs. It should eventually be
3275  * removed.
3276  *
3277  * Grabs the ni data from the ni structure and fills the out
3278  * parameters
3279  *
3280  * \param[in] ni network        interface structure
3281  * \param[out] config           config information
3282  */
3283 static void
3284 lnet_fill_ni_info_legacy(struct lnet_ni *ni,
3285                          struct lnet_ioctl_config_data *config)
3286 {
3287         struct lnet_ioctl_net_config *net_config;
3288         struct lnet_ioctl_config_lnd_tunables *lnd_cfg = NULL;
3289         size_t min_size, tunable_size = 0;
3290         int i;
3291
3292         if (!ni || !config || !nid_is_nid4(&ni->ni_nid))
3293                 return;
3294
3295         net_config = (struct lnet_ioctl_net_config *) config->cfg_bulk;
3296         if (!net_config)
3297                 return;
3298
3299         if (!ni->ni_interface)
3300                 return;
3301
3302         strncpy(net_config->ni_interface,
3303                 ni->ni_interface,
3304                 sizeof(net_config->ni_interface));
3305
3306         config->cfg_nid = lnet_nid_to_nid4(&ni->ni_nid);
3307         config->cfg_config_u.cfg_net.net_peer_timeout =
3308                 ni->ni_net->net_tunables.lct_peer_timeout;
3309         config->cfg_config_u.cfg_net.net_max_tx_credits =
3310                 ni->ni_net->net_tunables.lct_max_tx_credits;
3311         config->cfg_config_u.cfg_net.net_peer_tx_credits =
3312                 ni->ni_net->net_tunables.lct_peer_tx_credits;
3313         config->cfg_config_u.cfg_net.net_peer_rtr_credits =
3314                 ni->ni_net->net_tunables.lct_peer_rtr_credits;
3315
3316         net_config->ni_status = lnet_ni_get_status_locked(ni);
3317
3318         if (ni->ni_cpts) {
3319                 int num_cpts = min(ni->ni_ncpts, LNET_MAX_SHOW_NUM_CPT);
3320
3321                 for (i = 0; i < num_cpts; i++)
3322                         net_config->ni_cpts[i] = ni->ni_cpts[i];
3323
3324                 config->cfg_ncpts = num_cpts;
3325         }
3326
3327         /*
3328          * See if user land tools sent in a newer and larger version
3329          * of struct lnet_tunables than what the kernel uses.
3330          */
3331         min_size = sizeof(*config) + sizeof(*net_config);
3332
3333         if (config->cfg_hdr.ioc_len > min_size)
3334                 tunable_size = config->cfg_hdr.ioc_len - min_size;
3335
3336         /* Don't copy too much data to user space */
3337         min_size = min(tunable_size, sizeof(ni->ni_lnd_tunables));
3338         lnd_cfg = (struct lnet_ioctl_config_lnd_tunables *)net_config->cfg_bulk;
3339
3340         if (lnd_cfg && min_size) {
3341                 memcpy(&lnd_cfg->lt_tun, &ni->ni_lnd_tunables, min_size);
3342                 config->cfg_config_u.cfg_net.net_interface_count = 1;
3343
3344                 /* Tell user land that kernel side has less data */
3345                 if (tunable_size > sizeof(ni->ni_lnd_tunables)) {
3346                         min_size = tunable_size - sizeof(ni->ni_lnd_tunables);
3347                         config->cfg_hdr.ioc_len -= min_size;
3348                 }
3349         }
3350 }
3351
3352 struct lnet_ni *
3353 lnet_get_ni_idx_locked(int idx)
3354 {
3355         struct lnet_ni          *ni;
3356         struct lnet_net         *net;
3357
3358         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
3359                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
3360                         if (idx-- == 0)
3361                                 return ni;
3362                 }
3363         }
3364
3365         return NULL;
3366 }
3367
3368 int lnet_get_net_healthv_locked(struct lnet_net *net)
3369 {
3370         struct lnet_ni *ni;
3371         int best_healthv = 0;
3372         int healthv, ni_fatal;
3373
3374         list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
3375                 healthv = atomic_read(&ni->ni_healthv);
3376                 ni_fatal = atomic_read(&ni->ni_fatal_error_on);
3377                 if (!ni_fatal && healthv > best_healthv)
3378                         best_healthv = healthv;
3379         }
3380
3381         return best_healthv;
3382 }
3383
3384 struct lnet_ni *
3385 lnet_get_next_ni_locked(struct lnet_net *mynet, struct lnet_ni *prev)
3386 {
3387         struct lnet_ni          *ni;
3388         struct lnet_net         *net = mynet;
3389
3390         /*
3391          * It is possible that the net has been cleaned out while there is
3392          * a message being sent. This function accessed the net without
3393          * checking if the list is empty
3394          */
3395         if (!prev) {
3396                 if (!net)
3397                         net = list_first_entry(&the_lnet.ln_nets,
3398                                                struct lnet_net,
3399                                                net_list);
3400                 if (list_empty(&net->net_ni_list))
3401                         return NULL;
3402                 ni = list_first_entry(&net->net_ni_list, struct lnet_ni,
3403                                       ni_netlist);
3404
3405                 return ni;
3406         }
3407
3408         if (prev->ni_netlist.next == &prev->ni_net->net_ni_list) {
3409                 /* if you reached the end of the ni list and the net is
3410                  * specified, then there are no more nis in that net */
3411                 if (net != NULL)
3412                         return NULL;
3413
3414                 /* we reached the end of this net ni list. move to the
3415                  * next net */
3416                 if (prev->ni_net->net_list.next == &the_lnet.ln_nets)
3417                         /* no more nets and no more NIs. */
3418                         return NULL;
3419
3420                 /* get the next net */
3421                 net = list_first_entry(&prev->ni_net->net_list, struct lnet_net,
3422                                        net_list);
3423                 if (list_empty(&net->net_ni_list))
3424                         return NULL;
3425                 /* get the ni on it */
3426                 ni = list_first_entry(&net->net_ni_list, struct lnet_ni,
3427                                       ni_netlist);
3428
3429                 return ni;
3430         }
3431
3432         if (list_empty(&prev->ni_netlist))
3433                 return NULL;
3434
3435         /* there are more nis left */
3436         ni = list_first_entry(&prev->ni_netlist, struct lnet_ni, ni_netlist);
3437
3438         return ni;
3439 }
3440
3441 static int
3442 lnet_get_net_config(struct lnet_ioctl_config_data *config)
3443 {
3444         struct lnet_ni *ni;
3445         int cpt;
3446         int rc = -ENOENT;
3447         int idx = config->cfg_count;
3448
3449         cpt = lnet_net_lock_current();
3450
3451         ni = lnet_get_ni_idx_locked(idx);
3452
3453         if (ni != NULL) {
3454                 rc = 0;
3455                 lnet_ni_lock(ni);
3456                 lnet_fill_ni_info_legacy(ni, config);
3457                 lnet_ni_unlock(ni);
3458         }
3459
3460         lnet_net_unlock(cpt);
3461         return rc;
3462 }
3463
3464 static int
3465 lnet_get_ni_config(struct lnet_ioctl_config_ni *cfg_ni,
3466                    struct lnet_ioctl_config_lnd_tunables *tun,
3467                    struct lnet_ioctl_element_stats *stats,
3468                    __u32 tun_size)
3469 {
3470         struct lnet_ni          *ni;
3471         int                     cpt;
3472         int                     rc = -ENOENT;
3473
3474         if (!cfg_ni || !tun || !stats)
3475                 return -EINVAL;
3476
3477         cpt = lnet_net_lock_current();
3478
3479         ni = lnet_get_ni_idx_locked(cfg_ni->lic_idx);
3480
3481         if (ni) {
3482                 rc = 0;
3483                 lnet_ni_lock(ni);
3484                 lnet_fill_ni_info(ni, cfg_ni, tun, stats, tun_size);
3485                 lnet_ni_unlock(ni);
3486         }
3487
3488         lnet_net_unlock(cpt);
3489         return rc;
3490 }
3491
3492 static int lnet_get_ni_stats(struct lnet_ioctl_element_msg_stats *msg_stats)
3493 {
3494         struct lnet_ni *ni;
3495         int rc = -ENOENT;
3496
3497         if (!msg_stats)
3498                 return -EINVAL;
3499
3500         ni = lnet_get_ni_idx_locked(msg_stats->im_idx);
3501
3502         if (ni) {
3503                 lnet_usr_translate_stats(msg_stats, &ni->ni_stats);
3504                 rc = 0;
3505         }
3506
3507         return rc;
3508 }
3509
3510 static int lnet_add_net_common(struct lnet_net *net,
3511                                struct lnet_ioctl_config_lnd_tunables *tun)
3512 {
3513         struct lnet_handle_md ping_mdh;
3514         struct lnet_ping_buffer *pbuf;
3515         struct lnet_remotenet *rnet;
3516         struct lnet_ni *ni;
3517         u32 net_id;
3518         int rc;
3519
3520         lnet_net_lock(LNET_LOCK_EX);
3521         rnet = lnet_find_rnet_locked(net->net_id);
3522         lnet_net_unlock(LNET_LOCK_EX);
3523         /*
3524          * make sure that the net added doesn't invalidate the current
3525          * configuration LNet is keeping
3526          */
3527         if (rnet) {
3528                 CERROR("Adding net %s will invalidate routing configuration\n",
3529                        libcfs_net2str(net->net_id));
3530                 lnet_net_free(net);
3531                 return -EUSERS;
3532         }
3533
3534         if (tun)
3535                 memcpy(&net->net_tunables,
3536                        &tun->lt_cmn, sizeof(net->net_tunables));
3537         else
3538                 memset(&net->net_tunables, -1, sizeof(net->net_tunables));
3539
3540         net_id = net->net_id;
3541
3542         rc = lnet_startup_lndnet(net,
3543                                  (tun) ? &tun->lt_tun : NULL);
3544         if (rc < 0)
3545                 return rc;
3546
3547         /* make sure you calculate the correct number of slots in the ping
3548          * buffer. Since the ping info is a flattened list of all the NIs,
3549          * we should allocate enough slots to accomodate the number of NIs
3550          * which will be added.
3551          */
3552         rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
3553                                     LNET_PING_INFO_HDR_SIZE +
3554                                     lnet_get_ni_bytes(),
3555                                     false);
3556         if (rc < 0) {
3557                 lnet_shutdown_lndnet(net);
3558                 return rc;
3559         }
3560
3561         lnet_net_lock(LNET_LOCK_EX);
3562         net = lnet_get_net_locked(net_id);
3563         LASSERT(net);
3564
3565         /* apply the UDSPs */
3566         rc = lnet_udsp_apply_policies_on_net(net);
3567         if (rc)
3568                 CERROR("Failed to apply UDSPs on local net %s\n",
3569                        libcfs_net2str(net->net_id));
3570
3571         /* At this point we lost track of which NI was just added, so we
3572          * just re-apply the policies on all of the NIs on this net
3573          */
3574         list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
3575                 rc = lnet_udsp_apply_policies_on_ni(ni);
3576                 if (rc)
3577                         CERROR("Failed to apply UDSPs on ni %s\n",
3578                                libcfs_nidstr(&ni->ni_nid));
3579         }
3580         lnet_net_unlock(LNET_LOCK_EX);
3581
3582         /*
3583          * Start the acceptor thread if this is the first network
3584          * being added that requires the thread.
3585          */
3586         if (net->net_lnd->lnd_accept) {
3587                 rc = lnet_acceptor_start();
3588                 if (rc < 0) {
3589                         /* shutdown the net that we just started */
3590                         CERROR("Failed to start up acceptor thread\n");
3591                         lnet_shutdown_lndnet(net);
3592                         goto failed;
3593                 }
3594         }
3595
3596         lnet_net_lock(LNET_LOCK_EX);
3597         lnet_peer_net_added(net);
3598         lnet_net_unlock(LNET_LOCK_EX);
3599
3600         lnet_ping_target_update(pbuf, ping_mdh);
3601
3602         return 0;
3603
3604 failed:
3605         lnet_ping_md_unlink(pbuf, &ping_mdh);
3606         lnet_ping_buffer_decref(pbuf);
3607         return rc;
3608 }
3609
3610 static void
3611 lnet_set_tune_defaults(struct lnet_ioctl_config_lnd_tunables *tun)
3612 {
3613         if (tun) {
3614                 if (tun->lt_cmn.lct_peer_timeout < 0)
3615                         tun->lt_cmn.lct_peer_timeout = DEFAULT_PEER_TIMEOUT;
3616                 if (!tun->lt_cmn.lct_peer_tx_credits)
3617                         tun->lt_cmn.lct_peer_tx_credits = DEFAULT_PEER_CREDITS;
3618                 if (!tun->lt_cmn.lct_max_tx_credits)
3619                         tun->lt_cmn.lct_max_tx_credits = DEFAULT_CREDITS;
3620         }
3621 }
3622
3623 static int lnet_handle_legacy_ip2nets(char *ip2nets,
3624                                       struct lnet_ioctl_config_lnd_tunables *tun)
3625 {
3626         struct lnet_net *net;
3627         const char *nets;
3628         int rc;
3629         LIST_HEAD(net_head);
3630
3631         rc = lnet_parse_ip2nets(&nets, ip2nets);
3632         if (rc < 0)
3633                 return rc;
3634
3635         rc = lnet_parse_networks(&net_head, nets);
3636         if (rc < 0)
3637                 return rc;
3638
3639         lnet_set_tune_defaults(tun);
3640
3641         mutex_lock(&the_lnet.ln_api_mutex);
3642         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
3643                 rc = -ESHUTDOWN;
3644                 goto out;
3645         }
3646
3647         while ((net = list_first_entry_or_null(&net_head,
3648                                                struct lnet_net,
3649                                                net_list)) != NULL) {
3650                 list_del_init(&net->net_list);
3651                 rc = lnet_add_net_common(net, tun);
3652                 if (rc < 0)
3653                         goto out;
3654         }
3655
3656 out:
3657         mutex_unlock(&the_lnet.ln_api_mutex);
3658
3659         while ((net = list_first_entry_or_null(&net_head,
3660                                                struct lnet_net,
3661                                                net_list)) != NULL) {
3662                 list_del_init(&net->net_list);
3663                 lnet_net_free(net);
3664         }
3665         return rc;
3666 }
3667
3668 int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net_id,
3669                     struct lnet_nid *nid,
3670                     struct lnet_ioctl_config_lnd_tunables *tun)
3671 {
3672         struct lnet_net *net;
3673         struct lnet_ni *ni;
3674         int rc, i;
3675         u32 lnd_type;
3676
3677         /* handle legacy ip2nets from DLC */
3678         if (conf->lic_legacy_ip2nets[0] != '\0')
3679                 return lnet_handle_legacy_ip2nets(conf->lic_legacy_ip2nets,
3680                                                   tun);
3681
3682         lnd_type = LNET_NETTYP(net_id);
3683
3684         if (!libcfs_isknown_lnd(lnd_type)) {
3685                 CERROR("No valid net and lnd information provided\n");
3686                 return -ENOENT;
3687         }
3688
3689         net = lnet_net_alloc(net_id, NULL);
3690         if (!net)
3691                 return -ENOMEM;
3692
3693         for (i = 0; i < conf->lic_ncpts; i++) {
3694                 if (conf->lic_cpts[i] >= LNET_CPT_NUMBER) {
3695                         lnet_net_free(net);
3696                         return -ERANGE;
3697                 }
3698         }
3699
3700         ni = lnet_ni_alloc_w_cpt_array(net, nid, conf->lic_cpts,
3701                                        conf->lic_ncpts, conf->lic_ni_intf);
3702         if (!ni) {
3703                 lnet_net_free(net);
3704                 return -ENOMEM;
3705         }
3706
3707         lnet_set_tune_defaults(tun);
3708
3709         mutex_lock(&the_lnet.ln_api_mutex);
3710         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
3711                 lnet_net_free(net);
3712                 rc = -ESHUTDOWN;
3713         } else {
3714                 rc = lnet_add_net_common(net, tun);
3715         }
3716
3717         mutex_unlock(&the_lnet.ln_api_mutex);
3718
3719         /* If NI already exist delete this new unused copy */
3720         if (rc == -EEXIST)
3721                 lnet_ni_free(ni);
3722
3723         return rc;
3724 }
3725
3726 int lnet_dyn_del_ni(struct lnet_nid *nid)
3727 {
3728         struct lnet_net *net;
3729         struct lnet_ni *ni;
3730         u32 net_id = LNET_NID_NET(nid);
3731         struct lnet_ping_buffer *pbuf;
3732         struct lnet_handle_md ping_mdh;
3733         int net_bytes, rc;
3734         bool net_empty;
3735
3736         /* don't allow userspace to shutdown the LOLND */
3737         if (LNET_NETTYP(net_id) == LOLND)
3738                 return -EINVAL;
3739
3740         mutex_lock(&the_lnet.ln_api_mutex);
3741         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
3742                 rc = -ESHUTDOWN;
3743                 goto unlock_api_mutex;
3744         }
3745
3746         lnet_net_lock(0);
3747
3748         net = lnet_get_net_locked(net_id);
3749         if (!net) {
3750                 CERROR("net %s not found\n",
3751                        libcfs_net2str(net_id));
3752                 rc = -ENOENT;
3753                 goto unlock_net;
3754         }
3755
3756         if (!nid_addr_is_set(nid)) {
3757                 /* remove the entire net */
3758                 net_bytes = lnet_get_net_ni_bytes_locked(net);
3759
3760                 lnet_net_unlock(0);
3761
3762                 /* create and link a new ping info, before removing the old one */
3763                 rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
3764                                             LNET_PING_INFO_HDR_SIZE +
3765                                             lnet_get_ni_bytes() - net_bytes,
3766                                             false);
3767                 if (rc != 0)
3768                         goto unlock_api_mutex;
3769
3770                 lnet_shutdown_lndnet(net);
3771
3772                 lnet_acceptor_stop();
3773
3774                 lnet_ping_target_update(pbuf, ping_mdh);
3775
3776                 goto unlock_api_mutex;
3777         }
3778
3779         ni = lnet_nid_to_ni_locked(nid, 0);
3780         if (!ni) {
3781                 CERROR("nid %s not found\n", libcfs_nidstr(nid));
3782                 rc = -ENOENT;
3783                 goto unlock_net;
3784         }
3785
3786         net_bytes = lnet_get_net_ni_bytes_locked(net);
3787         net_empty = list_is_singular(&net->net_ni_list);
3788
3789         lnet_net_unlock(0);
3790
3791         /* create and link a new ping info, before removing the old one */
3792         rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
3793                                     (LNET_PING_INFO_HDR_SIZE +
3794                                      lnet_get_ni_bytes() -
3795                                      lnet_ping_sts_size(&ni->ni_nid)),
3796                                     false);
3797         if (rc != 0)
3798                 goto unlock_api_mutex;
3799
3800         lnet_shutdown_lndni(ni);
3801
3802         lnet_acceptor_stop();
3803
3804         lnet_ping_target_update(pbuf, ping_mdh);
3805
3806         /* check if the net is empty and remove it if it is */
3807         if (net_empty)
3808                 lnet_shutdown_lndnet(net);
3809
3810         goto unlock_api_mutex;
3811
3812 unlock_net:
3813         lnet_net_unlock(0);
3814 unlock_api_mutex:
3815         mutex_unlock(&the_lnet.ln_api_mutex);
3816
3817         return rc;
3818 }
3819
3820 /*
3821  * lnet_dyn_add_net and lnet_dyn_del_net are now deprecated.
3822  * They are only expected to be called for unique networks.
3823  * That can be as a result of older DLC library
3824  * calls. Multi-Rail DLC and beyond no longer uses these APIs.
3825  */
3826 int
3827 lnet_dyn_add_net(struct lnet_ioctl_config_data *conf)
3828 {
3829         struct lnet_net *net;
3830         LIST_HEAD(net_head);
3831         int rc;
3832         struct lnet_ioctl_config_lnd_tunables tun;
3833         const char *nets = conf->cfg_config_u.cfg_net.net_intf;
3834
3835         /* Create a net/ni structures for the network string */
3836         rc = lnet_parse_networks(&net_head, nets);
3837         if (rc <= 0)
3838                 return rc == 0 ? -EINVAL : rc;
3839
3840         mutex_lock(&the_lnet.ln_api_mutex);
3841         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
3842                 rc = -ESHUTDOWN;
3843                 goto out_unlock_clean;
3844         }
3845
3846         if (rc > 1) {
3847                 rc = -EINVAL; /* only add one network per call */
3848                 goto out_unlock_clean;
3849         }
3850
3851         net = list_first_entry(&net_head, struct lnet_net, net_list);
3852         list_del_init(&net->net_list);
3853
3854         LASSERT(lnet_net_unique(net->net_id, &the_lnet.ln_nets, NULL));
3855
3856         memset(&tun, 0, sizeof(tun));
3857
3858         tun.lt_cmn.lct_peer_timeout =
3859           (!conf->cfg_config_u.cfg_net.net_peer_timeout) ? DEFAULT_PEER_TIMEOUT :
3860                 conf->cfg_config_u.cfg_net.net_peer_timeout;
3861         tun.lt_cmn.lct_peer_tx_credits =
3862           (!conf->cfg_config_u.cfg_net.net_peer_tx_credits) ? DEFAULT_PEER_CREDITS :
3863                 conf->cfg_config_u.cfg_net.net_peer_tx_credits;
3864         tun.lt_cmn.lct_peer_rtr_credits =
3865           conf->cfg_config_u.cfg_net.net_peer_rtr_credits;
3866         tun.lt_cmn.lct_max_tx_credits =
3867           (!conf->cfg_config_u.cfg_net.net_max_tx_credits) ? DEFAULT_CREDITS :
3868                 conf->cfg_config_u.cfg_net.net_max_tx_credits;
3869
3870         rc = lnet_add_net_common(net, &tun);
3871
3872 out_unlock_clean:
3873         mutex_unlock(&the_lnet.ln_api_mutex);
3874         /* net_head list is empty in success case */
3875         while ((net = list_first_entry_or_null(&net_head,
3876                                                struct lnet_net,
3877                                                net_list)) != NULL) {
3878                 list_del_init(&net->net_list);
3879                 lnet_net_free(net);
3880         }
3881         return rc;
3882 }
3883
3884 int
3885 lnet_dyn_del_net(u32 net_id)
3886 {
3887         struct lnet_net *net;
3888         struct lnet_ping_buffer *pbuf;
3889         struct lnet_handle_md ping_mdh;
3890         int net_ni_bytes, rc;
3891
3892         /* don't allow userspace to shutdown the LOLND */
3893         if (LNET_NETTYP(net_id) == LOLND)
3894                 return -EINVAL;
3895
3896         mutex_lock(&the_lnet.ln_api_mutex);
3897         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
3898                 rc = -ESHUTDOWN;
3899                 goto out;
3900         }
3901
3902         lnet_net_lock(0);
3903
3904         net = lnet_get_net_locked(net_id);
3905         if (net == NULL) {
3906                 lnet_net_unlock(0);
3907                 rc = -EINVAL;
3908                 goto out;
3909         }
3910
3911         net_ni_bytes = lnet_get_net_ni_bytes_locked(net);
3912
3913         lnet_net_unlock(0);
3914
3915         /* create and link a new ping info, before removing the old one */
3916         rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
3917                                     LNET_PING_INFO_HDR_SIZE +
3918                                     lnet_get_ni_bytes() - net_ni_bytes,
3919                                     false);
3920         if (rc != 0)
3921                 goto out;
3922
3923         lnet_shutdown_lndnet(net);
3924
3925         lnet_acceptor_stop();
3926
3927         lnet_ping_target_update(pbuf, ping_mdh);
3928
3929 out:
3930         mutex_unlock(&the_lnet.ln_api_mutex);
3931
3932         return rc;
3933 }
3934
3935 void lnet_mark_ping_buffer_for_update(void)
3936 {
3937         if (the_lnet.ln_routing)
3938                 return;
3939
3940         atomic_set(&the_lnet.ln_update_ping_buf, 1);
3941         complete(&the_lnet.ln_mt_wait_complete);
3942 }
3943 EXPORT_SYMBOL(lnet_mark_ping_buffer_for_update);
3944
3945 void lnet_update_ping_buffer(struct work_struct *work)
3946 {
3947         struct lnet_ping_buffer *pbuf;
3948         struct lnet_handle_md ping_mdh;
3949
3950         mutex_lock(&the_lnet.ln_api_mutex);
3951
3952         atomic_set(&the_lnet.ln_pb_update_ready, 1);
3953
3954         if ((the_lnet.ln_state == LNET_STATE_RUNNING) &&
3955             !lnet_ping_target_setup(&pbuf, &ping_mdh,
3956                                     LNET_PING_INFO_HDR_SIZE +
3957                                     lnet_get_ni_bytes(),
3958                                     false))
3959                 lnet_ping_target_update(pbuf, ping_mdh);
3960
3961
3962         mutex_unlock(&the_lnet.ln_api_mutex);
3963 }
3964
3965
3966 void lnet_queue_ping_buffer_update(void)
3967 {
3968         /* don't queue pb update if it is not needed */
3969         if (atomic_dec_if_positive(&the_lnet.ln_update_ping_buf) < 0)
3970                 return;
3971
3972         /* don't queue pb update if already queued and not processed */
3973         if (atomic_dec_if_positive(&the_lnet.ln_pb_update_ready) < 0)
3974                 return;
3975
3976         INIT_WORK(&the_lnet.ln_pb_update_work, lnet_update_ping_buffer);
3977         queue_work(the_lnet.ln_pb_update_wq, &the_lnet.ln_pb_update_work);
3978 }
3979
3980 void lnet_incr_dlc_seq(void)
3981 {
3982         atomic_inc(&lnet_dlc_seq_no);
3983 }
3984
3985 __u32 lnet_get_dlc_seq_locked(void)
3986 {
3987         return atomic_read(&lnet_dlc_seq_no);
3988 }
3989
3990 static void
3991 lnet_ni_set_healthv(struct lnet_nid *nid, int value)
3992 {
3993         bool all = nid_same(nid, &LNET_ANY_NID);
3994         struct lnet_net *net;
3995         struct lnet_ni *ni;
3996
3997         lnet_net_lock(LNET_LOCK_EX);
3998         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
3999                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
4000                         if (!all && !nid_same(&ni->ni_nid, nid))
4001                                 continue;
4002
4003                         atomic_set(&ni->ni_healthv, value);
4004                         if (list_empty(&ni->ni_recovery) &&
4005                             value < LNET_MAX_HEALTH_VALUE) {
4006                                 CERROR("manually adding local NI %s to recovery\n",
4007                                        libcfs_nidstr(&ni->ni_nid));
4008                                 list_add_tail(&ni->ni_recovery,
4009                                               &the_lnet.ln_mt_localNIRecovq);
4010                                 lnet_ni_addref_locked(ni, 0);
4011                         }
4012                         if (!all) {
4013                                 lnet_net_unlock(LNET_LOCK_EX);
4014                                 return;
4015                         }
4016                 }
4017         }
4018         lnet_net_unlock(LNET_LOCK_EX);
4019 }
4020
4021 static void
4022 lnet_ni_set_conns_per_peer(lnet_nid_t nid, int value, bool all)
4023 {
4024         struct lnet_net *net;
4025         struct lnet_ni *ni;
4026
4027         lnet_net_lock(LNET_LOCK_EX);
4028         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
4029                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
4030                         if (lnet_nid_to_nid4(&ni->ni_nid) != nid && !all)
4031                                 continue;
4032                         if (LNET_NETTYP(net->net_id) == SOCKLND)
4033                                 ni->ni_lnd_tunables.lnd_tun_u.lnd_sock.lnd_conns_per_peer = value;
4034                         else if (LNET_NETTYP(net->net_id) == O2IBLND)
4035                                 ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer = value;
4036                         if (!all) {
4037                                 lnet_net_unlock(LNET_LOCK_EX);
4038                                 return;
4039                         }
4040                 }
4041         }
4042         lnet_net_unlock(LNET_LOCK_EX);
4043 }
4044
4045 static int
4046 lnet_get_local_ni_hstats(struct lnet_ioctl_local_ni_hstats *stats)
4047 {
4048         int cpt, rc = 0;
4049         struct lnet_ni *ni;
4050         struct lnet_nid nid;
4051
4052         lnet_nid4_to_nid(stats->hlni_nid, &nid);
4053         cpt = lnet_net_lock_current();
4054         ni = lnet_nid_to_ni_locked(&nid, cpt);
4055         if (!ni) {
4056                 rc = -ENOENT;
4057                 goto unlock;
4058         }
4059
4060         stats->hlni_local_interrupt = atomic_read(&ni->ni_hstats.hlt_local_interrupt);
4061         stats->hlni_local_dropped = atomic_read(&ni->ni_hstats.hlt_local_dropped);
4062         stats->hlni_local_aborted = atomic_read(&ni->ni_hstats.hlt_local_aborted);
4063         stats->hlni_local_no_route = atomic_read(&ni->ni_hstats.hlt_local_no_route);
4064         stats->hlni_local_timeout = atomic_read(&ni->ni_hstats.hlt_local_timeout);
4065         stats->hlni_local_error = atomic_read(&ni->ni_hstats.hlt_local_error);
4066         stats->hlni_fatal_error = atomic_read(&ni->ni_fatal_error_on);
4067         stats->hlni_health_value = atomic_read(&ni->ni_healthv);
4068         stats->hlni_ping_count = ni->ni_ping_count;
4069         stats->hlni_next_ping = ni->ni_next_ping;
4070
4071 unlock:
4072         lnet_net_unlock(cpt);
4073
4074         return rc;
4075 }
4076
4077 static int
4078 lnet_get_local_ni_recovery_list(struct lnet_ioctl_recovery_list *list)
4079 {
4080         struct lnet_ni *ni;
4081         int i = 0;
4082
4083         lnet_net_lock(LNET_LOCK_EX);
4084         list_for_each_entry(ni, &the_lnet.ln_mt_localNIRecovq, ni_recovery) {
4085                 if (!nid_is_nid4(&ni->ni_nid))
4086                         continue;
4087                 list->rlst_nid_array[i] = lnet_nid_to_nid4(&ni->ni_nid);
4088                 i++;
4089                 if (i >= LNET_MAX_SHOW_NUM_NID)
4090                         break;
4091         }
4092         lnet_net_unlock(LNET_LOCK_EX);
4093         list->rlst_num_nids = i;
4094
4095         return 0;
4096 }
4097
4098 static int
4099 lnet_get_peer_ni_recovery_list(struct lnet_ioctl_recovery_list *list)
4100 {
4101         struct lnet_peer_ni *lpni;
4102         int i = 0;
4103
4104         lnet_net_lock(LNET_LOCK_EX);
4105         list_for_each_entry(lpni, &the_lnet.ln_mt_peerNIRecovq, lpni_recovery) {
4106                 list->rlst_nid_array[i] = lnet_nid_to_nid4(&lpni->lpni_nid);
4107                 i++;
4108                 if (i >= LNET_MAX_SHOW_NUM_NID)
4109                         break;
4110         }
4111         lnet_net_unlock(LNET_LOCK_EX);
4112         list->rlst_num_nids = i;
4113
4114         return 0;
4115 }
4116
4117 /**
4118  * LNet ioctl handler.
4119  *
4120  */
4121 int
4122 LNetCtl(unsigned int cmd, void *arg)
4123 {
4124         struct libcfs_ioctl_data *data = arg;
4125         struct lnet_ioctl_config_data *config;
4126         struct lnet_ni           *ni;
4127         struct lnet_nid           nid;
4128         int                       rc;
4129
4130         BUILD_BUG_ON(sizeof(struct lnet_ioctl_net_config) +
4131                      sizeof(struct lnet_ioctl_config_data) > LIBCFS_IOC_DATA_MAX);
4132
4133         switch (cmd) {
4134         case IOC_LIBCFS_GET_NI: {
4135                 struct lnet_processid id = {};
4136
4137                 rc = LNetGetId(data->ioc_count, &id, false);
4138                 data->ioc_nid = lnet_nid_to_nid4(&id.nid);
4139                 return rc;
4140         }
4141         case IOC_LIBCFS_FAIL_NID:
4142                 return lnet_fail_nid(data->ioc_nid, data->ioc_count);
4143
4144         case IOC_LIBCFS_ADD_ROUTE: {
4145                 /* default router sensitivity to 1 */
4146                 unsigned int sensitivity = 1;
4147                 config = arg;
4148
4149                 if (config->cfg_hdr.ioc_len < sizeof(*config))
4150                         return -EINVAL;
4151
4152                 if (config->cfg_config_u.cfg_route.rtr_sensitivity) {
4153                         sensitivity =
4154                           config->cfg_config_u.cfg_route.rtr_sensitivity;
4155                 }
4156
4157                 lnet_nid4_to_nid(config->cfg_nid, &nid);
4158                 mutex_lock(&the_lnet.ln_api_mutex);
4159                 rc = lnet_add_route(config->cfg_net,
4160                                     config->cfg_config_u.cfg_route.rtr_hop,
4161                                     &nid,
4162                                     config->cfg_config_u.cfg_route.
4163                                         rtr_priority, sensitivity);
4164                 mutex_unlock(&the_lnet.ln_api_mutex);
4165                 return rc;
4166         }
4167
4168         case IOC_LIBCFS_DEL_ROUTE:
4169                 config = arg;
4170
4171                 if (config->cfg_hdr.ioc_len < sizeof(*config))
4172                         return -EINVAL;
4173
4174                 lnet_nid4_to_nid(config->cfg_nid, &nid);
4175                 mutex_lock(&the_lnet.ln_api_mutex);
4176                 rc = lnet_del_route(config->cfg_net, &nid);
4177                 mutex_unlock(&the_lnet.ln_api_mutex);
4178                 return rc;
4179
4180         case IOC_LIBCFS_GET_ROUTE:
4181                 config = arg;
4182
4183                 if (config->cfg_hdr.ioc_len < sizeof(*config))
4184                         return -EINVAL;
4185
4186                 mutex_lock(&the_lnet.ln_api_mutex);
4187                 rc = lnet_get_route(config->cfg_count,
4188                                     &config->cfg_net,
4189                                     &config->cfg_config_u.cfg_route.rtr_hop,
4190                                     &config->cfg_nid,
4191                                     &config->cfg_config_u.cfg_route.rtr_flags,
4192                                     &config->cfg_config_u.cfg_route.
4193                                         rtr_priority,
4194                                     &config->cfg_config_u.cfg_route.
4195                                         rtr_sensitivity);
4196                 mutex_unlock(&the_lnet.ln_api_mutex);
4197                 return rc;
4198
4199         case IOC_LIBCFS_GET_LOCAL_NI: {
4200                 struct lnet_ioctl_config_ni *cfg_ni;
4201                 struct lnet_ioctl_config_lnd_tunables *tun = NULL;
4202                 struct lnet_ioctl_element_stats *stats;
4203                 __u32 tun_size;
4204
4205                 cfg_ni = arg;
4206
4207                 /* get the tunables if they are available */
4208                 if (cfg_ni->lic_cfg_hdr.ioc_len <
4209                     sizeof(*cfg_ni) + sizeof(*stats) + sizeof(*tun))
4210                         return -EINVAL;
4211
4212                 stats = (struct lnet_ioctl_element_stats *)
4213                         cfg_ni->lic_bulk;
4214                 tun = (struct lnet_ioctl_config_lnd_tunables *)
4215                                 (cfg_ni->lic_bulk + sizeof(*stats));
4216
4217                 tun_size = cfg_ni->lic_cfg_hdr.ioc_len - sizeof(*cfg_ni) -
4218                         sizeof(*stats);
4219
4220                 mutex_lock(&the_lnet.ln_api_mutex);
4221                 rc = lnet_get_ni_config(cfg_ni, tun, stats, tun_size);
4222                 mutex_unlock(&the_lnet.ln_api_mutex);
4223                 return rc;
4224         }
4225
4226         case IOC_LIBCFS_GET_LOCAL_NI_MSG_STATS: {
4227                 struct lnet_ioctl_element_msg_stats *msg_stats = arg;
4228                 int cpt;
4229
4230                 if (msg_stats->im_hdr.ioc_len != sizeof(*msg_stats))
4231                         return -EINVAL;
4232
4233                 mutex_lock(&the_lnet.ln_api_mutex);
4234
4235                 cpt = lnet_net_lock_current();
4236                 rc = lnet_get_ni_stats(msg_stats);
4237                 lnet_net_unlock(cpt);
4238
4239                 mutex_unlock(&the_lnet.ln_api_mutex);
4240
4241                 return rc;
4242         }
4243
4244         case IOC_LIBCFS_GET_NET: {
4245                 size_t total = sizeof(*config) +
4246                                sizeof(struct lnet_ioctl_net_config);
4247                 config = arg;
4248
4249                 if (config->cfg_hdr.ioc_len < total)
4250                         return -EINVAL;
4251
4252                 mutex_lock(&the_lnet.ln_api_mutex);
4253                 rc = lnet_get_net_config(config);
4254                 mutex_unlock(&the_lnet.ln_api_mutex);
4255                 return rc;
4256         }
4257
4258         case IOC_LIBCFS_GET_LNET_STATS:
4259         {
4260                 struct lnet_ioctl_lnet_stats *lnet_stats = arg;
4261
4262                 if (lnet_stats->st_hdr.ioc_len < sizeof(*lnet_stats))
4263                         return -EINVAL;
4264
4265                 mutex_lock(&the_lnet.ln_api_mutex);
4266                 rc = lnet_counters_get(&lnet_stats->st_cntrs);
4267                 mutex_unlock(&the_lnet.ln_api_mutex);
4268                 return rc;
4269         }
4270
4271         case IOC_LIBCFS_RESET_LNET_STATS:
4272         {
4273                 mutex_lock(&the_lnet.ln_api_mutex);
4274                 lnet_counters_reset();
4275                 mutex_unlock(&the_lnet.ln_api_mutex);
4276                 return 0;
4277         }
4278
4279         case IOC_LIBCFS_CONFIG_RTR:
4280                 config = arg;
4281
4282                 if (config->cfg_hdr.ioc_len < sizeof(*config))
4283                         return -EINVAL;
4284
4285                 mutex_lock(&the_lnet.ln_api_mutex);
4286                 if (config->cfg_config_u.cfg_buffers.buf_enable) {
4287                         rc = lnet_rtrpools_enable();
4288                         mutex_unlock(&the_lnet.ln_api_mutex);
4289                         return rc;
4290                 }
4291                 lnet_rtrpools_disable();
4292                 mutex_unlock(&the_lnet.ln_api_mutex);
4293                 return 0;
4294
4295         case IOC_LIBCFS_ADD_BUF:
4296                 config = arg;
4297
4298                 if (config->cfg_hdr.ioc_len < sizeof(*config))
4299                         return -EINVAL;
4300
4301                 mutex_lock(&the_lnet.ln_api_mutex);
4302                 rc = lnet_rtrpools_adjust(config->cfg_config_u.cfg_buffers.
4303                                                 buf_tiny,
4304                                           config->cfg_config_u.cfg_buffers.
4305                                                 buf_small,
4306                                           config->cfg_config_u.cfg_buffers.
4307                                                 buf_large);
4308                 mutex_unlock(&the_lnet.ln_api_mutex);
4309                 return rc;
4310
4311         case IOC_LIBCFS_SET_NUMA_RANGE: {
4312                 struct lnet_ioctl_set_value *numa;
4313                 numa = arg;
4314                 if (numa->sv_hdr.ioc_len != sizeof(*numa))
4315                         return -EINVAL;
4316                 lnet_net_lock(LNET_LOCK_EX);
4317                 lnet_numa_range = numa->sv_value;
4318                 lnet_net_unlock(LNET_LOCK_EX);
4319                 return 0;
4320         }
4321
4322         case IOC_LIBCFS_GET_NUMA_RANGE: {
4323                 struct lnet_ioctl_set_value *numa;
4324                 numa = arg;
4325                 if (numa->sv_hdr.ioc_len != sizeof(*numa))
4326                         return -EINVAL;
4327                 numa->sv_value = lnet_numa_range;
4328                 return 0;
4329         }
4330
4331         case IOC_LIBCFS_GET_BUF: {
4332                 struct lnet_ioctl_pool_cfg *pool_cfg;
4333                 size_t total = sizeof(*config) + sizeof(*pool_cfg);
4334
4335                 config = arg;
4336
4337                 if (config->cfg_hdr.ioc_len < total)
4338                         return -EINVAL;
4339
4340                 pool_cfg = (struct lnet_ioctl_pool_cfg *)config->cfg_bulk;
4341
4342                 mutex_lock(&the_lnet.ln_api_mutex);
4343                 rc = lnet_get_rtr_pool_cfg(config->cfg_count, pool_cfg);
4344                 mutex_unlock(&the_lnet.ln_api_mutex);
4345                 return rc;
4346         }
4347
4348         case IOC_LIBCFS_GET_LOCAL_HSTATS: {
4349                 struct lnet_ioctl_local_ni_hstats *stats = arg;
4350
4351                 if (stats->hlni_hdr.ioc_len < sizeof(*stats))
4352                         return -EINVAL;
4353
4354                 mutex_lock(&the_lnet.ln_api_mutex);
4355                 rc = lnet_get_local_ni_hstats(stats);
4356                 mutex_unlock(&the_lnet.ln_api_mutex);
4357
4358                 return rc;
4359         }
4360
4361         case IOC_LIBCFS_GET_RECOVERY_QUEUE: {
4362                 struct lnet_ioctl_recovery_list *list = arg;
4363                 if (list->rlst_hdr.ioc_len < sizeof(*list))
4364                         return -EINVAL;
4365
4366                 mutex_lock(&the_lnet.ln_api_mutex);
4367                 if (list->rlst_type == LNET_HEALTH_TYPE_LOCAL_NI)
4368                         rc = lnet_get_local_ni_recovery_list(list);
4369                 else
4370                         rc = lnet_get_peer_ni_recovery_list(list);
4371                 mutex_unlock(&the_lnet.ln_api_mutex);
4372                 return rc;
4373         }
4374
4375         case IOC_LIBCFS_ADD_PEER_NI: {
4376                 struct lnet_ioctl_peer_cfg *cfg = arg;
4377                 struct lnet_nid prim_nid;
4378
4379                 if (cfg->prcfg_hdr.ioc_len < sizeof(*cfg))
4380                         return -EINVAL;
4381
4382                 mutex_lock(&the_lnet.ln_api_mutex);
4383                 lnet_nid4_to_nid(cfg->prcfg_prim_nid, &prim_nid);
4384                 lnet_nid4_to_nid(cfg->prcfg_cfg_nid, &nid);
4385                 rc = lnet_user_add_peer_ni(&prim_nid, &nid, cfg->prcfg_mr,
4386                                            cfg->prcfg_count == 1);
4387                 mutex_unlock(&the_lnet.ln_api_mutex);
4388                 return rc;
4389         }
4390
4391         case IOC_LIBCFS_DEL_PEER_NI: {
4392                 struct lnet_ioctl_peer_cfg *cfg = arg;
4393                 struct lnet_nid prim_nid;
4394
4395                 if (cfg->prcfg_hdr.ioc_len < sizeof(*cfg))
4396                         return -EINVAL;
4397
4398                 mutex_lock(&the_lnet.ln_api_mutex);
4399                 lnet_nid4_to_nid(cfg->prcfg_prim_nid, &prim_nid);
4400                 lnet_nid4_to_nid(cfg->prcfg_cfg_nid, &nid);
4401                 rc = lnet_del_peer_ni(&prim_nid,
4402                                       &nid,
4403                                       cfg->prcfg_count);
4404                 mutex_unlock(&the_lnet.ln_api_mutex);
4405                 return rc;
4406         }
4407
4408         case IOC_LIBCFS_GET_PEER_INFO: {
4409                 struct lnet_ioctl_peer *peer_info = arg;
4410
4411                 if (peer_info->pr_hdr.ioc_len < sizeof(*peer_info))
4412                         return -EINVAL;
4413
4414                 mutex_lock(&the_lnet.ln_api_mutex);
4415                 rc = lnet_get_peer_ni_info(
4416                    peer_info->pr_count,
4417                    &peer_info->pr_nid,
4418                    peer_info->pr_lnd_u.pr_peer_credits.cr_aliveness,
4419                    &peer_info->pr_lnd_u.pr_peer_credits.cr_ncpt,
4420                    &peer_info->pr_lnd_u.pr_peer_credits.cr_refcount,
4421                    &peer_info->pr_lnd_u.pr_peer_credits.cr_ni_peer_tx_credits,
4422                    &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_tx_credits,
4423                    &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_rtr_credits,
4424                    &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_min_tx_credits,
4425                    &peer_info->pr_lnd_u.pr_peer_credits.cr_peer_tx_qnob);
4426                 mutex_unlock(&the_lnet.ln_api_mutex);
4427                 return rc;
4428         }
4429
4430         case IOC_LIBCFS_GET_PEER_NI: {
4431                 struct lnet_ioctl_peer_cfg *cfg = arg;
4432
4433                 if (cfg->prcfg_hdr.ioc_len < sizeof(*cfg))
4434                         return -EINVAL;
4435
4436                 mutex_lock(&the_lnet.ln_api_mutex);
4437                 rc = lnet_get_peer_info(cfg,
4438                                         (void __user *)cfg->prcfg_bulk);
4439                 mutex_unlock(&the_lnet.ln_api_mutex);
4440                 return rc;
4441         }
4442
4443         case IOC_LIBCFS_GET_PEER_LIST: {
4444                 struct lnet_ioctl_peer_cfg *cfg = arg;
4445
4446                 if (cfg->prcfg_hdr.ioc_len < sizeof(*cfg))
4447                         return -EINVAL;
4448
4449                 mutex_lock(&the_lnet.ln_api_mutex);
4450                 rc = lnet_get_peer_list(&cfg->prcfg_count, &cfg->prcfg_size,
4451                                 (struct lnet_process_id __user *)cfg->prcfg_bulk);
4452                 mutex_unlock(&the_lnet.ln_api_mutex);
4453                 return rc;
4454         }
4455
4456         case IOC_LIBCFS_SET_HEALHV: {
4457                 struct lnet_ioctl_reset_health_cfg *cfg = arg;
4458                 int value;
4459
4460                 if (cfg->rh_hdr.ioc_len < sizeof(*cfg))
4461                         return -EINVAL;
4462                 if (cfg->rh_value < 0 ||
4463                     cfg->rh_value > LNET_MAX_HEALTH_VALUE)
4464                         value = LNET_MAX_HEALTH_VALUE;
4465                 else
4466                         value = cfg->rh_value;
4467                 CDEBUG(D_NET, "Manually setting healthv to %d for %s:%s. all = %d\n",
4468                        value, (cfg->rh_type == LNET_HEALTH_TYPE_LOCAL_NI) ?
4469                        "local" : "peer", libcfs_nid2str(cfg->rh_nid), cfg->rh_all);
4470                 lnet_nid4_to_nid(cfg->rh_nid, &nid);
4471                 mutex_lock(&the_lnet.ln_api_mutex);
4472                 if (cfg->rh_type == LNET_HEALTH_TYPE_LOCAL_NI) {
4473                         if (cfg->rh_all)
4474                                 nid = LNET_ANY_NID;
4475                         lnet_ni_set_healthv(&nid, value);
4476                 } else {
4477                         lnet_peer_ni_set_healthv(&nid, value, cfg->rh_all);
4478                 }
4479                 mutex_unlock(&the_lnet.ln_api_mutex);
4480                 return 0;
4481         }
4482
4483         case IOC_LIBCFS_SET_PEER: {
4484                 struct lnet_ioctl_peer_cfg *cfg = arg;
4485                 struct lnet_peer *lp;
4486
4487                 if (cfg->prcfg_hdr.ioc_len < sizeof(*cfg))
4488                         return -EINVAL;
4489
4490                 mutex_lock(&the_lnet.ln_api_mutex);
4491                 lnet_nid4_to_nid(cfg->prcfg_prim_nid, &nid);
4492                 lp = lnet_find_peer(&nid);
4493                 if (!lp) {
4494                         mutex_unlock(&the_lnet.ln_api_mutex);
4495                         return -ENOENT;
4496                 }
4497                 spin_lock(&lp->lp_lock);
4498                 lp->lp_state = cfg->prcfg_state;
4499                 spin_unlock(&lp->lp_lock);
4500                 lnet_peer_decref_locked(lp);
4501                 mutex_unlock(&the_lnet.ln_api_mutex);
4502                 CDEBUG(D_NET, "Set peer %s state to %u\n",
4503                        libcfs_nid2str(cfg->prcfg_prim_nid), cfg->prcfg_state);
4504                 return 0;
4505         }
4506
4507         case IOC_LIBCFS_SET_CONNS_PER_PEER: {
4508                 struct lnet_ioctl_reset_conns_per_peer_cfg *cfg = arg;
4509                 int value;
4510
4511                 if (cfg->rcpp_hdr.ioc_len < sizeof(*cfg))
4512                         return -EINVAL;
4513                 if (cfg->rcpp_value < 0)
4514                         value = 1;
4515                 else
4516                         value = cfg->rcpp_value;
4517                 CDEBUG(D_NET,
4518                        "Setting conns_per_peer to %d for %s. all = %d\n",
4519                        value, libcfs_nid2str(cfg->rcpp_nid), cfg->rcpp_all);
4520                 mutex_lock(&the_lnet.ln_api_mutex);
4521                 lnet_ni_set_conns_per_peer(cfg->rcpp_nid, value, cfg->rcpp_all);
4522                 mutex_unlock(&the_lnet.ln_api_mutex);
4523                 return 0;
4524         }
4525
4526         case IOC_LIBCFS_NOTIFY_ROUTER: {
4527                 /* Convert the user-supplied real time to monotonic.
4528                  * NB: "when" is always in the past
4529                  */
4530                 time64_t when = ktime_get_seconds() -
4531                                 (ktime_get_real_seconds() - data->ioc_u64[0]);
4532
4533                 lnet_nid4_to_nid(data->ioc_nid, &nid);
4534                 return lnet_notify(NULL, &nid, data->ioc_flags, false, when);
4535         }
4536
4537         case IOC_LIBCFS_LNET_DIST:
4538                 lnet_nid4_to_nid(data->ioc_nid, &nid);
4539                 rc = LNetDist(&nid, &nid, &data->ioc_u32[1]);
4540                 if (rc < 0 && rc != -EHOSTUNREACH)
4541                         return rc;
4542
4543                 data->ioc_nid = lnet_nid_to_nid4(&nid);
4544                 data->ioc_u32[0] = rc;
4545                 return 0;
4546
4547         case IOC_LIBCFS_TESTPROTOCOMPAT:
4548                 the_lnet.ln_testprotocompat = data->ioc_flags;
4549                 return 0;
4550
4551         case IOC_LIBCFS_LNET_FAULT:
4552                 return lnet_fault_ctl(data->ioc_flags, data);
4553
4554         case IOC_LIBCFS_PING_PEER: {
4555                 struct lnet_ioctl_ping_data *ping = arg;
4556                 struct lnet_process_id __user *ids = ping->ping_buf;
4557                 struct lnet_nid src_nid = LNET_ANY_NID;
4558                 struct lnet_genl_ping_list plist;
4559                 struct lnet_processid id;
4560                 struct lnet_peer *lp;
4561                 signed long timeout;
4562                 int count, i;
4563
4564                 /* Check if the supplied ping data supports source nid
4565                  * NB: This check is sufficient if lnet_ioctl_ping_data has
4566                  * additional fields added, but if they are re-ordered or
4567                  * fields removed then this will break. It is expected that
4568                  * these ioctls will be replaced with netlink implementation, so
4569                  * it is probably not worth coming up with a more robust version
4570                  * compatibility scheme.
4571                  */
4572                 if (ping->ping_hdr.ioc_len >= sizeof(struct lnet_ioctl_ping_data))
4573                         lnet_nid4_to_nid(ping->ping_src, &src_nid);
4574
4575                 /* If timeout is negative then set default of 3 minutes */
4576                 if (((s32)ping->op_param) <= 0 ||
4577                     ping->op_param > (DEFAULT_PEER_TIMEOUT * MSEC_PER_SEC))
4578                         timeout = cfs_time_seconds(DEFAULT_PEER_TIMEOUT);
4579                 else
4580                         timeout = nsecs_to_jiffies(ping->op_param * NSEC_PER_MSEC);
4581
4582                 id.pid = ping->ping_id.pid;
4583                 lnet_nid4_to_nid(ping->ping_id.nid, &id.nid);
4584                 rc = lnet_ping(&id, &src_nid, timeout, &plist,
4585                                ping->ping_count);
4586                 if (rc < 0)
4587                         goto report_ping_err;
4588                 count = rc;
4589                 rc = 0;
4590
4591                 for (i = 0; i < count; i++) {
4592                         struct lnet_processid *result;
4593                         struct lnet_process_id tmpid;
4594
4595                         result = genradix_ptr(&plist.lgpl_list, i);
4596                         memset(&tmpid, 0, sizeof(tmpid));
4597                         tmpid.pid = result->pid;
4598                         tmpid.nid = lnet_nid_to_nid4(&result->nid);
4599                         if (copy_to_user(&ids[i], &tmpid, sizeof(tmpid))) {
4600                                 rc = -EFAULT;
4601                                 goto report_ping_err;
4602                         }
4603                 }
4604
4605                 mutex_lock(&the_lnet.ln_api_mutex);
4606                 lp = lnet_find_peer(&id.nid);
4607                 if (lp) {
4608                         ping->ping_id.nid =
4609                                 lnet_nid_to_nid4(&lp->lp_primary_nid);
4610                         ping->mr_info = lnet_peer_is_multi_rail(lp);
4611                         lnet_peer_decref_locked(lp);
4612                 }
4613                 mutex_unlock(&the_lnet.ln_api_mutex);
4614
4615                 ping->ping_count = count;
4616 report_ping_err:
4617                 genradix_free(&plist.lgpl_list);
4618                 return rc;
4619         }
4620
4621         case IOC_LIBCFS_DISCOVER: {
4622                 struct lnet_ioctl_ping_data *discover = arg;
4623                 struct lnet_process_id __user *ids;
4624                 struct lnet_genl_ping_list dlists;
4625                 struct lnet_processid id;
4626                 struct lnet_peer *lp;
4627                 int count, i;
4628
4629                 if (discover->ping_count <= 0)
4630                         return -EINVAL;
4631
4632                 genradix_init(&dlists.lgpl_list);
4633                 /* If the user buffer has more space than the lnet_interfaces_max,
4634                  * then only fill it up to lnet_interfaces_max.
4635                  */
4636                 if (discover->ping_count > lnet_interfaces_max)
4637                         discover->ping_count = lnet_interfaces_max;
4638
4639                 id.pid = discover->ping_id.pid;
4640                 lnet_nid4_to_nid(discover->ping_id.nid, &id.nid);
4641                 rc = lnet_discover(&id, discover->op_param, &dlists);
4642                 if (rc < 0)
4643                         goto report_discover_err;
4644                 count = rc;
4645
4646                 ids = discover->ping_buf;
4647                 for (i = 0; i < count; i++) {
4648                         struct lnet_processid *result;
4649                         struct lnet_process_id tmpid;
4650
4651                         result = genradix_ptr(&dlists.lgpl_list, i);
4652                         memset(&tmpid, 0, sizeof(tmpid));
4653                         tmpid.pid = result->pid;
4654                         tmpid.nid = lnet_nid_to_nid4(&result->nid);
4655                         if (copy_to_user(&ids[i], &tmpid, sizeof(tmpid))) {
4656                                 rc = -EFAULT;
4657                                 goto report_discover_err;
4658                         }
4659
4660                         if (i >= discover->ping_count)
4661                                 break;
4662                 }
4663                 rc = 0;
4664
4665                 mutex_lock(&the_lnet.ln_api_mutex);
4666                 lp = lnet_find_peer(&id.nid);
4667                 if (lp) {
4668                         discover->ping_id.nid =
4669                                 lnet_nid_to_nid4(&lp->lp_primary_nid);
4670                         discover->mr_info = lnet_peer_is_multi_rail(lp);
4671                         lnet_peer_decref_locked(lp);
4672                 }
4673                 mutex_unlock(&the_lnet.ln_api_mutex);
4674
4675                 discover->ping_count = count;
4676 report_discover_err:
4677                 genradix_free(&dlists.lgpl_list);
4678                 return rc;
4679         }
4680
4681         case IOC_LIBCFS_ADD_UDSP: {
4682                 struct lnet_ioctl_udsp *ioc_udsp = arg;
4683                 __u32 bulk_size = ioc_udsp->iou_hdr.ioc_len;
4684
4685                 mutex_lock(&the_lnet.ln_api_mutex);
4686                 rc = lnet_udsp_demarshal_add(arg, bulk_size);
4687                 if (!rc) {
4688                         rc = lnet_udsp_apply_policies(NULL, false);
4689                         CDEBUG(D_NET, "policy application returned %d\n", rc);
4690                         rc = 0;
4691                 }
4692                 mutex_unlock(&the_lnet.ln_api_mutex);
4693
4694                 return rc;
4695         }
4696
4697         case IOC_LIBCFS_DEL_UDSP: {
4698                 struct lnet_ioctl_udsp *ioc_udsp = arg;
4699                 int idx = ioc_udsp->iou_idx;
4700
4701                 if (ioc_udsp->iou_hdr.ioc_len < sizeof(*ioc_udsp))
4702                         return -EINVAL;
4703
4704                 mutex_lock(&the_lnet.ln_api_mutex);
4705                 rc = lnet_udsp_del_policy(idx);
4706                 mutex_unlock(&the_lnet.ln_api_mutex);
4707
4708                 return rc;
4709         }
4710
4711         case IOC_LIBCFS_GET_UDSP_SIZE: {
4712                 struct lnet_ioctl_udsp *ioc_udsp = arg;
4713                 struct lnet_udsp *udsp;
4714
4715                 if (ioc_udsp->iou_hdr.ioc_len < sizeof(*ioc_udsp))
4716                         return -EINVAL;
4717
4718                 rc = 0;
4719
4720                 mutex_lock(&the_lnet.ln_api_mutex);
4721                 udsp = lnet_udsp_get_policy(ioc_udsp->iou_idx);
4722                 if (!udsp) {
4723                         rc = -ENOENT;
4724                 } else {
4725                         /* coming in iou_idx will hold the idx of the udsp
4726                          * to get the size of. going out the iou_idx will
4727                          * hold the size of the UDSP found at the passed
4728                          * in index.
4729                          */
4730                         ioc_udsp->iou_idx = lnet_get_udsp_size(udsp);
4731                         if (ioc_udsp->iou_idx < 0)
4732                                 rc = -EINVAL;
4733                 }
4734                 mutex_unlock(&the_lnet.ln_api_mutex);
4735
4736                 return rc;
4737         }
4738
4739         case IOC_LIBCFS_GET_UDSP: {
4740                 struct lnet_ioctl_udsp *ioc_udsp = arg;
4741                 struct lnet_udsp *udsp;
4742
4743                 if (ioc_udsp->iou_hdr.ioc_len < sizeof(*ioc_udsp))
4744                         return -EINVAL;
4745
4746                 rc = 0;
4747
4748                 mutex_lock(&the_lnet.ln_api_mutex);
4749                 udsp = lnet_udsp_get_policy(ioc_udsp->iou_idx);
4750                 if (!udsp)
4751                         rc = -ENOENT;
4752                 else
4753                         rc = lnet_udsp_marshal(udsp, ioc_udsp);
4754                 mutex_unlock(&the_lnet.ln_api_mutex);
4755
4756                 return rc;
4757         }
4758
4759         case IOC_LIBCFS_GET_CONST_UDSP_INFO: {
4760                 struct lnet_ioctl_construct_udsp_info *info = arg;
4761
4762                 if (info->cud_hdr.ioc_len < sizeof(*info))
4763                         return -EINVAL;
4764
4765                 CDEBUG(D_NET, "GET_UDSP_INFO for %s\n",
4766                        libcfs_nid2str(info->cud_nid));
4767
4768                 lnet_nid4_to_nid(info->cud_nid, &nid);
4769                 mutex_lock(&the_lnet.ln_api_mutex);
4770                 lnet_net_lock(0);
4771                 lnet_udsp_get_construct_info(info, &nid);
4772                 lnet_net_unlock(0);
4773                 mutex_unlock(&the_lnet.ln_api_mutex);
4774
4775                 return 0;
4776         }
4777
4778         default:
4779                 ni = lnet_net2ni_addref(data->ioc_net);
4780                 if (ni == NULL)
4781                         return -EINVAL;
4782
4783                 if (ni->ni_net->net_lnd->lnd_ctl == NULL)
4784                         rc = -EINVAL;
4785                 else
4786                         rc = ni->ni_net->net_lnd->lnd_ctl(ni, cmd, arg);
4787
4788                 lnet_ni_decref(ni);
4789                 return rc <= 0 ? rc : 0;
4790         }
4791         /* not reached */
4792 }
4793 EXPORT_SYMBOL(LNetCtl);
4794
4795 static int lnet_net_conf_cmd(struct sk_buff *skb, struct genl_info *info)
4796 {
4797         int rc = 0;
4798
4799         if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
4800                 /* NLM_F_EXCL means ignore module parameters */
4801                 if (info->nlhdr->nlmsg_flags & NLM_F_EXCL)
4802                         the_lnet.ln_nis_from_mod_params = true;
4803
4804                 rc = lnet_configure(NULL);
4805                 switch (rc) {
4806                 case -ENETDOWN:
4807                         GENL_SET_ERR_MSG(info,
4808                                          "Network is down");
4809                         break;
4810                 case -ENODEV:
4811                         GENL_SET_ERR_MSG(info,
4812                                          "LNET is currently not loaded");
4813                         break;
4814                 case -EBUSY:
4815                         GENL_SET_ERR_MSG(info, "LNET busy");
4816                         break;
4817                 default:
4818                         break;
4819                 }
4820         } else {
4821                 rc = lnet_unconfigure();
4822         }
4823
4824         return rc;
4825 };
4826
4827 struct lnet_nid_cpt {
4828         struct lnet_nid lnc_nid;
4829         unsigned int lnc_cpt;
4830 };
4831
4832 struct lnet_genl_nid_cpt_list {
4833         unsigned int lgncl_index;
4834         unsigned int lgncl_list_count;
4835         GENRADIX(struct lnet_nid_cpt) lgncl_lnc_list;
4836 };
4837
4838 static inline struct lnet_genl_nid_cpt_list *
4839 lnet_cpt_of_nid_dump_ctx(struct netlink_callback *cb)
4840 {
4841         return (struct lnet_genl_nid_cpt_list *)cb->args[0];
4842 }
4843
4844 static int lnet_cpt_of_nid_show_done(struct netlink_callback *cb)
4845 {
4846         struct lnet_genl_nid_cpt_list *lgncl;
4847
4848         lgncl = lnet_cpt_of_nid_dump_ctx(cb);
4849
4850         if (lgncl) {
4851                 genradix_free(&lgncl->lgncl_lnc_list);
4852                 LIBCFS_FREE(lgncl, sizeof(*lgncl));
4853                 cb->args[0] = 0;
4854         }
4855
4856         return 0;
4857 }
4858
4859 static int lnet_cpt_of_nid_show_start(struct netlink_callback *cb)
4860 {
4861         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
4862 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
4863         struct netlink_ext_ack *extack = NULL;
4864 #endif
4865         struct lnet_genl_nid_cpt_list *lgncl;
4866         int msg_len = genlmsg_len(gnlh);
4867         struct nlattr *params, *top;
4868         int rem, rc = 0;
4869
4870 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
4871         extack = cb->extack;
4872 #endif
4873
4874         mutex_lock(&the_lnet.ln_api_mutex);
4875         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
4876                 NL_SET_ERR_MSG(extack, "Network is down");
4877                 mutex_unlock(&the_lnet.ln_api_mutex);
4878                 return -ENETDOWN;
4879         }
4880
4881         msg_len = genlmsg_len(gnlh);
4882         if (!msg_len) {
4883                 NL_SET_ERR_MSG(extack, "Missing NID argument(s)");
4884                 mutex_unlock(&the_lnet.ln_api_mutex);
4885                 return -ENOENT;
4886         }
4887
4888         LIBCFS_ALLOC(lgncl, sizeof(*lgncl));
4889         if (!lgncl) {
4890                 mutex_unlock(&the_lnet.ln_api_mutex);
4891                 return -ENOMEM;
4892         }
4893
4894         genradix_init(&lgncl->lgncl_lnc_list);
4895         lgncl->lgncl_list_count = 0;
4896         cb->args[0] = (long)lgncl;
4897
4898         params = genlmsg_data(gnlh);
4899         nla_for_each_attr(top, params, msg_len, rem) {
4900                 struct nlattr *nids;
4901                 int rem2;
4902
4903                 switch (nla_type(top)) {
4904                 case LN_SCALAR_ATTR_LIST:
4905                         nla_for_each_nested(nids, top, rem2) {
4906                                 char nidstr[LNET_NIDSTR_SIZE + 1];
4907                                 struct lnet_nid_cpt *lnc;
4908
4909                                 if (nla_type(nids) != LN_SCALAR_ATTR_VALUE)
4910                                         continue;
4911
4912                                 memset(nidstr, 0, sizeof(nidstr));
4913                                 rc = nla_strscpy(nidstr, nids, sizeof(nidstr));
4914                                 if (rc < 0) {
4915                                         NL_SET_ERR_MSG(extack,
4916                                                        "failed to get NID");
4917                                         GOTO(report_err, rc);
4918                                 }
4919
4920                                 lnc = genradix_ptr_alloc(&lgncl->lgncl_lnc_list,
4921                                                       lgncl->lgncl_list_count++,
4922                                                       GFP_KERNEL);
4923                                 if (!lnc) {
4924                                         NL_SET_ERR_MSG(extack,
4925                                                       "failed to allocate NID");
4926                                         GOTO(report_err, rc = -ENOMEM);
4927                                 }
4928
4929                                 rc = libcfs_strnid(&lnc->lnc_nid,
4930                                                    strim(nidstr));
4931                                 if (rc < 0) {
4932                                         NL_SET_ERR_MSG(extack, "invalid NID");
4933                                         GOTO(report_err, rc);
4934                                 }
4935                                 rc = 0;
4936                                 CDEBUG(D_NET, "nid: %s\n",
4937                                        libcfs_nidstr(&lnc->lnc_nid));
4938                         }
4939                         fallthrough;
4940                 default:
4941                         break;
4942                 }
4943         }
4944 report_err:
4945         mutex_unlock(&the_lnet.ln_api_mutex);
4946
4947         if (rc < 0)
4948                 lnet_cpt_of_nid_show_done(cb);
4949
4950         return rc;
4951 }
4952
4953 static const struct ln_key_list cpt_of_nid_props_list = {
4954         .lkl_maxattr                    = LNET_CPT_OF_NID_ATTR_MAX,
4955         .lkl_list                       = {
4956                 [LNET_CPT_OF_NID_ATTR_HDR]      = {
4957                         .lkp_value              = "cpt-of-nid",
4958                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
4959                         .lkp_data_type          = NLA_NUL_STRING,
4960                 },
4961                 [LNET_CPT_OF_NID_ATTR_NID]      = {
4962                         .lkp_value              = "nid",
4963                         .lkp_data_type          = NLA_STRING,
4964                 },
4965                 [LNET_CPT_OF_NID_ATTR_CPT]      = {
4966                         .lkp_value              = "cpt",
4967                         .lkp_data_type          = NLA_U32,
4968                 },
4969         },
4970 };
4971
4972 static int lnet_cpt_of_nid_show_dump(struct sk_buff *msg,
4973                                      struct netlink_callback *cb)
4974 {
4975         struct lnet_genl_nid_cpt_list *lgncl;
4976 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
4977         struct netlink_ext_ack *extack = NULL;
4978 #endif
4979         int portid = NETLINK_CB(cb->skb).portid;
4980         int seq = cb->nlh->nlmsg_seq;
4981         int idx;
4982         int rc = 0;
4983         bool need_hdr = true;
4984
4985 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
4986         extack = cb->extack;
4987 #endif
4988
4989         mutex_lock(&the_lnet.ln_api_mutex);
4990         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
4991                 NL_SET_ERR_MSG(extack, "Network is down");
4992                 GOTO(send_error, rc = -ENETDOWN);
4993         }
4994
4995         lgncl = lnet_cpt_of_nid_dump_ctx(cb);
4996         idx = lgncl->lgncl_index;
4997
4998         if (!lgncl->lgncl_index) {
4999                 const struct ln_key_list *all[] = {
5000                         &cpt_of_nid_props_list, NULL, NULL
5001                 };
5002
5003                 rc = lnet_genl_send_scalar_list(msg, portid, seq, &lnet_family,
5004                                                 NLM_F_CREATE | NLM_F_MULTI,
5005                                                 LNET_CMD_CPT_OF_NID, all);
5006                 if (rc < 0) {
5007                         NL_SET_ERR_MSG(extack, "failed to send key table");
5008                         GOTO(send_error, rc);
5009                 }
5010         }
5011
5012         while (idx < lgncl->lgncl_list_count) {
5013                 struct lnet_nid_cpt *lnc;
5014                 void *hdr;
5015                 int cpt;
5016
5017                 lnc = genradix_ptr(&lgncl->lgncl_lnc_list, idx++);
5018
5019                 cpt = lnet_nid_cpt_hash(&lnc->lnc_nid, LNET_CPT_NUMBER);
5020
5021                 CDEBUG(D_NET, "nid: %s cpt: %d\n", libcfs_nidstr(&lnc->lnc_nid), cpt);
5022                 hdr = genlmsg_put(msg, portid, seq, &lnet_family,
5023                                   NLM_F_MULTI, LNET_CMD_CPT_OF_NID);
5024                 if (!hdr) {
5025                         NL_SET_ERR_MSG(extack, "failed to send values");
5026                         genlmsg_cancel(msg, hdr);
5027                         GOTO(send_error, rc = -EMSGSIZE);
5028                 }
5029
5030                 if (need_hdr) {
5031                         nla_put_string(msg, LNET_CPT_OF_NID_ATTR_HDR, "");
5032                         need_hdr = false;
5033                 }
5034
5035                 nla_put_string(msg, LNET_CPT_OF_NID_ATTR_NID,
5036                                libcfs_nidstr(&lnc->lnc_nid));
5037                 nla_put_u32(msg, LNET_CPT_OF_NID_ATTR_CPT, cpt);
5038
5039                 genlmsg_end(msg, hdr);
5040         }
5041
5042         genradix_free(&lgncl->lgncl_lnc_list);
5043         rc = 0;
5044         lgncl->lgncl_index = idx;
5045
5046 send_error:
5047         mutex_unlock(&the_lnet.ln_api_mutex);
5048
5049         return lnet_nl_send_error(cb->skb, portid, seq, rc);
5050 }
5051
5052 #ifndef HAVE_NETLINK_CALLBACK_START
5053 static int lnet_old_cpt_of_nid_show_dump(struct sk_buff *msg,
5054                                          struct netlink_callback *cb)
5055 {
5056         if (!cb->args[0]) {
5057                 int rc = lnet_cpt_of_nid_show_start(cb);
5058
5059                 if (rc < 0)
5060                         return lnet_nl_send_error(cb->skb,
5061                                                   NETLINK_CB(cb->skb).portid,
5062                                                   cb->nlh->nlmsg_seq,
5063                                                   rc);
5064         }
5065
5066         return lnet_cpt_of_nid_show_dump(msg, cb);
5067 }
5068 #endif
5069
5070 /* This is the keys for the UDSP info which is used by many
5071  * Netlink commands.
5072  */
5073 static const struct ln_key_list udsp_info_list = {
5074         .lkl_maxattr                    = LNET_UDSP_INFO_ATTR_MAX,
5075         .lkl_list                       = {
5076                 [LNET_UDSP_INFO_ATTR_NET_PRIORITY]              = {
5077                         .lkp_value      = "net priority",
5078                         .lkp_data_type  = NLA_S32
5079                 },
5080                 [LNET_UDSP_INFO_ATTR_NID_PRIORITY]              = {
5081                         .lkp_value      = "nid priority",
5082                         .lkp_data_type  = NLA_S32
5083                 },
5084                 [LNET_UDSP_INFO_ATTR_PREF_RTR_NIDS_LIST]        = {
5085                         .lkp_value      = "Preferred gateway NIDs",
5086                         .lkp_key_format = LNKF_MAPPING,
5087                         .lkp_data_type  = NLA_NESTED,
5088                 },
5089                 [LNET_UDSP_INFO_ATTR_PREF_NIDS_LIST]            = {
5090                         .lkp_value      = "Preferred source NIDs",
5091                         .lkp_key_format = LNKF_MAPPING,
5092                         .lkp_data_type  = NLA_NESTED,
5093                 },
5094         },
5095 };
5096
5097 static const struct ln_key_list udsp_info_pref_nids_list = {
5098         .lkl_maxattr                    = LNET_UDSP_INFO_PREF_NIDS_ATTR_MAX,
5099         .lkl_list                       = {
5100                 [LNET_UDSP_INFO_PREF_NIDS_ATTR_INDEX]           = {
5101                         .lkp_value      = "NID-0",
5102                         .lkp_data_type  = NLA_NUL_STRING,
5103                 },
5104                 [LNET_UDSP_INFO_PREF_NIDS_ATTR_NID]             = {
5105                         .lkp_value      = "0@lo",
5106                         .lkp_data_type  = NLA_STRING,
5107                 },
5108         },
5109 };
5110
5111 static int lnet_udsp_info_send(struct sk_buff *msg, int attr,
5112                                struct lnet_nid *nid, bool remote)
5113 {
5114         struct lnet_ioctl_construct_udsp_info *udsp;
5115         struct nlattr *udsp_attr, *udsp_info;
5116         struct nlattr *udsp_list_attr;
5117         struct nlattr *udsp_list_info;
5118         int i;
5119
5120         CFS_ALLOC_PTR(udsp);
5121         if (!udsp)
5122                 return -ENOMEM;
5123
5124         udsp->cud_peer = remote;
5125         lnet_udsp_get_construct_info(udsp, nid);
5126
5127         udsp_info = nla_nest_start(msg, attr);
5128         udsp_attr = nla_nest_start(msg, 0);
5129         nla_put_s32(msg, LNET_UDSP_INFO_ATTR_NET_PRIORITY,
5130                     udsp->cud_net_priority);
5131         nla_put_s32(msg, LNET_UDSP_INFO_ATTR_NID_PRIORITY,
5132                     udsp->cud_nid_priority);
5133
5134         if (udsp->cud_pref_rtr_nid[0] == 0)
5135                 goto skip_list;
5136
5137         udsp_list_info = nla_nest_start(msg,
5138                                         LNET_UDSP_INFO_ATTR_PREF_RTR_NIDS_LIST);
5139         for (i = 0; i < LNET_MAX_SHOW_NUM_NID; i++) {
5140                 char tmp[8]; /* NID-"3 number"\0 */
5141
5142                 if (udsp->cud_pref_rtr_nid[i] == 0)
5143                         break;
5144
5145                 udsp_list_attr = nla_nest_start(msg, i);
5146                 snprintf(tmp, sizeof(tmp), "NID-%d", i);
5147                 nla_put_string(msg, LNET_UDSP_INFO_PREF_NIDS_ATTR_INDEX,
5148                                tmp);
5149                 nla_put_string(msg, LNET_UDSP_INFO_PREF_NIDS_ATTR_NID,
5150                                libcfs_nid2str(udsp->cud_pref_rtr_nid[i]));
5151                 nla_nest_end(msg, udsp_list_attr);
5152         }
5153         nla_nest_end(msg, udsp_list_info);
5154 skip_list:
5155         nla_nest_end(msg, udsp_attr);
5156         nla_nest_end(msg, udsp_info);
5157         LIBCFS_FREE(udsp, sizeof(*udsp));
5158
5159         return 0;
5160 }
5161
5162 /* LNet NI handling */
5163 static const struct ln_key_list net_props_list = {
5164         .lkl_maxattr                    = LNET_NET_ATTR_MAX,
5165         .lkl_list                       = {
5166                 [LNET_NET_ATTR_HDR]             = {
5167                         .lkp_value              = "net",
5168                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
5169                         .lkp_data_type          = NLA_NUL_STRING,
5170                 },
5171                 [LNET_NET_ATTR_TYPE]            = {
5172                         .lkp_value              = "net type",
5173                         .lkp_data_type          = NLA_STRING
5174                 },
5175                 [LNET_NET_ATTR_LOCAL]           = {
5176                         .lkp_value              = "local NI(s)",
5177                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
5178                         .lkp_data_type          = NLA_NESTED
5179                 },
5180         },
5181 };
5182
5183 static struct ln_key_list local_ni_list = {
5184         .lkl_maxattr                    = LNET_NET_LOCAL_NI_ATTR_MAX,
5185         .lkl_list                       = {
5186                 [LNET_NET_LOCAL_NI_ATTR_NID]            = {
5187                         .lkp_value              = "nid",
5188                         .lkp_data_type          = NLA_STRING
5189                 },
5190                 [LNET_NET_LOCAL_NI_ATTR_STATUS]         = {
5191                         .lkp_value              = "status",
5192                         .lkp_data_type          = NLA_STRING
5193                 },
5194                 [LNET_NET_LOCAL_NI_ATTR_INTERFACE]      = {
5195                         .lkp_value              = "interfaces",
5196                         .lkp_key_format         = LNKF_MAPPING,
5197                         .lkp_data_type          = NLA_NESTED
5198                 },
5199                 [LNET_NET_LOCAL_NI_ATTR_STATS]          = {
5200                         .lkp_value              = "statistics",
5201                         .lkp_key_format         = LNKF_MAPPING,
5202                         .lkp_data_type          = NLA_NESTED
5203                 },
5204                 [LNET_NET_LOCAL_NI_ATTR_UDSP_INFO]      = {
5205                         .lkp_value              = "udsp info",
5206                         .lkp_key_format         = LNKF_MAPPING,
5207                         .lkp_data_type          = NLA_NESTED
5208                 },
5209                 [LNET_NET_LOCAL_NI_ATTR_SEND_STATS]     = {
5210                         .lkp_value              = "sent_stats",
5211                         .lkp_key_format         = LNKF_MAPPING,
5212                         .lkp_data_type          = NLA_NESTED
5213                 },
5214                 [LNET_NET_LOCAL_NI_ATTR_RECV_STATS]     = {
5215                         .lkp_value              = "received_stats",
5216                         .lkp_key_format         = LNKF_MAPPING,
5217                         .lkp_data_type          = NLA_NESTED
5218                 },
5219                 [LNET_NET_LOCAL_NI_ATTR_DROPPED_STATS]  = {
5220                         .lkp_value              = "dropped_stats",
5221                         .lkp_key_format         = LNKF_MAPPING,
5222                         .lkp_data_type          = NLA_NESTED
5223
5224                 },
5225                 [LNET_NET_LOCAL_NI_ATTR_HEALTH_STATS]   = {
5226                         .lkp_value              = "health stats",
5227                         .lkp_key_format         = LNKF_MAPPING,
5228                         .lkp_data_type          = NLA_NESTED
5229                 },
5230                 [LNET_NET_LOCAL_NI_ATTR_TUNABLES]       = {
5231                         .lkp_value              = "tunables",
5232                         .lkp_key_format         = LNKF_MAPPING,
5233                         .lkp_data_type          = NLA_NESTED
5234                 },
5235                 [LNET_NET_LOCAL_NI_ATTR_LND_TUNABLES]   = {
5236                         .lkp_value              = "lnd tunables",
5237                         .lkp_key_format         = LNKF_MAPPING,
5238                         .lkp_data_type          = NLA_NESTED
5239                 },
5240                 [LNET_NET_LOCAL_NI_DEV_CPT]             = {
5241                         .lkp_value              = "dev cpt",
5242                         .lkp_data_type          = NLA_S32,
5243                 },
5244                 [LNET_NET_LOCAL_NI_CPTS]                = {
5245                         .lkp_value              = "CPT",
5246                         .lkp_data_type          = NLA_STRING,
5247                 },
5248         },
5249 };
5250
5251 static const struct ln_key_list local_ni_interfaces_list = {
5252         .lkl_maxattr                    = LNET_NET_LOCAL_NI_INTF_ATTR_MAX,
5253         .lkl_list                       = {
5254                 [LNET_NET_LOCAL_NI_INTF_ATTR_TYPE] = {
5255                         .lkp_value      = "0",
5256                         .lkp_data_type  = NLA_STRING
5257                 },
5258         },
5259 };
5260
5261 static const struct ln_key_list local_ni_stats_list = {
5262         .lkl_maxattr                    = LNET_NET_LOCAL_NI_STATS_ATTR_MAX,
5263         .lkl_list                       = {
5264                 [LNET_NET_LOCAL_NI_STATS_ATTR_SEND_COUNT]       = {
5265                         .lkp_value      = "send_count",
5266                         .lkp_data_type  = NLA_U32
5267                 },
5268                 [LNET_NET_LOCAL_NI_STATS_ATTR_RECV_COUNT]       = {
5269                         .lkp_value      = "recv_count",
5270                         .lkp_data_type  = NLA_U32
5271                 },
5272                 [LNET_NET_LOCAL_NI_STATS_ATTR_DROP_COUNT]       = {
5273                         .lkp_value      = "drop_count",
5274                         .lkp_data_type  = NLA_U32
5275                 },
5276         },
5277 };
5278
5279 static const struct ln_key_list local_ni_msg_stats_list = {
5280         .lkl_maxattr                    = LNET_NET_LOCAL_NI_MSG_STATS_ATTR_MAX,
5281         .lkl_list                       = {
5282                 [LNET_NET_LOCAL_NI_MSG_STATS_ATTR_PUT_COUNT]    = {
5283                         .lkp_value      = "put",
5284                         .lkp_data_type  = NLA_U32
5285                 },
5286                 [LNET_NET_LOCAL_NI_MSG_STATS_ATTR_GET_COUNT]    = {
5287                         .lkp_value      = "get",
5288                         .lkp_data_type  = NLA_U32
5289                 },
5290                 [LNET_NET_LOCAL_NI_MSG_STATS_ATTR_REPLY_COUNT]  = {
5291                         .lkp_value      = "reply",
5292                         .lkp_data_type  = NLA_U32
5293                 },
5294                 [LNET_NET_LOCAL_NI_MSG_STATS_ATTR_ACK_COUNT]    = {
5295                         .lkp_value      = "ack",
5296                         .lkp_data_type  = NLA_U32
5297                 },
5298                 [LNET_NET_LOCAL_NI_MSG_STATS_ATTR_HELLO_COUNT]  = {
5299                         .lkp_value      = "hello",
5300                         .lkp_data_type  = NLA_U32
5301                 },
5302         },
5303 };
5304
5305 static const struct ln_key_list local_ni_health_stats_list = {
5306         .lkl_maxattr                    = LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_MAX,
5307         .lkl_list                       = {
5308                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_FATAL_ERRORS] = {
5309                         .lkp_value      = "fatal_error",
5310                         .lkp_data_type  = NLA_S32
5311                 },
5312                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_LEVEL] = {
5313                         .lkp_value      = "health value",
5314                         .lkp_data_type  = NLA_S32
5315                 },
5316                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_INTERRUPTS] = {
5317                         .lkp_value      = "interrupts",
5318                         .lkp_data_type  = NLA_U32
5319                 },
5320                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_DROPPED] = {
5321                         .lkp_value      = "dropped",
5322                         .lkp_data_type  = NLA_U32
5323                 },
5324                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_ABORTED] = {
5325                         .lkp_value      = "aborted",
5326                         .lkp_data_type  = NLA_U32
5327                 },
5328                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_NO_ROUTE] = {
5329                         .lkp_value      = "no route",
5330                         .lkp_data_type  = NLA_U32
5331                 },
5332                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_TIMEOUTS] = {
5333                         .lkp_value      = "timeouts",
5334                         .lkp_data_type  = NLA_U32
5335                 },
5336                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_ERROR] = {
5337                         .lkp_value      = "error",
5338                         .lkp_data_type  = NLA_U32
5339                 },
5340                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_PING_COUNT] = {
5341                         .lkp_value      = "ping_count",
5342                         .lkp_data_type  = NLA_U32,
5343                 },
5344                 [LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_NEXT_PING] = {
5345                         .lkp_value      = "next_ping",
5346                         .lkp_data_type  = NLA_U64
5347                 },
5348         },
5349 };
5350
5351 static const struct ln_key_list local_ni_tunables_list = {
5352         .lkl_maxattr                    = LNET_NET_LOCAL_NI_TUNABLES_ATTR_MAX,
5353         .lkl_list                       = {
5354                 [LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_TIMEOUT]  = {
5355                         .lkp_value      = "peer_timeout",
5356                         .lkp_data_type  = NLA_S32
5357                 },
5358                 [LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_CREDITS]  = {
5359                         .lkp_value      = "peer_credits",
5360                         .lkp_data_type  = NLA_S32
5361                 },
5362                 [LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_BUFFER_CREDITS] = {
5363                         .lkp_value      = "peer_buffer_credits",
5364                         .lkp_data_type  = NLA_S32
5365                 },
5366                 [LNET_NET_LOCAL_NI_TUNABLES_ATTR_CREDITS] = {
5367                         .lkp_value      = "credits",
5368                         .lkp_data_type  = NLA_S32
5369                 },
5370         },
5371 };
5372
5373 /* Use an index since the traversal is across LNet nets and ni collections */
5374 struct lnet_genl_net_list {
5375         unsigned int    lngl_net_id;
5376         unsigned int    lngl_idx;
5377 };
5378
5379 static inline struct lnet_genl_net_list *
5380 lnet_net_dump_ctx(struct netlink_callback *cb)
5381 {
5382         return (struct lnet_genl_net_list *)cb->args[0];
5383 }
5384
5385 static int lnet_net_show_done(struct netlink_callback *cb)
5386 {
5387         struct lnet_genl_net_list *nlist = lnet_net_dump_ctx(cb);
5388
5389         if (nlist) {
5390                 LIBCFS_FREE(nlist, sizeof(*nlist));
5391                 cb->args[0] = 0;
5392         }
5393
5394         return 0;
5395 }
5396
5397 /* LNet net ->start() handler for GET requests */
5398 static int lnet_net_show_start(struct netlink_callback *cb)
5399 {
5400         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
5401 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
5402         struct netlink_ext_ack *extack = NULL;
5403 #endif
5404         struct lnet_genl_net_list *nlist;
5405         int msg_len = genlmsg_len(gnlh);
5406         struct nlattr *params, *top;
5407         int rem, rc = 0;
5408
5409 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
5410         extack = cb->extack;
5411 #endif
5412         if (the_lnet.ln_refcount == 0) {
5413                 NL_SET_ERR_MSG(extack, "LNet stack down");
5414                 return -ENETDOWN;
5415         }
5416
5417         LIBCFS_ALLOC(nlist, sizeof(*nlist));
5418         if (!nlist)
5419                 return -ENOMEM;
5420
5421         nlist->lngl_net_id = LNET_NET_ANY;
5422         nlist->lngl_idx = 0;
5423         cb->args[0] = (long)nlist;
5424
5425         cb->min_dump_alloc = U16_MAX;
5426         if (!msg_len)
5427                 return 0;
5428
5429         params = genlmsg_data(gnlh);
5430         if (!(nla_type(params) & LN_SCALAR_ATTR_LIST)) {
5431                 NL_SET_ERR_MSG(extack, "invalid configuration");
5432                 return -EINVAL;
5433         }
5434
5435         nla_for_each_nested(top, params, rem) {
5436                 struct nlattr *net;
5437                 int rem2;
5438
5439                 nla_for_each_nested(net, top, rem2) {
5440                         char filter[LNET_NIDSTR_SIZE];
5441
5442                         if (nla_type(net) != LN_SCALAR_ATTR_VALUE ||
5443                             nla_strcmp(net, "net type") != 0)
5444                                 continue;
5445
5446                         net = nla_next(net, &rem2);
5447                         if (nla_type(net) != LN_SCALAR_ATTR_VALUE) {
5448                                 NL_SET_ERR_MSG(extack, "invalid config param");
5449                                 GOTO(report_err, rc = -EINVAL);
5450                         }
5451
5452                         rc = nla_strscpy(filter, net, sizeof(filter));
5453                         if (rc < 0) {
5454                                 NL_SET_ERR_MSG(extack, "failed to get param");
5455                                 GOTO(report_err, rc);
5456                         }
5457                         rc = 0;
5458
5459                         nlist->lngl_net_id = libcfs_str2net(filter);
5460                         if (nlist->lngl_net_id == LNET_NET_ANY) {
5461                                 NL_SET_ERR_MSG(extack, "cannot parse net");
5462                                 GOTO(report_err, rc = -ENOENT);
5463                         }
5464                 }
5465         }
5466 report_err:
5467         if (rc < 0)
5468                 lnet_net_show_done(cb);
5469
5470         return rc;
5471 }
5472
5473 static const struct ln_key_list net_update_props_list = {
5474         .lkl_maxattr                    = LNET_NET_ATTR_MAX,
5475         .lkl_list                       = {
5476                 [LNET_NET_ATTR_HDR]             = {
5477                         .lkp_value              = "",
5478                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
5479                         .lkp_data_type          = NLA_NUL_STRING,
5480                 },
5481                 [LNET_NET_ATTR_TYPE]            = {
5482                         .lkp_value              = "net type",
5483                         .lkp_data_type          = NLA_STRING
5484                 },
5485                 [LNET_NET_ATTR_LOCAL]           = {
5486                         .lkp_value              = "local NI(s)",
5487                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
5488                         .lkp_data_type          = NLA_NESTED
5489                 },
5490         },
5491 };
5492
5493 static int lnet_net_show_dump(struct sk_buff *msg,
5494                               struct netlink_callback *cb)
5495 {
5496         struct lnet_genl_net_list *nlist = lnet_net_dump_ctx(cb);
5497 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
5498         struct netlink_ext_ack *extack = NULL;
5499 #endif
5500         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
5501         int portid = NETLINK_CB(cb->skb).portid;
5502         bool found = false, started = false;
5503         const struct lnet_lnd *lnd = NULL;
5504         int idx = nlist->lngl_idx, rc = 0;
5505         int seq = cb->nlh->nlmsg_seq;
5506         struct lnet_net *net;
5507         void *hdr = NULL;
5508
5509 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
5510         extack = cb->extack;
5511 #endif
5512         lnet_net_lock(LNET_LOCK_EX);
5513
5514         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
5515                 struct nlattr *local_ni, *ni_attr;
5516                 struct lnet_ni *ni;
5517                 int dev = 0;
5518
5519                 if (nlist->lngl_net_id != LNET_NET_ANY &&
5520                     nlist->lngl_net_id != net->net_id)
5521                         continue;
5522
5523                 if (cb->nlh->nlmsg_flags & NLM_F_DUMP_FILTERED &&
5524                     LNET_NETTYP(net->net_id) == LOLND)
5525                         continue;
5526
5527                 if (gnlh->version && LNET_NETTYP(net->net_id) != LOLND) {
5528                         if (!net->net_lnd) {
5529                                 NL_SET_ERR_MSG(extack,
5530                                                "LND not setup for NI");
5531                                 GOTO(net_unlock, rc = -ENODEV);
5532                         }
5533                         if (net->net_lnd != lnd)
5534                                 lnd = net->net_lnd;
5535                         else
5536                                 lnd = NULL;
5537                 }
5538
5539                 /* We need to resend the key table every time the base LND
5540                  * changed.
5541                  */
5542                 if (!idx || lnd) {
5543                         const struct ln_key_list *all[] = {
5544                                 &net_props_list, &local_ni_list,
5545                                 &local_ni_interfaces_list,
5546                                 &local_ni_stats_list,
5547                                 &udsp_info_list,
5548                                 &udsp_info_pref_nids_list,
5549                                 &udsp_info_pref_nids_list,
5550                                 &local_ni_msg_stats_list,
5551                                 &local_ni_msg_stats_list,
5552                                 &local_ni_msg_stats_list,
5553                                 &local_ni_health_stats_list,
5554                                 &local_ni_tunables_list,
5555                                 NULL, /* lnd tunables */
5556                                 NULL
5557                         };
5558                         int flags = NLM_F_CREATE | NLM_F_MULTI;
5559
5560                         if (lnd) {
5561                                 all[ARRAY_SIZE(all) - 2] = lnd->lnd_keys;
5562                                 if (idx) {
5563                                         all[0] = &net_update_props_list;
5564                                         flags |= NLM_F_REPLACE;
5565                                 }
5566                         }
5567
5568                         rc = lnet_genl_send_scalar_list(msg, portid, seq,
5569                                                         &lnet_family, flags,
5570                                                         LNET_CMD_NETS, all);
5571                         if (rc < 0) {
5572                                 NL_SET_ERR_MSG(extack, "failed to send key table");
5573                                 GOTO(net_unlock, rc);
5574                         }
5575                         started = true;
5576                 }
5577
5578                 hdr = genlmsg_put(msg, portid, seq, &lnet_family,
5579                                   NLM_F_MULTI, LNET_CMD_NETS);
5580                 if (!hdr) {
5581                         NL_SET_ERR_MSG(extack, "failed to send values");
5582                         GOTO(net_unlock, rc = -EMSGSIZE);
5583                 }
5584
5585                 if (started) {
5586                         nla_put_string(msg, LNET_NET_ATTR_HDR, "");
5587                         started = false;
5588                 }
5589
5590                 nla_put_string(msg, LNET_NET_ATTR_TYPE,
5591                                libcfs_net2str(net->net_id));
5592
5593                 local_ni = nla_nest_start(msg, LNET_NET_ATTR_LOCAL);
5594                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
5595                         char *status = "up";
5596
5597                         if (idx++ < nlist->lngl_idx)
5598                                 continue;
5599
5600                         ni_attr = nla_nest_start(msg, dev++);
5601                         found = true;
5602                         lnet_ni_lock(ni);
5603                         if (!(cb->nlh->nlmsg_flags & NLM_F_DUMP_FILTERED)) {
5604                                 nla_put_string(msg, LNET_NET_LOCAL_NI_ATTR_NID,
5605                                                libcfs_nidstr(&ni->ni_nid));
5606                                 if (!nid_is_lo0(&ni->ni_nid) &&
5607                                     lnet_ni_get_status_locked(ni) != LNET_NI_STATUS_UP)
5608                                         status = "down";
5609                                 nla_put_string(msg, LNET_NET_LOCAL_NI_ATTR_STATUS,
5610                                                status);
5611                         }
5612
5613                         if (!nid_is_lo0(&ni->ni_nid) && ni->ni_interface) {
5614                                 struct nlattr *intf_nest, *intf_attr;
5615
5616                                 intf_nest = nla_nest_start(msg,
5617                                                            LNET_NET_LOCAL_NI_ATTR_INTERFACE);
5618                                 intf_attr = nla_nest_start(msg, 0);
5619                                 nla_put_string(msg,
5620                                                LNET_NET_LOCAL_NI_INTF_ATTR_TYPE,
5621                                                ni->ni_interface);
5622                                 nla_nest_end(msg, intf_attr);
5623                                 nla_nest_end(msg, intf_nest);
5624                         }
5625
5626                         if (gnlh->version) {
5627                                 char cpts[LNET_MAX_SHOW_NUM_CPT * 4 + 4], *cpt;
5628                                 struct lnet_ioctl_element_msg_stats msg_stats;
5629                                 struct lnet_ioctl_element_stats stats;
5630                                 size_t buf_len = sizeof(cpts), len;
5631                                 struct nlattr *health_attr, *health_stats;
5632                                 struct nlattr *send_attr, *send_stats;
5633                                 struct nlattr *recv_attr, *recv_stats;
5634                                 struct nlattr *drop_attr, *drop_stats;
5635                                 struct nlattr *stats_attr, *ni_stats;
5636                                 struct nlattr *tun_attr, *ni_tun;
5637                                 int j;
5638
5639                                 if (cb->nlh->nlmsg_flags & NLM_F_DUMP_FILTERED) {
5640                                         lnet_ni_unlock(ni);
5641                                         goto skip_msg_stats;
5642                                 }
5643
5644                                 stats.iel_send_count = lnet_sum_stats(&ni->ni_stats,
5645                                                                       LNET_STATS_TYPE_SEND);
5646                                 stats.iel_recv_count = lnet_sum_stats(&ni->ni_stats,
5647                                                                       LNET_STATS_TYPE_RECV);
5648                                 stats.iel_drop_count = lnet_sum_stats(&ni->ni_stats,
5649                                                                       LNET_STATS_TYPE_DROP);
5650                                 lnet_ni_unlock(ni);
5651
5652                                 stats_attr = nla_nest_start(msg, LNET_NET_LOCAL_NI_ATTR_STATS);
5653                                 ni_stats = nla_nest_start(msg, 0);
5654                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_STATS_ATTR_SEND_COUNT,
5655                                             stats.iel_send_count);
5656                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_STATS_ATTR_RECV_COUNT,
5657                                             stats.iel_recv_count);
5658                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_STATS_ATTR_DROP_COUNT,
5659                                             stats.iel_drop_count);
5660                                 nla_nest_end(msg, ni_stats);
5661                                 nla_nest_end(msg, stats_attr);
5662
5663                                 if (gnlh->version < 4)
5664                                         goto skip_udsp;
5665
5666                                 /* UDSP info */
5667                                 rc = lnet_udsp_info_send(msg, LNET_NET_LOCAL_NI_ATTR_UDSP_INFO,
5668                                                          &ni->ni_nid, false);
5669                                 if (rc < 0) {
5670                                         NL_SET_ERR_MSG(extack,
5671                                                        "Failed to get udsp info");
5672                                         genlmsg_cancel(msg, hdr);
5673                                         GOTO(net_unlock, rc = -ENOMEM);
5674                                 }
5675 skip_udsp:
5676                                 if (gnlh->version < 2)
5677                                         goto skip_msg_stats;
5678
5679                                 msg_stats.im_idx = idx - 1;
5680                                 rc = lnet_get_ni_stats(&msg_stats);
5681                                 if (rc < 0) {
5682                                         NL_SET_ERR_MSG(extack,
5683                                                        "failed to get msg stats");
5684                                         genlmsg_cancel(msg, hdr);
5685                                         GOTO(net_unlock, rc = -ENOMEM);
5686                                 }
5687
5688                                 send_stats = nla_nest_start(msg, LNET_NET_LOCAL_NI_ATTR_SEND_STATS);
5689                                 send_attr = nla_nest_start(msg, 0);
5690                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_PUT_COUNT,
5691                                             msg_stats.im_send_stats.ico_get_count);
5692                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_GET_COUNT,
5693                                             msg_stats.im_send_stats.ico_put_count);
5694                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_REPLY_COUNT,
5695                                             msg_stats.im_send_stats.ico_reply_count);
5696                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_ACK_COUNT,
5697                                             msg_stats.im_send_stats.ico_ack_count);
5698                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_HELLO_COUNT,
5699                                             msg_stats.im_send_stats.ico_hello_count);
5700                                 nla_nest_end(msg, send_attr);
5701                                 nla_nest_end(msg, send_stats);
5702
5703                                 recv_stats = nla_nest_start(msg, LNET_NET_LOCAL_NI_ATTR_RECV_STATS);
5704                                 recv_attr = nla_nest_start(msg, 0);
5705                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_PUT_COUNT,
5706                                             msg_stats.im_recv_stats.ico_get_count);
5707                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_GET_COUNT,
5708                                             msg_stats.im_recv_stats.ico_put_count);
5709                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_REPLY_COUNT,
5710                                             msg_stats.im_recv_stats.ico_reply_count);
5711                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_ACK_COUNT,
5712                                             msg_stats.im_recv_stats.ico_ack_count);
5713                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_HELLO_COUNT,
5714                                             msg_stats.im_recv_stats.ico_hello_count);
5715                                 nla_nest_end(msg, recv_attr);
5716                                 nla_nest_end(msg, recv_stats);
5717
5718                                 drop_stats = nla_nest_start(msg,
5719                                                             LNET_NET_LOCAL_NI_ATTR_DROPPED_STATS);
5720                                 drop_attr = nla_nest_start(msg, 0);
5721                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_PUT_COUNT,
5722                                             msg_stats.im_drop_stats.ico_get_count);
5723                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_GET_COUNT,
5724                                             msg_stats.im_drop_stats.ico_put_count);
5725                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_REPLY_COUNT,
5726                                             msg_stats.im_drop_stats.ico_reply_count);
5727                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_ACK_COUNT,
5728                                             msg_stats.im_drop_stats.ico_ack_count);
5729                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_MSG_STATS_ATTR_HELLO_COUNT,
5730                                             msg_stats.im_drop_stats.ico_hello_count);
5731                                 nla_nest_end(msg, drop_attr);
5732                                 nla_nest_end(msg, drop_stats);
5733
5734                                 /* health stats */
5735                                 health_stats = nla_nest_start(msg,
5736                                                               LNET_NET_LOCAL_NI_ATTR_HEALTH_STATS);
5737                                 health_attr = nla_nest_start(msg, 0);
5738                                 nla_put_s32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_FATAL_ERRORS,
5739                                             atomic_read(&ni->ni_fatal_error_on));
5740                                 nla_put_s32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_LEVEL,
5741                                             atomic_read(&ni->ni_healthv));
5742                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_INTERRUPTS,
5743                                             atomic_read(&ni->ni_hstats.hlt_local_interrupt));
5744                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_DROPPED,
5745                                             atomic_read(&ni->ni_hstats.hlt_local_dropped));
5746                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_ABORTED,
5747                                             atomic_read(&ni->ni_hstats.hlt_local_aborted));
5748                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_NO_ROUTE,
5749                                             atomic_read(&ni->ni_hstats.hlt_local_no_route));
5750                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_TIMEOUTS,
5751                                             atomic_read(&ni->ni_hstats.hlt_local_timeout));
5752                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_ERROR,
5753                                             atomic_read(&ni->ni_hstats.hlt_local_error));
5754                                 nla_put_u32(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_PING_COUNT,
5755                                             ni->ni_ping_count);
5756                                 nla_put_u64_64bit(msg, LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_NEXT_PING,
5757                                                   ni->ni_next_ping,
5758                                                   LNET_NET_LOCAL_NI_HEALTH_STATS_ATTR_PAD);
5759                                 nla_nest_end(msg, health_attr);
5760                                 nla_nest_end(msg, health_stats);
5761 skip_msg_stats:
5762                                 /* Report net tunables */
5763                                 tun_attr = nla_nest_start(msg, LNET_NET_LOCAL_NI_ATTR_TUNABLES);
5764                                 ni_tun = nla_nest_start(msg, 0);
5765                                 nla_put_s32(msg, LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_TIMEOUT,
5766                                             ni->ni_net->net_tunables.lct_peer_timeout);
5767                                 nla_put_s32(msg, LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_CREDITS,
5768                                             ni->ni_net->net_tunables.lct_peer_tx_credits);
5769                                 nla_put_s32(msg, LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_BUFFER_CREDITS,
5770                                             ni->ni_net->net_tunables.lct_peer_rtr_credits);
5771                                 nla_put_s32(msg, LNET_NET_LOCAL_NI_TUNABLES_ATTR_CREDITS,
5772                                             ni->ni_net->net_tunables.lct_max_tx_credits);
5773                                 nla_nest_end(msg, ni_tun);
5774
5775                                 nla_nest_end(msg, tun_attr);
5776
5777                                 if (lnd && lnd->lnd_nl_get && lnd->lnd_keys) {
5778                                         struct nlattr *lnd_tun_attr, *lnd_ni_tun;
5779
5780                                         lnd_tun_attr = nla_nest_start(msg,
5781                                                                       LNET_NET_LOCAL_NI_ATTR_LND_TUNABLES);
5782                                         lnd_ni_tun = nla_nest_start(msg, 0);
5783                                         rc = lnd->lnd_nl_get(LNET_CMD_NETS, msg,
5784                                                              LNET_NET_LOCAL_NI_ATTR_LND_TUNABLES,
5785                                                              ni);
5786                                         if (rc < 0) {
5787                                                 NL_SET_ERR_MSG(extack,
5788                                                                "failed to get lnd tunables");
5789                                                 genlmsg_cancel(msg, hdr);
5790                                                 GOTO(net_unlock, rc);
5791                                         }
5792                                         nla_nest_end(msg, lnd_ni_tun);
5793                                         nla_nest_end(msg, lnd_tun_attr);
5794                                 }
5795
5796                                 if (!(cb->nlh->nlmsg_flags & NLM_F_DUMP_FILTERED))
5797                                         nla_put_s32(msg, LNET_NET_LOCAL_NI_DEV_CPT,
5798                                                     ni->ni_dev_cpt);
5799
5800                                 /* Report cpts. We could send this as a nested list
5801                                  * of integers but older versions of the tools
5802                                  * except a string. The new versions can handle
5803                                  * both formats so in the future we can change
5804                                  * this to a nested list.
5805                                  */
5806                                 len = snprintf(cpts, buf_len, "\"[");
5807                                 cpt = cpts + len;
5808                                 buf_len -= len;
5809
5810                                 if (ni->ni_ncpts == LNET_CPT_NUMBER && !ni->ni_cpts)  {
5811                                         for (j = 0; j < ni->ni_ncpts; j++) {
5812                                                 len = snprintf(cpt, buf_len, "%d,", j);
5813                                                 buf_len -= len;
5814                                                 cpt += len;
5815                                         }
5816                                 } else {
5817                                         for (j = 0;
5818                                              ni->ni_cpts && j < ni->ni_ncpts &&
5819                                              j < LNET_MAX_SHOW_NUM_CPT; j++) {
5820                                                 len = snprintf(cpt, buf_len, "%d,",
5821                                                                ni->ni_cpts[j]);
5822                                                 buf_len -= len;
5823                                                 cpt += len;
5824                                         }
5825                                 }
5826                                 snprintf(cpt - 1, sizeof(cpts), "]\"");
5827
5828                                 nla_put_string(msg, LNET_NET_LOCAL_NI_CPTS, cpts);
5829                         } else {
5830                                 lnet_ni_unlock(ni);
5831                         }
5832                         nla_nest_end(msg, ni_attr);
5833                 }
5834                 nla_nest_end(msg, local_ni);
5835
5836                 genlmsg_end(msg, hdr);
5837         }
5838
5839         if (!found) {
5840                 struct nlmsghdr *nlh = nlmsg_hdr(msg);
5841
5842                 nlmsg_cancel(msg, nlh);
5843                 NL_SET_ERR_MSG(extack, "Network is down");
5844                 rc = -ESRCH;
5845         }
5846         nlist->lngl_idx = idx;
5847 net_unlock:
5848         lnet_net_unlock(LNET_LOCK_EX);
5849
5850         return lnet_nl_send_error(cb->skb, portid, seq, rc);
5851 }
5852
5853 #ifndef HAVE_NETLINK_CALLBACK_START
5854 static int lnet_old_net_show_dump(struct sk_buff *msg,
5855                                    struct netlink_callback *cb)
5856 {
5857         if (!cb->args[0]) {
5858                 int rc = lnet_net_show_start(cb);
5859
5860                 if (rc < 0)
5861                         return lnet_nl_send_error(cb->skb,
5862                                                   NETLINK_CB(cb->skb).portid,
5863                                                   cb->nlh->nlmsg_seq,
5864                                                   rc);
5865         }
5866
5867         return lnet_net_show_dump(msg, cb);
5868 }
5869 #endif
5870
5871 static int lnet_genl_parse_tunables(struct nlattr *settings,
5872                                     struct lnet_ioctl_config_lnd_tunables *tun)
5873 {
5874         struct nlattr *param;
5875         int rem, rc = 0;
5876
5877         nla_for_each_nested(param, settings, rem) {
5878                 int type = LNET_NET_LOCAL_NI_TUNABLES_ATTR_UNSPEC;
5879                 s64 num;
5880
5881                 if (nla_type(param) != LN_SCALAR_ATTR_VALUE)
5882                         continue;
5883
5884                 if (nla_strcmp(param, "peer_timeout") == 0)
5885                         type = LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_TIMEOUT;
5886                 else if (nla_strcmp(param, "peer_credits") == 0)
5887                         type = LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_CREDITS;
5888                 else if (nla_strcmp(param, "peer_buffer_credits") == 0)
5889                         type = LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_BUFFER_CREDITS;
5890                 else if (nla_strcmp(param, "credits") == 0)
5891                         type = LNET_NET_LOCAL_NI_TUNABLES_ATTR_CREDITS;
5892
5893                 param = nla_next(param, &rem);
5894                 if (nla_type(param) != LN_SCALAR_ATTR_INT_VALUE)
5895                         return -EINVAL;
5896
5897                 num = nla_get_s64(param);
5898                 switch (type) {
5899                 case LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_TIMEOUT:
5900                         if (num >= 0)
5901                                 tun->lt_cmn.lct_peer_timeout = num;
5902                         break;
5903                 case LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_CREDITS:
5904                         if (num > 0)
5905                                 tun->lt_cmn.lct_peer_tx_credits = num;
5906                         break;
5907                 case LNET_NET_LOCAL_NI_TUNABLES_ATTR_PEER_BUFFER_CREDITS:
5908                         if (num > 0)
5909                                 tun->lt_cmn.lct_peer_rtr_credits = num;
5910                         break;
5911                 case LNET_NET_LOCAL_NI_TUNABLES_ATTR_CREDITS:
5912                         if (num > 0)
5913                                 tun->lt_cmn.lct_max_tx_credits = num;
5914                         break;
5915                 default:
5916                         rc = -EINVAL;
5917                         break;
5918                 }
5919         }
5920         return rc;
5921 }
5922
5923 static int lnet_genl_parse_lnd_tunables(struct nlattr *settings,
5924                                         struct lnet_lnd_tunables *tun,
5925                                         const struct lnet_lnd *lnd)
5926 {
5927         const struct ln_key_list *list = lnd->lnd_keys;
5928         struct nlattr *param;
5929         int rem, rc = 0;
5930         int i = 1;
5931
5932         /* silently ignore these setting if the LND driver doesn't
5933          * support any LND tunables
5934          */
5935         if (!list || !lnd->lnd_nl_set || !list->lkl_maxattr)
5936                 return 0;
5937
5938         nla_for_each_nested(param, settings, rem) {
5939                 if (nla_type(param) != LN_SCALAR_ATTR_VALUE)
5940                         continue;
5941
5942                 for (i = 1; i <= list->lkl_maxattr; i++) {
5943                         if (!list->lkl_list[i].lkp_value ||
5944                             nla_strcmp(param, list->lkl_list[i].lkp_value) != 0)
5945                                 continue;
5946
5947                         param = nla_next(param, &rem);
5948                         rc = lnd->lnd_nl_set(LNET_CMD_NETS, param, i, tun);
5949                         if (rc < 0)
5950                                 return rc;
5951                 }
5952         }
5953
5954         return rc;
5955 }
5956
5957 static inline void
5958 lnet_genl_init_tunables(const struct lnet_lnd *lnd,
5959                         struct lnet_ioctl_config_lnd_tunables *tun)
5960 {
5961         const struct ln_key_list *list = lnd ? lnd->lnd_keys : NULL;
5962         int i;
5963
5964         tun->lt_cmn.lct_peer_timeout = -1;
5965         tun->lt_cmn.lct_peer_tx_credits = -1;
5966         tun->lt_cmn.lct_peer_rtr_credits = -1;
5967         tun->lt_cmn.lct_max_tx_credits = -1;
5968
5969         if (!list || !lnd->lnd_nl_set || !list->lkl_maxattr)
5970                 return;
5971
5972         /* init lnd tunables with default values */
5973         for (i = 1; i <= list->lkl_maxattr; i++)
5974                 lnd->lnd_nl_set(LNET_CMD_NETS, NULL, i, &tun->lt_tun);
5975 }
5976
5977 static int
5978 lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info,
5979                          int net_id, struct lnet_ioctl_config_ni *conf,
5980                          bool *ni_list)
5981 {
5982         struct lnet_ioctl_config_lnd_tunables *tun;
5983         struct lnet_nid nid = LNET_ANY_NID;
5984         const struct lnet_lnd *lnd = NULL;
5985         struct nlattr *settings;
5986         int healthv = -1;
5987         int rem3, rc = 0;
5988
5989         if (net_id != LNET_NET_ANY) {
5990                 lnd = lnet_load_lnd(LNET_NETTYP(net_id));
5991                 if (IS_ERR(lnd)) {
5992                         GENL_SET_ERR_MSG(info, "LND type not supported");
5993                         RETURN(PTR_ERR(lnd));
5994                 }
5995         }
5996
5997         LIBCFS_ALLOC(tun, sizeof(struct lnet_ioctl_config_lnd_tunables));
5998         if (!tun) {
5999                 GENL_SET_ERR_MSG(info, "cannot allocate memory for tunables");
6000                 GOTO(out, rc = -ENOMEM);
6001         }
6002
6003         /* Use LND defaults */
6004         lnet_genl_init_tunables(lnd, tun);
6005         conf->lic_ncpts = 0;
6006
6007         nla_for_each_nested(settings, entry, rem3) {
6008                 if (nla_type(settings) != LN_SCALAR_ATTR_VALUE)
6009                         continue;
6010
6011                 if (nla_strcmp(settings, "interfaces") == 0) {
6012                         struct nlattr *intf;
6013                         int rem4;
6014
6015                         settings = nla_next(settings, &rem3);
6016                         if (nla_type(settings) !=
6017                             LN_SCALAR_ATTR_LIST) {
6018                                 GENL_SET_ERR_MSG(info,
6019                                                  "invalid interfaces");
6020                                 GOTO(out, rc = -EINVAL);
6021                         }
6022
6023                         nla_for_each_nested(intf, settings, rem4) {
6024                                 intf = nla_next(intf, &rem4);
6025                                 if (nla_type(intf) !=
6026                                     LN_SCALAR_ATTR_VALUE) {
6027                                         GENL_SET_ERR_MSG(info,
6028                                                          "cannot parse interface");
6029                                         GOTO(out, rc = -EINVAL);
6030                                 }
6031
6032                                 rc = nla_strscpy(conf->lic_ni_intf, intf,
6033                                                  sizeof(conf->lic_ni_intf));
6034                                 if (rc < 0) {
6035                                         GENL_SET_ERR_MSG(info,
6036                                                          "failed to parse interfaces");
6037                                         GOTO(out, rc);
6038                                 }
6039                         }
6040                         *ni_list = true;
6041                 } else if (nla_strcmp(settings, "nid") == 0 &&
6042                            net_id != LNET_NET_ANY) {
6043                         char nidstr[LNET_NIDSTR_SIZE];
6044
6045                         settings = nla_next(settings, &rem3);
6046                         if (nla_type(settings) != LN_SCALAR_ATTR_VALUE) {
6047                                 GENL_SET_ERR_MSG(info, "cannot parse NID");
6048                                 GOTO(out, rc = -EINVAL);
6049                         }
6050
6051                         rc = nla_strscpy(nidstr, settings, sizeof(nidstr));
6052                         if (rc < 0) {
6053                                 GENL_SET_ERR_MSG(info,
6054                                                  "failed to parse NID");
6055                                 GOTO(out, rc);
6056                         }
6057
6058                         CDEBUG(D_NET, "Requested NID %s\n", nidstr);
6059                         rc = libcfs_strnid(&nid, strim(nidstr));
6060                         if (rc < 0) {
6061                                 GENL_SET_ERR_MSG(info, "unsupported NID");
6062                                 GOTO(out, rc);
6063                         }
6064
6065                         if (!(info->nlhdr->nlmsg_flags & NLM_F_REPLACE) &&
6066                              nid_same(&nid, &LNET_ANY_NID)) {
6067                                 GENL_SET_ERR_MSG(info, "any NID not supported");
6068                                 GOTO(out, rc = -EINVAL);
6069                         }
6070                         *ni_list = true;
6071                 } else if (nla_strcmp(settings, "health stats") == 0 &&
6072                            info->nlhdr->nlmsg_flags & NLM_F_REPLACE) {
6073                         struct nlattr *health;
6074                         int rem4;
6075
6076                         settings = nla_next(settings, &rem3);
6077                         if (nla_type(settings) != LN_SCALAR_ATTR_LIST) {
6078                                 GENL_SET_ERR_MSG(info,
6079                                                  "cannot parse health stats");
6080                                 GOTO(out, rc = -EINVAL);
6081                         }
6082
6083                         nla_for_each_nested(health, settings, rem4) {
6084                                 if (nla_type(health) != LN_SCALAR_ATTR_VALUE ||
6085                                     nla_strcmp(health, "health value") != 0) {
6086                                         GENL_SET_ERR_MSG(info,
6087                                                          "wrong health config format");
6088                                         GOTO(out, rc = -EINVAL);
6089                                 }
6090
6091                                 health = nla_next(health, &rem4);
6092                                 if (nla_type(health) !=
6093                                     LN_SCALAR_ATTR_INT_VALUE) {
6094                                         GENL_SET_ERR_MSG(info,
6095                                                          "invalid health config format");
6096                                         GOTO(out, rc = -EINVAL);
6097                                 }
6098
6099                                 healthv = nla_get_s64(health);
6100                                 clamp_t(s64, healthv, 0, LNET_MAX_HEALTH_VALUE);
6101                         }
6102                 } else if (nla_strcmp(settings, "tunables") == 0) {
6103                         settings = nla_next(settings, &rem3);
6104                         if (nla_type(settings) !=
6105                             LN_SCALAR_ATTR_LIST) {
6106                                 GENL_SET_ERR_MSG(info,
6107                                                  "invalid tunables");
6108                                 GOTO(out, rc = -EINVAL);
6109                         }
6110
6111                         rc = lnet_genl_parse_tunables(settings, tun);
6112                         if (rc < 0) {
6113                                 GENL_SET_ERR_MSG(info,
6114                                                  "failed to parse tunables");
6115                                 GOTO(out, rc);
6116                         }
6117                 } else if ((nla_strcmp(settings, "lnd tunables") == 0)) {
6118                         settings = nla_next(settings, &rem3);
6119                         if (nla_type(settings) !=
6120                             LN_SCALAR_ATTR_LIST) {
6121                                 GENL_SET_ERR_MSG(info,
6122                                                  "lnd tunables should be list\n");
6123                                 GOTO(out, rc = -EINVAL);
6124                         }
6125
6126                         if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE &&
6127                             net_id == LNET_NET_ANY) {
6128                                 struct lnet_net *net;
6129
6130                                 lnet_net_lock(LNET_LOCK_EX);
6131                                 list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
6132                                         struct lnet_ni *ni;
6133
6134                                         if (!net->net_lnd)
6135                                                 continue;
6136
6137                                         list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
6138                                                 if (!nid_same(&nid, &LNET_ANY_NID) &&
6139                                                     !nid_same(&nid, &ni->ni_nid))
6140                                                         continue;
6141
6142                                                 rc = lnet_genl_parse_lnd_tunables(settings,
6143                                                                                   &ni->ni_lnd_tunables,
6144                                                                                   net->net_lnd);
6145                                                 if (rc < 0) {
6146                                                         GENL_SET_ERR_MSG(info,
6147                                                                          "failed to parse lnd tunables");
6148                                                         lnet_net_unlock(LNET_LOCK_EX);
6149                                                         GOTO(out, rc);
6150                                                 }
6151                                         }
6152                                 }
6153                                 lnet_net_unlock(LNET_LOCK_EX);
6154                         } else {
6155                                 lnd = lnet_load_lnd(LNET_NETTYP(net_id));
6156                                 if (IS_ERR(lnd)) {
6157                                         GENL_SET_ERR_MSG(info,
6158                                                          "LND type not supported");
6159                                         GOTO(out, rc = PTR_ERR(lnd));
6160                                 }
6161
6162                                 rc = lnet_genl_parse_lnd_tunables(settings,
6163                                                                   &tun->lt_tun, lnd);
6164                                 if (rc < 0) {
6165                                         GENL_SET_ERR_MSG(info,
6166                                                          "failed to parse lnd tunables");
6167                                         GOTO(out, rc);
6168                                 }
6169                         }
6170                 } else if (nla_strcmp(settings, "CPT") == 0) {
6171                         struct nlattr *cpt;
6172                         int rem4;
6173
6174                         settings = nla_next(settings, &rem3);
6175                         if (nla_type(settings) != LN_SCALAR_ATTR_LIST) {
6176                                 GENL_SET_ERR_MSG(info,
6177                                                  "CPT should be list");
6178                                 GOTO(out, rc = -EINVAL);
6179                         }
6180
6181                         nla_for_each_nested(cpt, settings, rem4) {
6182                                 s64 core;
6183
6184                                 if (nla_type(cpt) !=
6185                                     LN_SCALAR_ATTR_INT_VALUE) {
6186                                         GENL_SET_ERR_MSG(info,
6187                                                          "invalid CPT config");
6188                                         GOTO(out, rc = -EINVAL);
6189                                 }
6190
6191                                 core = nla_get_s64(cpt);
6192                                 if (core >= LNET_CPT_NUMBER) {
6193                                         GENL_SET_ERR_MSG(info,
6194                                                          "invalid CPT value");
6195                                         GOTO(out, rc = -ERANGE);
6196                                 }
6197
6198                                 conf->lic_cpts[conf->lic_ncpts] = core;
6199                                 conf->lic_ncpts++;
6200                         }
6201                 }
6202         }
6203
6204         if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
6205                 if (nid_same(&nid, &LNET_ANY_NID) &&
6206                     !strlen(conf->lic_ni_intf)) {
6207                         GENL_SET_ERR_MSG(info,
6208                                          "interface / NID is missing");
6209                         GOTO(out, rc);
6210                 }
6211
6212                 rc = lnet_dyn_add_ni(conf, net_id, &nid, tun);
6213                 switch (rc) {
6214                 case -ENOENT:
6215                         GENL_SET_ERR_MSG(info,
6216                                          "cannot parse net");
6217                         break;
6218                 case -ERANGE:
6219                         GENL_SET_ERR_MSG(info,
6220                                          "invalid CPT set");
6221                         break;
6222                 default:
6223                         GENL_SET_ERR_MSG(info,
6224                                          "cannot add LNet NI");
6225                 case 0:
6226                         break;
6227                 }
6228         } else if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE && healthv != -1) {
6229                 lnet_ni_set_healthv(&nid, healthv);
6230         } else if (!(info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_REPLACE))) {
6231                 struct lnet_ni *ni;
6232
6233                 /* delete case */
6234                 rc = -ENODEV;
6235                 if (!strlen(conf->lic_ni_intf) &&
6236                     nid_same(&nid, &LNET_ANY_NID)) {
6237                         GENL_SET_ERR_MSG(info,
6238                                          "interface / NID is missing");
6239                         GOTO(out, rc);
6240                 }
6241
6242                 if (nid_same(&nid, &LNET_ANY_NID)) {
6243                         struct lnet_net *net;
6244                         bool found = false;
6245
6246                         lnet_net_lock(LNET_LOCK_EX);
6247                         net = lnet_get_net_locked(net_id);
6248                         if (!net) {
6249                                 GENL_SET_ERR_MSG(info,
6250                                                  "LNet net doesn't exist");
6251                                 lnet_net_unlock(LNET_LOCK_EX);
6252                                 GOTO(out, rc);
6253                         }
6254
6255                         list_for_each_entry(ni, &net->net_ni_list,
6256                                             ni_netlist) {
6257                                 if (!ni->ni_interface ||
6258                                     strcmp(ni->ni_interface,
6259                                           conf->lic_ni_intf) != 0)
6260                                         continue;
6261
6262                                 found = true;
6263                                 lnet_net_unlock(LNET_LOCK_EX);
6264                                 rc = lnet_dyn_del_ni(&ni->ni_nid);
6265                                 break;
6266                         }
6267
6268                         if (rc < 0 && !found) { /* will be -ENODEV */
6269                                 GENL_SET_ERR_MSG(info,
6270                                                  "interface invalid for deleting LNet NI");
6271                                 lnet_net_unlock(LNET_LOCK_EX);
6272                         }
6273                 } else {
6274                         rc = lnet_dyn_del_ni(&nid);
6275                 }
6276
6277                 if (rc < 0) {
6278                         GENL_SET_ERR_MSG(info,
6279                                          "cannot del LNet NI");
6280                         GOTO(out, rc);
6281                 }
6282         }
6283 out:
6284         if (tun)
6285                 LIBCFS_FREE(tun, sizeof(struct lnet_ioctl_config_lnd_tunables));
6286
6287         return rc;
6288 }
6289
6290 static int lnet_net_cmd(struct sk_buff *skb, struct genl_info *info)
6291 {
6292         struct nlmsghdr *nlh = nlmsg_hdr(skb);
6293         struct genlmsghdr *gnlh = nlmsg_data(nlh);
6294         struct nlattr *params = genlmsg_data(gnlh);
6295         int msg_len, rem, rc = 0;
6296         struct nlattr *attr;
6297
6298         msg_len = genlmsg_len(gnlh);
6299         if (!msg_len) {
6300                 GENL_SET_ERR_MSG(info, "no configuration");
6301                 return -ENOMSG;
6302         }
6303
6304         if (!(nla_type(params) & LN_SCALAR_ATTR_LIST)) {
6305                 GENL_SET_ERR_MSG(info, "invalid configuration");
6306                 return -EINVAL;
6307         }
6308
6309         nla_for_each_nested(attr, params, rem) {
6310                 bool ni_list = false, ipnets = false;
6311                 struct lnet_ioctl_config_ni conf;
6312                 u32 net_id = LNET_NET_ANY;
6313                 struct nlattr *entry;
6314                 int rem2;
6315
6316                 if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
6317                         continue;
6318
6319                 nla_for_each_nested(entry, attr, rem2) {
6320                         switch (nla_type(entry)) {
6321                         case LN_SCALAR_ATTR_VALUE: {
6322                                 ssize_t len;
6323
6324                                 memset(&conf, 0, sizeof(conf));
6325                                 if (nla_strcmp(entry, "ip2net") == 0) {
6326                                         entry = nla_next(entry, &rem2);
6327                                         if (nla_type(entry) !=
6328                                             LN_SCALAR_ATTR_VALUE) {
6329                                                 GENL_SET_ERR_MSG(info,
6330                                                                  "ip2net has invalid key");
6331                                                 GOTO(out, rc = -EINVAL);
6332                                         }
6333
6334                                         len = nla_strscpy(conf.lic_legacy_ip2nets,
6335                                                           entry,
6336                                                           sizeof(conf.lic_legacy_ip2nets));
6337                                         if (len < 0) {
6338                                                 GENL_SET_ERR_MSG(info,
6339                                                                  "ip2net key string is invalid");
6340                                                 GOTO(out, rc = len);
6341                                         }
6342                                         ni_list = true;
6343                                         ipnets = true;
6344                                 } else if (nla_strcmp(entry, "net type") == 0) {
6345                                         char tmp[LNET_NIDSTR_SIZE];
6346
6347                                         entry = nla_next(entry, &rem2);
6348                                         if (nla_type(entry) !=
6349                                             LN_SCALAR_ATTR_VALUE) {
6350                                                 GENL_SET_ERR_MSG(info,
6351                                                                  "net type has invalid key");
6352                                                 GOTO(out, rc = -EINVAL);
6353                                         }
6354
6355                                         len = nla_strscpy(tmp, entry,
6356                                                           sizeof(tmp));
6357                                         if (len < 0) {
6358                                                 GENL_SET_ERR_MSG(info,
6359                                                                  "net type key string is invalid");
6360                                                 GOTO(out, rc = len);
6361                                         }
6362
6363                                         net_id = libcfs_str2net(tmp);
6364                                         if (!net_id) {
6365                                                 GENL_SET_ERR_MSG(info,
6366                                                                  "cannot parse net");
6367                                                 GOTO(out, rc = -ENODEV);
6368                                         }
6369                                         if (LNET_NETTYP(net_id) == LOLND) {
6370                                                 GENL_SET_ERR_MSG(info,
6371                                                                  "setting @lo not allowed");
6372                                                 GOTO(out, rc = -ENODEV);
6373                                         }
6374                                         conf.lic_legacy_ip2nets[0] = '\0';
6375                                         conf.lic_ni_intf[0] = '\0';
6376                                         ni_list = false;
6377                                 }
6378                                 if (rc < 0)
6379                                         GOTO(out, rc);
6380                                 break;
6381                         }
6382                         case LN_SCALAR_ATTR_LIST: {
6383                                 struct nlattr *interface;
6384                                 int rem3;
6385
6386                                 ipnets = false;
6387                                 nla_for_each_nested(interface, entry, rem3) {
6388                                         rc = lnet_genl_parse_local_ni(interface, info,
6389                                                                       net_id, &conf,
6390                                                                       &ni_list);
6391                                         if (rc < 0)
6392                                                 GOTO(out, rc);
6393                                 }
6394                                 break;
6395                         }
6396                         /* it is possible a newer version of the user land send
6397                          * values older kernels doesn't handle. So silently
6398                          * ignore these values
6399                          */
6400                         default:
6401                                 break;
6402                         }
6403                 }
6404
6405                 /* Handle case of just sent NET with no list of NIDs */
6406                 if (!(info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_REPLACE)) &&
6407                     !ni_list) {
6408                         rc = lnet_dyn_del_net(net_id);
6409                         if (rc < 0) {
6410                                 GENL_SET_ERR_MSG(info,
6411                                                  "cannot del network");
6412                         }
6413                 } else if ((info->nlhdr->nlmsg_flags & NLM_F_CREATE) &&
6414                            ipnets && ni_list) {
6415                         rc = lnet_handle_legacy_ip2nets(conf.lic_legacy_ip2nets,
6416                                                         NULL);
6417                         if (rc < 0)
6418                                 GENL_SET_ERR_MSG(info,
6419                                                  "cannot setup ip2nets");
6420                 }
6421         }
6422 out:
6423         return rc;
6424 }
6425
6426 /* Called with ln_api_mutex */
6427 static int lnet_parse_peer_nis(struct nlattr *rlist, struct genl_info *info,
6428                                struct lnet_nid *pnid, bool mr,
6429                                bool *create_some)
6430 {
6431         struct lnet_nid snid = LNET_ANY_NID;
6432         struct nlattr *props;
6433         int rem, rc = 0;
6434         s64 num = -1;
6435
6436         nla_for_each_nested(props, rlist, rem) {
6437                 if (nla_type(props) != LN_SCALAR_ATTR_VALUE)
6438                         continue;
6439
6440                 if (nla_strcmp(props, "nid") == 0) {
6441                         char nidstr[LNET_NIDSTR_SIZE];
6442
6443                         props = nla_next(props, &rem);
6444                         if (nla_type(props) != LN_SCALAR_ATTR_VALUE) {
6445                                 GENL_SET_ERR_MSG(info,
6446                                                  "invalid secondary NID");
6447                                 GOTO(report_err, rc = -EINVAL);
6448                         }
6449
6450                         rc = nla_strscpy(nidstr, props, sizeof(nidstr));
6451                         if (rc < 0) {
6452                                 GENL_SET_ERR_MSG(info,
6453                                                  "failed to get secondary NID");
6454                                 GOTO(report_err, rc);
6455                         }
6456
6457                         rc = libcfs_strnid(&snid, strim(nidstr));
6458                         if (rc < 0) {
6459                                 GENL_SET_ERR_MSG(info, "unsupported secondary NID");
6460                                 GOTO(report_err, rc);
6461                         }
6462
6463                         if (LNET_NID_IS_ANY(&snid) || nid_same(&snid, pnid))
6464                                 *create_some = false;
6465                 } else if (nla_strcmp(props, "health stats") == 0) {
6466                         struct nlattr *health;
6467                         int rem2;
6468
6469                         props = nla_next(props, &rem);
6470                         if (nla_type(props) !=
6471                               LN_SCALAR_ATTR_LIST) {
6472                                 GENL_SET_ERR_MSG(info,
6473                                                  "invalid health configuration");
6474                                 GOTO(report_err, rc = -EINVAL);
6475                         }
6476
6477                         nla_for_each_nested(health, props, rem2) {
6478                                 if (nla_type(health) != LN_SCALAR_ATTR_VALUE ||
6479                                     nla_strcmp(health, "health value") != 0) {
6480                                         GENL_SET_ERR_MSG(info,
6481                                                          "wrong health config format");
6482                                         GOTO(report_err, rc = -EINVAL);
6483                                 }
6484
6485                                 health = nla_next(health, &rem2);
6486                                 if (nla_type(health) !=
6487                                     LN_SCALAR_ATTR_INT_VALUE) {
6488                                         GENL_SET_ERR_MSG(info,
6489                                                          "invalid health config format");
6490                                         GOTO(report_err, rc = -EINVAL);
6491                                 }
6492
6493                                 num = nla_get_s64(health);
6494                                 clamp_t(s64, num, 0, LNET_MAX_HEALTH_VALUE);
6495                         }
6496                 }
6497         }
6498
6499         if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE && num != -1) {
6500                 lnet_peer_ni_set_healthv(pnid, num, !*create_some);
6501         } else if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
6502                 bool lock_prim = info->nlhdr->nlmsg_flags & NLM_F_EXCL;
6503
6504                 rc = lnet_user_add_peer_ni(pnid, &snid, mr, lock_prim);
6505                 if (rc < 0)
6506                         GENL_SET_ERR_MSG(info,
6507                                          "failed to add peer");
6508         } else if (!(info->nlhdr->nlmsg_flags & NLM_F_CREATE) && *create_some) {
6509                 bool force = info->nlhdr->nlmsg_flags & NLM_F_EXCL;
6510
6511                 rc = lnet_del_peer_ni(pnid, &snid, force);
6512                 if (rc < 0)
6513                         GENL_SET_ERR_MSG(info,
6514                                          "failed to del peer");
6515         }
6516 report_err:
6517         return rc;
6518 }
6519
6520 static int lnet_peer_ni_cmd(struct sk_buff *skb, struct genl_info *info)
6521 {
6522         struct nlmsghdr *nlh = nlmsg_hdr(skb);
6523         struct genlmsghdr *gnlh = nlmsg_data(nlh);
6524         struct nlattr *params = genlmsg_data(gnlh);
6525         int msg_len, rem, rc = 0;
6526         struct lnet_nid pnid;
6527         struct nlattr *attr;
6528
6529         mutex_lock(&the_lnet.ln_api_mutex);
6530         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
6531                 GENL_SET_ERR_MSG(info, "Network is down");
6532                 mutex_unlock(&the_lnet.ln_api_mutex);
6533                 return -ENETDOWN;
6534         }
6535
6536         msg_len = genlmsg_len(gnlh);
6537         if (!msg_len) {
6538                 GENL_SET_ERR_MSG(info, "no configuration");
6539                 mutex_unlock(&the_lnet.ln_api_mutex);
6540                 return -ENOMSG;
6541         }
6542
6543         if (!(nla_type(params) & LN_SCALAR_ATTR_LIST)) {
6544                 GENL_SET_ERR_MSG(info, "invalid configuration");
6545                 mutex_unlock(&the_lnet.ln_api_mutex);
6546                 return -EINVAL;
6547         }
6548
6549         nla_for_each_nested(attr, params, rem) {
6550                 bool parse_peer_nis = false;
6551                 struct nlattr *pnid_prop;
6552                 int rem2;
6553
6554                 if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
6555                         continue;
6556
6557                 pnid = LNET_ANY_NID;
6558                 nla_for_each_nested(pnid_prop, attr, rem2) {
6559                         bool mr = true;
6560
6561                         if (nla_type(pnid_prop) != LN_SCALAR_ATTR_VALUE)
6562                                 continue;
6563
6564                         if (nla_strcmp(pnid_prop, "primary nid") == 0) {
6565                                 char nidstr[LNET_NIDSTR_SIZE];
6566
6567                                 pnid_prop = nla_next(pnid_prop, &rem2);
6568                                 if (nla_type(pnid_prop) !=
6569                                     LN_SCALAR_ATTR_VALUE) {
6570                                         GENL_SET_ERR_MSG(info,
6571                                                           "invalid primary NID type");
6572                                         GOTO(report_err, rc = -EINVAL);
6573                                 }
6574
6575                                 rc = nla_strscpy(nidstr, pnid_prop,
6576                                                  sizeof(nidstr));
6577                                 if (rc < 0) {
6578                                         GENL_SET_ERR_MSG(info,
6579                                                          "failed to get primary NID");
6580                                         GOTO(report_err, rc);
6581                                 }
6582
6583                                 rc = libcfs_strnid(&pnid, strim(nidstr));
6584                                 if (rc < 0) {
6585                                         GENL_SET_ERR_MSG(info,
6586                                                          "unsupported primary NID");
6587                                         GOTO(report_err, rc);
6588                                 }
6589
6590                                 /* we must create primary NID for peer ni
6591                                  * creation
6592                                  */
6593                                 if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
6594                                         bool lock_prim;
6595
6596                                         lock_prim = info->nlhdr->nlmsg_flags & NLM_F_EXCL;
6597                                         rc = lnet_user_add_peer_ni(&pnid,
6598                                                                    &LNET_ANY_NID,
6599                                                                    true, lock_prim);
6600                                         if (rc < 0) {
6601                                                 GENL_SET_ERR_MSG(info,
6602                                                                  "failed to add primary peer");
6603                                                 GOTO(report_err, rc);
6604                                         }
6605                                 }
6606                         } else if (nla_strcmp(pnid_prop, "Multi-Rail") == 0) {
6607                                 pnid_prop = nla_next(pnid_prop, &rem2);
6608                                 if (nla_type(pnid_prop) !=
6609                                     LN_SCALAR_ATTR_INT_VALUE) {
6610                                         GENL_SET_ERR_MSG(info,
6611                                                           "invalid MR flag param");
6612                                         GOTO(report_err, rc = -EINVAL);
6613                                 }
6614
6615                                 if (nla_get_s64(pnid_prop) == 0)
6616                                         mr = false;
6617                         } else if (nla_strcmp(pnid_prop, "peer state") == 0) {
6618                                 struct lnet_peer_ni *lpni;
6619                                 struct lnet_peer *lp;
6620
6621                                 pnid_prop = nla_next(pnid_prop, &rem2);
6622                                 if (nla_type(pnid_prop) !=
6623                                     LN_SCALAR_ATTR_INT_VALUE) {
6624                                         GENL_SET_ERR_MSG(info,
6625                                                           "invalid peer state param");
6626                                         GOTO(report_err, rc = -EINVAL);
6627                                 }
6628
6629                                 lpni = lnet_peer_ni_find_locked(&pnid);
6630                                 if (!lpni) {
6631                                         GENL_SET_ERR_MSG(info,
6632                                                           "invalid peer state param");
6633                                         GOTO(report_err, rc = -ENOENT);
6634                                 }
6635                                 lnet_peer_ni_decref_locked(lpni);
6636                                 lp = lpni->lpni_peer_net->lpn_peer;
6637                                 lp->lp_state = nla_get_s64(pnid_prop);
6638                         } else if (nla_strcmp(pnid_prop, "peer ni") == 0) {
6639                                 struct nlattr *rlist;
6640                                 int rem3;
6641
6642                                 if (!(info->nlhdr->nlmsg_flags & NLM_F_REPLACE) &&
6643                                     LNET_NID_IS_ANY(&pnid)) {
6644                                         GENL_SET_ERR_MSG(info,
6645                                                          "missing required primary NID");
6646                                         GOTO(report_err, rc);
6647                                 }
6648
6649                                 pnid_prop = nla_next(pnid_prop, &rem2);
6650                                 if (nla_type(pnid_prop) !=
6651                                     LN_SCALAR_ATTR_LIST) {
6652                                         GENL_SET_ERR_MSG(info,
6653                                                           "invalid NIDs list");
6654                                         GOTO(report_err, rc = -EINVAL);
6655                                 }
6656
6657                                 parse_peer_nis = true;
6658                                 nla_for_each_nested(rlist, pnid_prop, rem3) {
6659                                         rc = lnet_parse_peer_nis(rlist, info,
6660                                                                  &pnid, mr,
6661                                                                  &parse_peer_nis);
6662                                         if (rc < 0)
6663                                                 GOTO(report_err, rc);
6664                                 }
6665                         }
6666                 }
6667
6668                 /* If we have remote peer ni's we already add /del peers */
6669                 if (parse_peer_nis)
6670                         continue;
6671
6672                 if (LNET_NID_IS_ANY(&pnid)) {
6673                         GENL_SET_ERR_MSG(info, "missing primary NID");
6674                         GOTO(report_err, rc);
6675                 }
6676
6677                 if (!(info->nlhdr->nlmsg_flags & NLM_F_CREATE)) {
6678                         bool force = info->nlhdr->nlmsg_flags & NLM_F_EXCL;
6679
6680                         rc = lnet_del_peer_ni(&pnid, &LNET_ANY_NID,
6681                                               force);
6682                         if (rc < 0) {
6683                                 GENL_SET_ERR_MSG(info,
6684                                                  "failed to del primary peer");
6685                                 GOTO(report_err, rc);
6686                         }
6687                 }
6688         }
6689 report_err:
6690         /* If we failed on creation and encounter a latter error then
6691          * delete the primary nid.
6692          */
6693         if (rc < 0 && info->nlhdr->nlmsg_flags & NLM_F_CREATE &&
6694             !LNET_NID_IS_ANY(&pnid))
6695                 lnet_del_peer_ni(&pnid, &LNET_ANY_NID,
6696                                  info->nlhdr->nlmsg_flags & NLM_F_EXCL);
6697         mutex_unlock(&the_lnet.ln_api_mutex);
6698
6699         return rc;
6700 }
6701
6702 /** LNet route handling */
6703
6704 /* We can't use struct lnet_ioctl_config_data since it lacks
6705  * support for large NIDS
6706  */
6707 struct lnet_route_properties {
6708         struct lnet_nid         lrp_gateway;
6709         u32                     lrp_net;
6710         s32                     lrp_hop;
6711         u32                     lrp_flags;
6712         u32                     lrp_priority;
6713         u32                     lrp_sensitivity;
6714 };
6715
6716 struct lnet_genl_route_list {
6717         unsigned int                            lgrl_index;
6718         unsigned int                            lgrl_count;
6719         GENRADIX(struct lnet_route_properties)  lgrl_list;
6720 };
6721
6722 static inline struct lnet_genl_route_list *
6723 lnet_route_dump_ctx(struct netlink_callback *cb)
6724 {
6725         return (struct lnet_genl_route_list *)cb->args[0];
6726 }
6727
6728 static int lnet_route_show_done(struct netlink_callback *cb)
6729 {
6730         struct lnet_genl_route_list *rlist = lnet_route_dump_ctx(cb);
6731
6732         if (rlist) {
6733                 genradix_free(&rlist->lgrl_list);
6734                 CFS_FREE_PTR(rlist);
6735         }
6736         cb->args[0] = 0;
6737
6738         return 0;
6739 }
6740
6741 static int lnet_scan_route(struct lnet_genl_route_list *rlist,
6742                     struct lnet_route_properties *settings)
6743 {
6744         struct lnet_remotenet *rnet;
6745         struct list_head *rn_list;
6746         struct lnet_route *route;
6747         int cpt, i, rc = 0;
6748
6749         cpt = lnet_net_lock_current();
6750
6751         for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++) {
6752                 rn_list = &the_lnet.ln_remote_nets_hash[i];
6753                 list_for_each_entry(rnet, rn_list, lrn_list) {
6754                         if (settings->lrp_net != LNET_NET_ANY &&
6755                             settings->lrp_net != rnet->lrn_net)
6756                                 continue;
6757
6758                         list_for_each_entry(route, &rnet->lrn_routes,
6759                                             lr_list) {
6760                                 struct lnet_route_properties *prop;
6761
6762                                 if (!LNET_NID_IS_ANY(&settings->lrp_gateway) &&
6763                                     !nid_same(&settings->lrp_gateway,
6764                                               &route->lr_nid)) {
6765                                         continue;
6766                                 }
6767
6768                                 if (settings->lrp_hop != -1 &&
6769                                     settings->lrp_hop != route->lr_hops)
6770                                         continue;
6771
6772                                 if (settings->lrp_priority != -1 &&
6773                                     settings->lrp_priority != route->lr_priority)
6774                                         continue;
6775
6776                                 if (settings->lrp_sensitivity != -1 &&
6777                                     settings->lrp_sensitivity !=
6778                                     route->lr_gateway->lp_health_sensitivity)
6779                                         continue;
6780
6781                                 prop = genradix_ptr_alloc(&rlist->lgrl_list,
6782                                                           rlist->lgrl_count++,
6783                                                           GFP_ATOMIC);
6784                                 if (!prop)
6785                                         GOTO(failed_alloc, rc = -ENOMEM);
6786
6787                                 prop->lrp_net = rnet->lrn_net;
6788                                 prop->lrp_gateway = route->lr_nid;
6789                                 prop->lrp_hop = route->lr_hops;
6790                                 prop->lrp_priority = route->lr_priority;
6791                                 prop->lrp_sensitivity =
6792                                         route->lr_gateway->lp_health_sensitivity;
6793                                 if (lnet_is_route_alive(route))
6794                                         prop->lrp_flags |= LNET_RT_ALIVE;
6795                                 else
6796                                         prop->lrp_flags &= ~LNET_RT_ALIVE;
6797                                 if (route->lr_single_hop)
6798                                         prop->lrp_flags &= ~LNET_RT_MULTI_HOP;
6799                                 else
6800                                         prop->lrp_flags |= LNET_RT_MULTI_HOP;
6801                         }
6802                 }
6803         }
6804
6805 failed_alloc:
6806         lnet_net_unlock(cpt);
6807         return rc;
6808 }
6809
6810 /* LNet route ->start() handler for GET requests */
6811 static int lnet_route_show_start(struct netlink_callback *cb)
6812 {
6813         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
6814 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
6815         struct netlink_ext_ack *extack = NULL;
6816 #endif
6817         struct lnet_genl_route_list *rlist;
6818         int msg_len = genlmsg_len(gnlh);
6819         int rc = 0;
6820
6821 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
6822         extack = cb->extack;
6823 #endif
6824         if (the_lnet.ln_refcount == 0 ||
6825             the_lnet.ln_state != LNET_STATE_RUNNING) {
6826                 NL_SET_ERR_MSG(extack, "Network is down");
6827                 return -ENETDOWN;
6828         }
6829
6830         CFS_ALLOC_PTR(rlist);
6831         if (!rlist) {
6832                 NL_SET_ERR_MSG(extack, "No memory for route list");
6833                 return -ENOMEM;
6834         }
6835
6836         genradix_init(&rlist->lgrl_list);
6837         rlist->lgrl_count = 0;
6838         rlist->lgrl_index = 0;
6839         cb->args[0] = (long)rlist;
6840
6841         mutex_lock(&the_lnet.ln_api_mutex);
6842         if (!msg_len) {
6843                 struct lnet_route_properties tmp = {
6844                         .lrp_gateway            = LNET_ANY_NID,
6845                         .lrp_net                = LNET_NET_ANY,
6846                         .lrp_hop                = -1,
6847                         .lrp_priority           = -1,
6848                         .lrp_sensitivity        = -1,
6849                 };
6850
6851                 rc = lnet_scan_route(rlist, &tmp);
6852                 if (rc < 0) {
6853                         NL_SET_ERR_MSG(extack,
6854                                        "failed to allocate router data");
6855                         GOTO(report_err, rc);
6856                 }
6857         } else {
6858                 struct nlattr *params = genlmsg_data(gnlh);
6859                 struct nlattr *attr;
6860                 int rem;
6861
6862                 nla_for_each_nested(attr, params, rem) {
6863                         struct lnet_route_properties tmp = {
6864                                 .lrp_gateway            = LNET_ANY_NID,
6865                                 .lrp_net                = LNET_NET_ANY,
6866                                 .lrp_hop                = -1,
6867                                 .lrp_priority           = -1,
6868                                 .lrp_sensitivity        = -1,
6869                         };
6870                         struct nlattr *route;
6871                         int rem2;
6872
6873                         if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
6874                                 continue;
6875
6876                         nla_for_each_nested(route, attr, rem2) {
6877                                 if (nla_type(route) != LN_SCALAR_ATTR_VALUE)
6878                                         continue;
6879
6880                                 if (nla_strcmp(route, "net") == 0) {
6881                                         char nw[LNET_NIDSTR_SIZE];
6882
6883                                         route = nla_next(route, &rem2);
6884                                         if (nla_type(route) !=
6885                                             LN_SCALAR_ATTR_VALUE) {
6886                                                 NL_SET_ERR_MSG(extack,
6887                                                                "invalid net param");
6888                                                 GOTO(report_err, rc = -EINVAL);
6889                                         }
6890
6891                                         rc = nla_strscpy(nw, route, sizeof(nw));
6892                                         if (rc < 0) {
6893                                                 NL_SET_ERR_MSG(extack,
6894                                                                "failed to get route param");
6895                                                 GOTO(report_err, rc);
6896                                         }
6897                                         rc = 0;
6898                                         tmp.lrp_net = libcfs_str2net(strim(nw));
6899                                 } else if (nla_strcmp(route, "gateway") == 0) {
6900                                         char gw[LNET_NIDSTR_SIZE];
6901
6902                                         route = nla_next(route, &rem2);
6903                                         if (nla_type(route) !=
6904                                             LN_SCALAR_ATTR_VALUE) {
6905                                                 NL_SET_ERR_MSG(extack,
6906                                                                "invalid gateway param");
6907                                                 GOTO(report_err, rc = -EINVAL);
6908                                         }
6909
6910                                         rc = nla_strscpy(gw, route, sizeof(gw));
6911                                         if (rc < 0) {
6912                                                 NL_SET_ERR_MSG(extack,
6913                                                                "failed to get route param");
6914                                                 GOTO(report_err, rc);
6915                                         }
6916
6917                                         rc = libcfs_strnid(&tmp.lrp_gateway, strim(gw));
6918                                         if (rc < 0) {
6919                                                 NL_SET_ERR_MSG(extack,
6920                                                                "cannot parse gateway");
6921                                                 GOTO(report_err, rc = -ENODEV);
6922                                         }
6923                                         rc = 0;
6924                                 } else if (nla_strcmp(route, "hop") == 0) {
6925                                         route = nla_next(route, &rem2);
6926                                         if (nla_type(route) !=
6927                                             LN_SCALAR_ATTR_INT_VALUE) {
6928                                                 NL_SET_ERR_MSG(extack,
6929                                                                "invalid hop param");
6930                                                 GOTO(report_err, rc = -EINVAL);
6931                                         }
6932
6933                                         tmp.lrp_hop = nla_get_s64(route);
6934                                         if (tmp.lrp_hop != -1)
6935                                                 clamp_t(s32, tmp.lrp_hop, 1, 127);
6936                                 } else if (nla_strcmp(route, "priority") == 0) {
6937                                         route = nla_next(route, &rem2);
6938                                         if (nla_type(route) !=
6939                                             LN_SCALAR_ATTR_INT_VALUE) {
6940                                                 NL_SET_ERR_MSG(extack,
6941                                                                "invalid priority param");
6942                                                 GOTO(report_err, rc = -EINVAL);
6943                                         }
6944
6945                                         tmp.lrp_priority = nla_get_s64(route);
6946                                 }
6947                         }
6948
6949                         rc = lnet_scan_route(rlist, &tmp);
6950                         if (rc < 0) {
6951                                 NL_SET_ERR_MSG(extack,
6952                                                "failed to allocate router data");
6953                                 GOTO(report_err, rc);
6954                         }
6955                 }
6956         }
6957 report_err:
6958         mutex_unlock(&the_lnet.ln_api_mutex);
6959
6960         if (rc < 0)
6961                 lnet_route_show_done(cb);
6962
6963         return rc;
6964 }
6965
6966 static const struct ln_key_list route_props_list = {
6967         .lkl_maxattr                    = LNET_ROUTE_ATTR_MAX,
6968         .lkl_list                       = {
6969                 [LNET_ROUTE_ATTR_HDR]                   = {
6970                         .lkp_value                      = "route",
6971                         .lkp_key_format                 = LNKF_SEQUENCE | LNKF_MAPPING,
6972                         .lkp_data_type                  = NLA_NUL_STRING,
6973                 },
6974                 [LNET_ROUTE_ATTR_NET]                   = {
6975                         .lkp_value                      = "net",
6976                         .lkp_data_type                  = NLA_STRING
6977                 },
6978                 [LNET_ROUTE_ATTR_GATEWAY]               = {
6979                         .lkp_value                      = "gateway",
6980                         .lkp_data_type                  = NLA_STRING
6981                 },
6982                 [LNET_ROUTE_ATTR_HOP]                   = {
6983                         .lkp_value                      = "hop",
6984                         .lkp_data_type                  = NLA_S32
6985                 },
6986                 [LNET_ROUTE_ATTR_PRIORITY]              = {
6987                         .lkp_value                      = "priority",
6988                         .lkp_data_type                  = NLA_U32
6989                 },
6990                 [LNET_ROUTE_ATTR_HEALTH_SENSITIVITY]    = {
6991                         .lkp_value                      = "health_sensitivity",
6992                         .lkp_data_type                  = NLA_U32
6993                 },
6994                 [LNET_ROUTE_ATTR_STATE] = {
6995                         .lkp_value                      = "state",
6996                         .lkp_data_type                  = NLA_STRING,
6997                 },
6998                 [LNET_ROUTE_ATTR_TYPE]  = {
6999                         .lkp_value                      = "type",
7000                         .lkp_data_type                  = NLA_STRING,
7001                 },
7002         },
7003 };
7004
7005
7006 static int lnet_route_show_dump(struct sk_buff *msg,
7007                                 struct netlink_callback *cb)
7008 {
7009         struct lnet_genl_route_list *rlist = lnet_route_dump_ctx(cb);
7010         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
7011 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
7012         struct netlink_ext_ack *extack = NULL;
7013 #endif
7014         int portid = NETLINK_CB(cb->skb).portid;
7015         int seq = cb->nlh->nlmsg_seq;
7016         int idx = rlist->lgrl_index;
7017         int msg_len = genlmsg_len(gnlh);
7018         int rc = 0;
7019
7020 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
7021         extack = cb->extack;
7022 #endif
7023         if (!rlist->lgrl_count) {
7024                 NL_SET_ERR_MSG(extack, "No routes found");
7025                 GOTO(send_error, rc = msg_len ? -ENOENT : 0);
7026         }
7027
7028         if (!idx) {
7029                 const struct ln_key_list *all[] = {
7030                         &route_props_list, NULL
7031                 };
7032
7033                 rc = lnet_genl_send_scalar_list(msg, portid, seq,
7034                                                 &lnet_family,
7035                                                 NLM_F_CREATE | NLM_F_MULTI,
7036                                                 LNET_CMD_ROUTES, all);
7037                 if (rc < 0) {
7038                         NL_SET_ERR_MSG(extack, "failed to send key table");
7039                         GOTO(send_error, rc);
7040                 }
7041         }
7042
7043         while (idx < rlist->lgrl_count) {
7044                 struct lnet_route_properties *prop;
7045                 void *hdr;
7046
7047                 prop = genradix_ptr(&rlist->lgrl_list, idx++);
7048
7049                 hdr = genlmsg_put(msg, portid, seq, &lnet_family,
7050                                   NLM_F_MULTI, LNET_CMD_ROUTES);
7051                 if (!hdr) {
7052                         NL_SET_ERR_MSG(extack, "failed to send values");
7053                         genlmsg_cancel(msg, hdr);
7054                         GOTO(send_error, rc = -EMSGSIZE);
7055                 }
7056
7057                 if (idx == 1)
7058                         nla_put_string(msg, LNET_ROUTE_ATTR_HDR, "");
7059
7060                 nla_put_string(msg, LNET_ROUTE_ATTR_NET,
7061                                libcfs_net2str(prop->lrp_net));
7062                 nla_put_string(msg, LNET_ROUTE_ATTR_GATEWAY,
7063                                libcfs_nidstr(&prop->lrp_gateway));
7064                 if (gnlh->version) {
7065                         nla_put_s32(msg, LNET_ROUTE_ATTR_HOP, prop->lrp_hop);
7066                         nla_put_u32(msg, LNET_ROUTE_ATTR_PRIORITY, prop->lrp_priority);
7067                         nla_put_u32(msg, LNET_ROUTE_ATTR_HEALTH_SENSITIVITY,
7068                                     prop->lrp_sensitivity);
7069
7070                         if (!(cb->nlh->nlmsg_flags & NLM_F_DUMP_FILTERED)) {
7071                                 nla_put_string(msg, LNET_ROUTE_ATTR_STATE,
7072                                                prop->lrp_flags & LNET_RT_ALIVE ?
7073                                                "up" : "down");
7074                                 nla_put_string(msg, LNET_ROUTE_ATTR_TYPE,
7075                                                prop->lrp_flags & LNET_RT_MULTI_HOP ?
7076                                                "multi-hop" : "single-hop");
7077                         }
7078                 }
7079                 genlmsg_end(msg, hdr);
7080         }
7081         rlist->lgrl_index = idx;
7082 send_error:
7083         return lnet_nl_send_error(cb->skb, portid, seq, rc);
7084 };
7085
7086 #ifndef HAVE_NETLINK_CALLBACK_START
7087 static int lnet_old_route_show_dump(struct sk_buff *msg,
7088                                     struct netlink_callback *cb)
7089 {
7090         if (!cb->args[0]) {
7091                 int rc = lnet_route_show_start(cb);
7092
7093                 if (rc < 0)
7094                         return lnet_nl_send_error(cb->skb,
7095                                                   NETLINK_CB(cb->skb).portid,
7096                                                   cb->nlh->nlmsg_seq,
7097                                                   rc);
7098         }
7099
7100         return lnet_route_show_dump(msg, cb);
7101 }
7102 #endif /* !HAVE_NETLINK_CALLBACK_START */
7103
7104 /** LNet peer handling */
7105 struct lnet_genl_processid_list {
7106         unsigned int                    lgpl_index;
7107         unsigned int                    lgpl_count;
7108         GENRADIX(struct lnet_processid) lgpl_list;
7109 };
7110
7111 static inline struct lnet_genl_processid_list *
7112 lnet_peer_dump_ctx(struct netlink_callback *cb)
7113 {
7114         return (struct lnet_genl_processid_list *)cb->args[0];
7115 }
7116
7117 static int lnet_peer_ni_show_done(struct netlink_callback *cb)
7118 {
7119         struct lnet_genl_processid_list *plist = lnet_peer_dump_ctx(cb);
7120
7121         if (plist) {
7122                 genradix_free(&plist->lgpl_list);
7123                 CFS_FREE_PTR(plist);
7124         }
7125         cb->args[0] = 0;
7126
7127         return 0;
7128 }
7129
7130 /* LNet peer ->start() handler for GET requests */
7131 static int lnet_peer_ni_show_start(struct netlink_callback *cb)
7132 {
7133         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
7134 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
7135         struct netlink_ext_ack *extack = NULL;
7136 #endif
7137         struct lnet_genl_processid_list *plist;
7138         int msg_len = genlmsg_len(gnlh);
7139         int rc = 0;
7140
7141 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
7142         extack = cb->extack;
7143 #endif
7144         mutex_lock(&the_lnet.ln_api_mutex);
7145         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
7146                 NL_SET_ERR_MSG(extack, "Network is down");
7147                 mutex_unlock(&the_lnet.ln_api_mutex);
7148                 return -ENETDOWN;
7149         }
7150
7151         CFS_ALLOC_PTR(plist);
7152         if (!plist) {
7153                 NL_SET_ERR_MSG(extack, "No memory for peer list");
7154                 mutex_unlock(&the_lnet.ln_api_mutex);
7155                 return -ENOMEM;
7156         }
7157
7158         genradix_init(&plist->lgpl_list);
7159         plist->lgpl_count = 0;
7160         plist->lgpl_index = 0;
7161         cb->args[0] = (long)plist;
7162
7163         if (!msg_len) {
7164                 struct lnet_peer_table *ptable;
7165                 int cpt;
7166
7167                 cfs_percpt_for_each(ptable, cpt, the_lnet.ln_peer_tables) {
7168                         struct lnet_peer *lp;
7169
7170                         list_for_each_entry(lp, &ptable->pt_peer_list,
7171                                             lp_peer_list) {
7172                                 struct lnet_processid *lpi;
7173
7174                                 lpi = genradix_ptr_alloc(&plist->lgpl_list,
7175                                                          plist->lgpl_count++,
7176                                                          GFP_KERNEL);
7177                                 if (!lpi) {
7178                                         NL_SET_ERR_MSG(extack,
7179                                                       "failed to allocate NID");
7180                                         GOTO(report_err, rc = -ENOMEM);
7181                                 }
7182
7183                                 lpi->pid = LNET_PID_LUSTRE;
7184                                 lpi->nid = lp->lp_primary_nid;
7185                         }
7186                 }
7187         } else {
7188                 struct nlattr *params = genlmsg_data(gnlh);
7189                 struct nlattr *attr;
7190                 int rem;
7191
7192                 nla_for_each_nested(attr, params, rem) {
7193                         struct nlattr *nid;
7194                         int rem2;
7195
7196                         if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
7197                                 continue;
7198
7199                         nla_for_each_nested(nid, attr, rem2) {
7200                                 char addr[LNET_NIDSTR_SIZE];
7201                                 struct lnet_processid *id;
7202
7203                                 if (nla_type(nid) != LN_SCALAR_ATTR_VALUE ||
7204                                     nla_strcmp(nid, "primary nid") != 0)
7205                                         continue;
7206
7207                                 nid = nla_next(nid, &rem2);
7208                                 if (nla_type(nid) != LN_SCALAR_ATTR_VALUE) {
7209                                         NL_SET_ERR_MSG(extack,
7210                                                        "invalid primary nid param");
7211                                         GOTO(report_err, rc = -EINVAL);
7212                                 }
7213
7214                                 rc = nla_strscpy(addr, nid, sizeof(addr));
7215                                 if (rc < 0) {
7216                                         NL_SET_ERR_MSG(extack,
7217                                                        "failed to get primary nid param");
7218                                         GOTO(report_err, rc);
7219                                 }
7220
7221                                 id = genradix_ptr_alloc(&plist->lgpl_list,
7222                                                         plist->lgpl_count++,
7223                                                         GFP_KERNEL);
7224                                 if (!id) {
7225                                         NL_SET_ERR_MSG(extack, "failed to allocate NID");
7226                                         GOTO(report_err, rc = -ENOMEM);
7227                                 }
7228
7229                                 rc = libcfs_strid(id, strim(addr));
7230                                 if (rc < 0) {
7231                                         NL_SET_ERR_MSG(extack, "invalid NID");
7232                                         GOTO(report_err, rc);
7233                                 }
7234                                 rc = 0;
7235                         }
7236                 }
7237         }
7238 report_err:
7239         mutex_unlock(&the_lnet.ln_api_mutex);
7240
7241         if (rc < 0)
7242                 lnet_peer_ni_show_done(cb);
7243
7244         return rc;
7245 }
7246
7247 static const struct ln_key_list lnet_peer_ni_keys = {
7248         .lkl_maxattr                    = LNET_PEER_NI_ATTR_MAX,
7249         .lkl_list                       = {
7250                 [LNET_PEER_NI_ATTR_HDR]  = {
7251                         .lkp_value              = "peer",
7252                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
7253                         .lkp_data_type          = NLA_NUL_STRING,
7254                 },
7255                 [LNET_PEER_NI_ATTR_PRIMARY_NID] = {
7256                         .lkp_value              = "primary nid",
7257                         .lkp_data_type          = NLA_STRING,
7258                 },
7259                 [LNET_PEER_NI_ATTR_MULTIRAIL]   = {
7260                         .lkp_value              = "Multi-Rail",
7261                         .lkp_data_type          = NLA_FLAG
7262                 },
7263                 [LNET_PEER_NI_ATTR_STATE]       = {
7264                         .lkp_value              = "peer state",
7265                         .lkp_data_type          = NLA_U32
7266                 },
7267                 [LNET_PEER_NI_ATTR_PEER_NI_LIST] = {
7268                         .lkp_value              = "peer ni",
7269                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
7270                         .lkp_data_type          = NLA_NESTED,
7271                 },
7272         },
7273 };
7274
7275 static const struct ln_key_list lnet_peer_ni_list = {
7276         .lkl_maxattr                    = LNET_PEER_NI_LIST_ATTR_MAX,
7277         .lkl_list                       = {
7278                 [LNET_PEER_NI_LIST_ATTR_NID]            = {
7279                         .lkp_value                      = "nid",
7280                         .lkp_data_type                  = NLA_STRING,
7281                 },
7282                 [LNET_PEER_NI_LIST_ATTR_UDSP_INFO]      = {
7283                         .lkp_value                      = "udsp info",
7284                         .lkp_key_format                 = LNKF_MAPPING,
7285                         .lkp_data_type                  = NLA_NESTED,
7286                 },
7287                 [LNET_PEER_NI_LIST_ATTR_STATE]          = {
7288                         .lkp_value                      = "state",
7289                         .lkp_data_type                  = NLA_STRING,
7290                 },
7291                 [LNET_PEER_NI_LIST_ATTR_MAX_TX_CREDITS] = {
7292                         .lkp_value                      = "max_ni_tx_credits",
7293                         .lkp_data_type                  = NLA_U32,
7294                 },
7295                 [LNET_PEER_NI_LIST_ATTR_CUR_TX_CREDITS] = {
7296                         .lkp_value                      = "available_tx_credits",
7297                         .lkp_data_type                  = NLA_U32,
7298                 },
7299                 [LNET_PEER_NI_LIST_ATTR_MIN_TX_CREDITS] = {
7300                         .lkp_value                      = "min_tx_credits",
7301                         .lkp_data_type                  = NLA_U32,
7302                 },
7303                 [LNET_PEER_NI_LIST_ATTR_QUEUE_BUF_COUNT] = {
7304                         .lkp_value                      = "tx_q_num_of_buf",
7305                         .lkp_data_type                  = NLA_U32,
7306                 },
7307                 [LNET_PEER_NI_LIST_ATTR_CUR_RTR_CREDITS] = {
7308                         .lkp_value                      = "available_rtr_credits",
7309                         .lkp_data_type                  = NLA_U32,
7310                 },
7311                 [LNET_PEER_NI_LIST_ATTR_MIN_RTR_CREDITS] = {
7312                         .lkp_value                      = "min_rtr_credits",
7313                         .lkp_data_type                  = NLA_U32,
7314                 },
7315                 [LNET_PEER_NI_LIST_ATTR_REFCOUNT]       = {
7316                         .lkp_value                      = "refcount",
7317                         .lkp_data_type                  = NLA_U32,
7318                 },
7319                 [LNET_PEER_NI_LIST_ATTR_STATS_COUNT]    = {
7320                         .lkp_value                      = "statistics",
7321                         .lkp_key_format                 = LNKF_MAPPING,
7322                         .lkp_data_type                  = NLA_NESTED
7323                 },
7324                 [LNET_PEER_NI_LIST_ATTR_SENT_STATS]     = {
7325                         .lkp_value                      = "sent_stats",
7326                         .lkp_key_format                 = LNKF_MAPPING,
7327                         .lkp_data_type                  = NLA_NESTED
7328                 },
7329                 [LNET_PEER_NI_LIST_ATTR_RECV_STATS]     = {
7330                         .lkp_value                      = "received_stats",
7331                         .lkp_key_format                 = LNKF_MAPPING,
7332                         .lkp_data_type                  = NLA_NESTED
7333                 },
7334                 [LNET_PEER_NI_LIST_ATTR_DROP_STATS]     = {
7335                         .lkp_value                      = "dropped_stats",
7336                         .lkp_key_format                 = LNKF_MAPPING,
7337                         .lkp_data_type                  = NLA_NESTED
7338                 },
7339                 [LNET_PEER_NI_LIST_ATTR_HEALTH_STATS]   = {
7340                         .lkp_value                      = "health stats",
7341                         .lkp_key_format                 = LNKF_MAPPING,
7342                         .lkp_data_type                  = NLA_NESTED
7343                 },
7344         },
7345 };
7346
7347 static const struct ln_key_list lnet_peer_ni_list_stats_count = {
7348         .lkl_maxattr                    = LNET_PEER_NI_LIST_STATS_COUNT_ATTR_MAX,
7349         .lkl_list                       = {
7350                 [LNET_PEER_NI_LIST_STATS_COUNT_ATTR_SEND_COUNT] = {
7351                         .lkp_value                              = "send_count",
7352                         .lkp_data_type                          = NLA_U32,
7353                 },
7354                 [LNET_PEER_NI_LIST_STATS_COUNT_ATTR_RECV_COUNT] = {
7355                         .lkp_value                              = "recv_count",
7356                         .lkp_data_type                          = NLA_U32,
7357                 },
7358                 [LNET_PEER_NI_LIST_STATS_COUNT_ATTR_DROP_COUNT] = {
7359                         .lkp_value                              = "drop_count",
7360                         .lkp_data_type                          = NLA_U32,
7361                 },
7362         },
7363 };
7364
7365 static const struct ln_key_list lnet_peer_ni_list_stats = {
7366         .lkl_maxattr                    = LNET_PEER_NI_LIST_STATS_ATTR_MAX,
7367         .lkl_list                       = {
7368                 [LNET_PEER_NI_LIST_STATS_ATTR_PUT]      = {
7369                         .lkp_value                      = "put",
7370                         .lkp_data_type                  = NLA_U32,
7371                 },
7372                 [LNET_PEER_NI_LIST_STATS_ATTR_GET]      = {
7373                         .lkp_value                      = "get",
7374                         .lkp_data_type                  = NLA_U32,
7375                 },
7376                 [LNET_PEER_NI_LIST_STATS_ATTR_REPLY]    = {
7377                         .lkp_value                      = "reply",
7378                         .lkp_data_type                  = NLA_U32,
7379                 },
7380                 [LNET_PEER_NI_LIST_STATS_ATTR_ACK]      = {
7381                         .lkp_value                      = "ack",
7382                         .lkp_data_type                  = NLA_U32,
7383                 },
7384                 [LNET_PEER_NI_LIST_STATS_ATTR_HELLO]    = {
7385                         .lkp_value                      = "hello",
7386                         .lkp_data_type                  = NLA_U32,
7387                 },
7388         },
7389 };
7390
7391 static const struct ln_key_list lnet_peer_ni_list_health = {
7392         .lkl_maxattr                    = LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_MAX,
7393         .lkl_list                       = {
7394                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_VALUE]     = {
7395                         .lkp_value                      = "health value",
7396                         .lkp_data_type                  = NLA_S32,
7397                 },
7398                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_DROPPED]   = {
7399                         .lkp_value                      = "dropped",
7400                         .lkp_data_type                  = NLA_U32,
7401                 },
7402                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_TIMEOUT]   = {
7403                         .lkp_value                      = "timeout",
7404                         .lkp_data_type                  = NLA_U32,
7405                 },
7406                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_ERROR]     = {
7407                         .lkp_value                      = "error",
7408                         .lkp_data_type                  = NLA_U32,
7409                 },
7410                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_NETWORK_TIMEOUT] = {
7411                         .lkp_value                      = "network timeout",
7412                         .lkp_data_type                  = NLA_U32,
7413                 },
7414                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_PING_COUNT] = {
7415                         .lkp_value                      = "ping_count",
7416                         .lkp_data_type                  = NLA_U32,
7417                 },
7418                 [LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_NEXT_PING] = {
7419                         .lkp_value                      = "next_ping",
7420                         .lkp_data_type                  = NLA_S64,
7421                 },
7422         },
7423 };
7424
7425 static int lnet_peer_ni_show_dump(struct sk_buff *msg,
7426                                   struct netlink_callback *cb)
7427 {
7428         struct lnet_genl_processid_list *plist = lnet_peer_dump_ctx(cb);
7429         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
7430 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
7431         struct netlink_ext_ack *extack = NULL;
7432 #endif
7433         int portid = NETLINK_CB(cb->skb).portid;
7434         int seq = cb->nlh->nlmsg_seq;
7435         int idx = plist->lgpl_index;
7436         int msg_len = genlmsg_len(gnlh);
7437         int rc = 0;
7438
7439 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
7440         extack = cb->extack;
7441 #endif
7442         if (!plist->lgpl_count) {
7443                 NL_SET_ERR_MSG(extack, "No peers found");
7444                 GOTO(send_error, rc = msg_len ? -ENOENT : 0);
7445         }
7446
7447         if (!idx) {
7448                 const struct ln_key_list *all[] = {
7449                         &lnet_peer_ni_keys, &lnet_peer_ni_list,
7450                         &udsp_info_list, &udsp_info_pref_nids_list,
7451                         &udsp_info_pref_nids_list,
7452                         &lnet_peer_ni_list_stats_count,
7453                         &lnet_peer_ni_list_stats, /* send_stats */
7454                         &lnet_peer_ni_list_stats, /* recv_stats */
7455                         &lnet_peer_ni_list_stats, /* drop stats */
7456                         &lnet_peer_ni_list_health,
7457                         NULL
7458                 };
7459
7460                 rc = lnet_genl_send_scalar_list(msg, portid, seq,
7461                                                 &lnet_family,
7462                                                 NLM_F_CREATE | NLM_F_MULTI,
7463                                                 LNET_CMD_PEERS, all);
7464                 if (rc < 0) {
7465                         NL_SET_ERR_MSG(extack, "failed to send key table");
7466                         GOTO(send_error, rc);
7467                 }
7468         }
7469
7470         mutex_lock(&the_lnet.ln_api_mutex);
7471         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
7472                 NL_SET_ERR_MSG(extack, "Network is down");
7473                 GOTO(unlock_api_mutex, rc = -ENETDOWN);
7474         }
7475
7476         while (idx < plist->lgpl_count) {
7477                 struct lnet_processid *id;
7478                 struct lnet_peer_ni *lpni = NULL;
7479                 struct nlattr *nid_list;
7480                 struct lnet_peer *lp;
7481                 int count = 1;
7482                 void *hdr;
7483
7484                 id = genradix_ptr(&plist->lgpl_list, idx++);
7485                 if (nid_is_lo0(&id->nid))
7486                         continue;
7487
7488                 hdr = genlmsg_put(msg, portid, seq, &lnet_family,
7489                                   NLM_F_MULTI, LNET_CMD_PEERS);
7490                 if (!hdr) {
7491                         NL_SET_ERR_MSG(extack, "failed to send values");
7492                         genlmsg_cancel(msg, hdr);
7493                         GOTO(unlock_api_mutex, rc = -EMSGSIZE);
7494                 }
7495
7496                 lp = lnet_find_peer(&id->nid);
7497                 if (!lp) {
7498                         NL_SET_ERR_MSG(extack, "cannot find peer");
7499                         GOTO(unlock_api_mutex, rc = -ENOENT);
7500                 }
7501
7502                 if (idx == 1)
7503                         nla_put_string(msg, LNET_PEER_NI_ATTR_HDR, "");
7504
7505                 nla_put_string(msg, LNET_PEER_NI_ATTR_PRIMARY_NID,
7506                                libcfs_nidstr(&lp->lp_primary_nid));
7507                 if (lnet_peer_is_multi_rail(lp))
7508                         nla_put_flag(msg, LNET_PEER_NI_ATTR_MULTIRAIL);
7509
7510                 if (gnlh->version >= 3)
7511                         nla_put_u32(msg, LNET_PEER_NI_ATTR_STATE, lp->lp_state);
7512
7513                 nid_list = nla_nest_start(msg, LNET_PEER_NI_ATTR_PEER_NI_LIST);
7514                 while ((lpni = lnet_get_next_peer_ni_locked(lp, NULL, lpni)) != NULL) {
7515                         struct nlattr *peer_nid = nla_nest_start(msg, count++);
7516
7517                         nla_put_string(msg, LNET_PEER_NI_LIST_ATTR_NID,
7518                                        libcfs_nidstr(&lpni->lpni_nid));
7519
7520                         if (gnlh->version >= 4) {
7521                                 rc = lnet_udsp_info_send(msg,
7522                                                          LNET_PEER_NI_LIST_ATTR_UDSP_INFO,
7523                                                          &lpni->lpni_nid, true);
7524                                 if (rc < 0) {
7525                                         lnet_peer_decref_locked(lp);
7526                                         NL_SET_ERR_MSG(extack,
7527                                                        "failed to get UDSP info");
7528                                         GOTO(unlock_api_mutex, rc);
7529                                 }
7530                         }
7531
7532                         if (cb->nlh->nlmsg_flags & NLM_F_DUMP_FILTERED)
7533                                 goto skip_state;
7534
7535                         if (lnet_isrouter(lpni) ||
7536                             lnet_peer_aliveness_enabled(lpni)) {
7537                                 nla_put_string(msg, LNET_PEER_NI_LIST_ATTR_STATE,
7538                                                lnet_is_peer_ni_alive(lpni) ?
7539                                                "up" : "down");
7540                         } else {
7541                                 nla_put_string(msg, LNET_PEER_NI_LIST_ATTR_STATE,
7542                                                "NA");
7543                         }
7544 skip_state:
7545                         if (gnlh->version) {
7546                                 struct lnet_ioctl_element_msg_stats lpni_msg_stats;
7547                                 struct nlattr *send_stats_list, *send_stats;
7548                                 struct nlattr *recv_stats_list, *recv_stats;
7549                                 struct nlattr *drop_stats_list, *drop_stats;
7550                                 struct nlattr *health_list, *health_stats;
7551                                 struct lnet_ioctl_element_stats stats;
7552                                 struct nlattr *stats_attr, *ni_stats;
7553
7554                                 nla_put_u32(msg,
7555                                             LNET_PEER_NI_LIST_ATTR_MAX_TX_CREDITS,
7556                                             lpni->lpni_net ?
7557                                                 lpni->lpni_net->net_tunables.lct_peer_tx_credits : 0);
7558                                 nla_put_u32(msg,
7559                                             LNET_PEER_NI_LIST_ATTR_CUR_TX_CREDITS,
7560                                             lpni->lpni_txcredits);
7561                                 nla_put_u32(msg,
7562                                             LNET_PEER_NI_LIST_ATTR_MIN_TX_CREDITS,
7563                                             lpni->lpni_mintxcredits);
7564                                 nla_put_u32(msg,
7565                                             LNET_PEER_NI_LIST_ATTR_QUEUE_BUF_COUNT,
7566                                             lpni->lpni_txqnob);
7567                                 nla_put_u32(msg,
7568                                             LNET_PEER_NI_LIST_ATTR_CUR_RTR_CREDITS,
7569                                             lpni->lpni_rtrcredits);
7570                                 nla_put_u32(msg,
7571                                             LNET_PEER_NI_LIST_ATTR_MIN_RTR_CREDITS,
7572                                             lpni->lpni_minrtrcredits);
7573                                 nla_put_u32(msg,
7574                                             LNET_PEER_NI_LIST_ATTR_REFCOUNT,
7575                                             kref_read(&lpni->lpni_kref));
7576
7577                                 memset(&stats, 0, sizeof(stats));
7578                                 stats.iel_send_count = lnet_sum_stats(&lpni->lpni_stats,
7579                                                                       LNET_STATS_TYPE_SEND);
7580                                 stats.iel_recv_count = lnet_sum_stats(&lpni->lpni_stats,
7581                                                                       LNET_STATS_TYPE_RECV);
7582                                 stats.iel_drop_count = lnet_sum_stats(&lpni->lpni_stats,
7583                                                                       LNET_STATS_TYPE_DROP);
7584
7585                                 stats_attr = nla_nest_start(msg,
7586                                                             LNET_PEER_NI_LIST_ATTR_STATS_COUNT);
7587                                 ni_stats = nla_nest_start(msg, 0);
7588                                 nla_put_u32(msg,
7589                                             LNET_PEER_NI_LIST_STATS_COUNT_ATTR_SEND_COUNT,
7590                                             stats.iel_send_count);
7591                                 nla_put_u32(msg,
7592                                             LNET_PEER_NI_LIST_STATS_COUNT_ATTR_RECV_COUNT,
7593                                             stats.iel_recv_count);
7594                                 nla_put_u32(msg,
7595                                             LNET_PEER_NI_LIST_STATS_COUNT_ATTR_DROP_COUNT,
7596                                             stats.iel_drop_count);
7597                                 nla_nest_end(msg, ni_stats);
7598                                 nla_nest_end(msg, stats_attr);
7599
7600                                 if (gnlh->version < 2)
7601                                         goto skip_msg_stats;
7602
7603                                 lnet_usr_translate_stats(&lpni_msg_stats, &lpni->lpni_stats);
7604
7605                                 send_stats_list = nla_nest_start(msg,
7606                                                                  LNET_PEER_NI_LIST_ATTR_SENT_STATS);
7607                                 send_stats = nla_nest_start(msg, 0);
7608                                 nla_put_u32(msg,
7609                                             LNET_PEER_NI_LIST_STATS_ATTR_PUT,
7610                                             lpni_msg_stats.im_send_stats.ico_put_count);
7611                                 nla_put_u32(msg,
7612                                             LNET_PEER_NI_LIST_STATS_ATTR_GET,
7613                                             lpni_msg_stats.im_send_stats.ico_get_count);
7614                                 nla_put_u32(msg,
7615                                             LNET_PEER_NI_LIST_STATS_ATTR_REPLY,
7616                                             lpni_msg_stats.im_send_stats.ico_reply_count);
7617                                 nla_put_u32(msg,
7618                                             LNET_PEER_NI_LIST_STATS_ATTR_ACK,
7619                                             lpni_msg_stats.im_send_stats.ico_ack_count);
7620                                 nla_put_u32(msg,
7621                                             LNET_PEER_NI_LIST_STATS_ATTR_HELLO,
7622                                             lpni_msg_stats.im_send_stats.ico_hello_count);
7623                                 nla_nest_end(msg, send_stats);
7624                                 nla_nest_end(msg, send_stats_list);
7625
7626                                 recv_stats_list = nla_nest_start(msg,
7627                                                                  LNET_PEER_NI_LIST_ATTR_RECV_STATS);
7628                                 recv_stats = nla_nest_start(msg, 0);
7629                                 nla_put_u32(msg,
7630                                             LNET_PEER_NI_LIST_STATS_ATTR_PUT,
7631                                             lpni_msg_stats.im_recv_stats.ico_put_count);
7632                                 nla_put_u32(msg,
7633                                             LNET_PEER_NI_LIST_STATS_ATTR_GET,
7634                                             lpni_msg_stats.im_recv_stats.ico_get_count);
7635                                 nla_put_u32(msg,
7636                                             LNET_PEER_NI_LIST_STATS_ATTR_REPLY,
7637                                             lpni_msg_stats.im_recv_stats.ico_reply_count);
7638                                 nla_put_u32(msg,
7639                                             LNET_PEER_NI_LIST_STATS_ATTR_ACK,
7640                                             lpni_msg_stats.im_recv_stats.ico_ack_count);
7641                                 nla_put_u32(msg,
7642                                             LNET_PEER_NI_LIST_STATS_ATTR_HELLO,
7643                                             lpni_msg_stats.im_recv_stats.ico_hello_count);
7644                                 nla_nest_end(msg, recv_stats);
7645                                 nla_nest_end(msg, recv_stats_list);
7646
7647                                 drop_stats_list = nla_nest_start(msg,
7648                                                                  LNET_PEER_NI_LIST_ATTR_DROP_STATS);
7649                                 drop_stats = nla_nest_start(msg, 0);
7650                                 nla_put_u32(msg,
7651                                             LNET_PEER_NI_LIST_STATS_ATTR_PUT,
7652                                             lpni_msg_stats.im_drop_stats.ico_put_count);
7653                                 nla_put_u32(msg,
7654                                             LNET_PEER_NI_LIST_STATS_ATTR_GET,
7655                                             lpni_msg_stats.im_drop_stats.ico_get_count);
7656                                 nla_put_u32(msg,
7657                                             LNET_PEER_NI_LIST_STATS_ATTR_REPLY,
7658                                             lpni_msg_stats.im_drop_stats.ico_reply_count);
7659                                 nla_put_u32(msg,
7660                                             LNET_PEER_NI_LIST_STATS_ATTR_ACK,
7661                                             lpni_msg_stats.im_drop_stats.ico_ack_count);
7662                                 nla_put_u32(msg,
7663                                             LNET_PEER_NI_LIST_STATS_ATTR_HELLO,
7664                                             lpni_msg_stats.im_drop_stats.ico_hello_count);
7665                                 nla_nest_end(msg, drop_stats);
7666                                 nla_nest_end(msg, drop_stats_list);
7667
7668                                 health_list = nla_nest_start(msg,
7669                                                              LNET_PEER_NI_LIST_ATTR_HEALTH_STATS);
7670                                 health_stats = nla_nest_start(msg, 0);
7671                                 nla_put_s32(msg,
7672                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_VALUE,
7673                                             atomic_read(&lpni->lpni_healthv));
7674                                 nla_put_u32(msg,
7675                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_DROPPED,
7676                                             atomic_read(&lpni->lpni_hstats.hlt_remote_dropped));
7677                                 nla_put_u32(msg,
7678                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_TIMEOUT,
7679                                             atomic_read(&lpni->lpni_hstats.hlt_remote_timeout));
7680                                 nla_put_u32(msg,
7681                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_ERROR,
7682                                             atomic_read(&lpni->lpni_hstats.hlt_remote_error));
7683                                 nla_put_u32(msg,
7684                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_NETWORK_TIMEOUT,
7685                                             atomic_read(&lpni->lpni_hstats.hlt_network_timeout));
7686                                 nla_put_u32(msg,
7687                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_PING_COUNT,
7688                                             lpni->lpni_ping_count);
7689                                 nla_put_s64(msg,
7690                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_NEXT_PING,
7691                                             lpni->lpni_next_ping,
7692                                             LNET_PEER_NI_LIST_HEALTH_STATS_ATTR_PAD);
7693                                 nla_nest_end(msg, health_stats);
7694                                 nla_nest_end(msg, health_list);
7695                         }
7696 skip_msg_stats:
7697                         nla_nest_end(msg, peer_nid);
7698                 }
7699                 nla_nest_end(msg, nid_list);
7700
7701                 genlmsg_end(msg, hdr);
7702                 lnet_peer_decref_locked(lp);
7703         }
7704         plist->lgpl_index = idx;
7705 unlock_api_mutex:
7706         mutex_unlock(&the_lnet.ln_api_mutex);
7707 send_error:
7708         return lnet_nl_send_error(cb->skb, portid, seq, rc);
7709 };
7710
7711 #ifndef HAVE_NETLINK_CALLBACK_START
7712 static int lnet_old_peer_ni_show_dump(struct sk_buff *msg,
7713                                       struct netlink_callback *cb)
7714 {
7715         if (!cb->args[0]) {
7716                 int rc = lnet_peer_ni_show_start(cb);
7717
7718                 if (rc < 0)
7719                         return lnet_nl_send_error(cb->skb,
7720                                                   NETLINK_CB(cb->skb).portid,
7721                                                   cb->nlh->nlmsg_seq,
7722                                                   rc);
7723         }
7724
7725         return lnet_peer_ni_show_dump(msg, cb);
7726 }
7727 #endif
7728
7729 static int lnet_route_cmd(struct sk_buff *skb, struct genl_info *info)
7730 {
7731         struct nlmsghdr *nlh = nlmsg_hdr(skb);
7732         struct genlmsghdr *gnlh = nlmsg_data(nlh);
7733         struct nlattr *params = genlmsg_data(gnlh);
7734         int msg_len, rem, rc = 0;
7735         struct nlattr *attr;
7736
7737         mutex_lock(&the_lnet.ln_api_mutex);
7738         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
7739                 GENL_SET_ERR_MSG(info, "Network is down");
7740                 mutex_unlock(&the_lnet.ln_api_mutex);
7741                 return -ENETDOWN;
7742         }
7743
7744         msg_len = genlmsg_len(gnlh);
7745         if (!msg_len) {
7746                 GENL_SET_ERR_MSG(info, "no configuration");
7747                 mutex_unlock(&the_lnet.ln_api_mutex);
7748                 return -ENOMSG;
7749         }
7750
7751         if (!(nla_type(params) & LN_SCALAR_ATTR_LIST)) {
7752                 GENL_SET_ERR_MSG(info, "invalid configuration");
7753                 mutex_unlock(&the_lnet.ln_api_mutex);
7754                 return -EINVAL;
7755         }
7756
7757         nla_for_each_nested(attr, params, rem) {
7758                 u32 net_id = LNET_NET_ANY, hops = LNET_UNDEFINED_HOPS;
7759                 u32 priority = 0, sensitivity = 1;
7760                 struct lnet_nid gw_nid = LNET_ANY_NID;
7761                 struct nlattr *route_prop;
7762                 bool alive = true;
7763                 s64 when = 0;
7764                 int rem2;
7765
7766                 if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
7767                         continue;
7768
7769                 nla_for_each_nested(route_prop, attr, rem2) {
7770                         char tmp[LNET_NIDSTR_SIZE];
7771                         ssize_t len;
7772                         s64 num;
7773
7774                         if (nla_type(route_prop) != LN_SCALAR_ATTR_VALUE)
7775                                 continue;
7776
7777                         if (nla_strcmp(route_prop, "net") == 0) {
7778                                 route_prop = nla_next(route_prop, &rem2);
7779                                 if (nla_type(route_prop) !=
7780                                     LN_SCALAR_ATTR_VALUE) {
7781                                         GENL_SET_ERR_MSG(info,
7782                                                          "net is invalid key");
7783                                         GOTO(report_err, rc = -EINVAL);
7784                                 }
7785
7786                                 len = nla_strscpy(tmp, route_prop, sizeof(tmp));
7787                                 if (len < 0) {
7788                                         GENL_SET_ERR_MSG(info,
7789                                                          "net key string is invalid");
7790                                         GOTO(report_err, rc = len);
7791                                 }
7792
7793                                 net_id = libcfs_str2net(tmp);
7794                                 if (!net_id) {
7795                                         GENL_SET_ERR_MSG(info,
7796                                                          "cannot parse remote net");
7797                                         GOTO(report_err, rc = -ENODEV);
7798                                 }
7799
7800                                 if (LNET_NETTYP(net_id) == LOLND) {
7801                                         GENL_SET_ERR_MSG(info,
7802                                                          "setting @lo not allowed");
7803                                         GOTO(report_err, rc = -EACCES);
7804                                 }
7805
7806                                 if (net_id == LNET_NET_ANY) {
7807                                         GENL_SET_ERR_MSG(info,
7808                                                          "setting LNET_NET_ANY not allowed");
7809                                         GOTO(report_err, rc = -ENXIO);
7810                                 }
7811                         } else if (nla_strcmp(route_prop, "gateway") == 0) {
7812                                 route_prop = nla_next(route_prop, &rem2);
7813                                 if (nla_type(route_prop) !=
7814                                     LN_SCALAR_ATTR_VALUE) {
7815                                         GENL_SET_ERR_MSG(info,
7816                                                          "gateway is invalid key");
7817                                         GOTO(report_err, rc = -EINVAL);
7818                                 }
7819
7820                                 len = nla_strscpy(tmp, route_prop, sizeof(tmp));
7821                                 if (len < 0) {
7822                                         GENL_SET_ERR_MSG(info,
7823                                                          "gateway string is invalid");
7824                                         GOTO(report_err, rc = len);
7825                                 }
7826
7827                                 rc = libcfs_strnid(&gw_nid, strim(tmp));
7828                                 if (rc < 0) {
7829                                         GENL_SET_ERR_MSG(info,
7830                                                          "cannot parse gateway");
7831                                         GOTO(report_err, rc = -ENODEV);
7832                                 }
7833                         } else if (nla_strcmp(route_prop, "state") == 0) {
7834                                 route_prop = nla_next(route_prop, &rem2);
7835                                 if (nla_type(route_prop) !=
7836                                     LN_SCALAR_ATTR_VALUE) {
7837                                         GENL_SET_ERR_MSG(info,
7838                                                          "state is invalid key");
7839                                         GOTO(report_err, rc = -EINVAL);
7840                                 }
7841
7842                                 if (nla_strcmp(route_prop, "down") == 0) {
7843                                         alive = false;
7844                                 } else if (nla_strcmp(route_prop, "up") == 0) {
7845                                         alive = true;
7846                                 } else {
7847                                         GENL_SET_ERR_MSG(info,
7848                                                          "status string bad value");
7849                                         GOTO(report_err, rc = -EINVAL);
7850                                 }
7851                         } else if (nla_strcmp(route_prop, "notify_time") == 0) {
7852                                 route_prop = nla_next(route_prop, &rem2);
7853                                 if (nla_type(route_prop) !=
7854                                     LN_SCALAR_ATTR_INT_VALUE) {
7855                                         GENL_SET_ERR_MSG(info,
7856                                                          "notify_time is invalid key");
7857                                         GOTO(report_err, rc = -EINVAL);
7858                                 }
7859
7860                                 when = nla_get_s64(route_prop);
7861                                 if (ktime_get_real_seconds() < when) {
7862                                         GENL_SET_ERR_MSG(info,
7863                                                          "notify_time is in the future");
7864                                         GOTO(report_err, rc = -EINVAL);
7865                                 }
7866                         } else if (nla_strcmp(route_prop, "hop") == 0) {
7867                                 route_prop = nla_next(route_prop, &rem2);
7868                                 if (nla_type(route_prop) !=
7869                                     LN_SCALAR_ATTR_INT_VALUE) {
7870                                         GENL_SET_ERR_MSG(info,
7871                                                          "hop has invalid key");
7872                                         GOTO(report_err, rc = -EINVAL);
7873                                 }
7874
7875                                 hops = nla_get_s64(route_prop);
7876                                 if ((hops < 1 || hops > 255) && hops != -1) {
7877                                         GENL_SET_ERR_MSG(info,
7878                                                          "invalid hop count must be between 1 and 255");
7879                                         GOTO(report_err, rc = -EINVAL);
7880                                 }
7881                         } else if (nla_strcmp(route_prop, "priority") == 0) {
7882                                 route_prop = nla_next(route_prop, &rem2);
7883                                 if (nla_type(route_prop) !=
7884                                     LN_SCALAR_ATTR_INT_VALUE) {
7885                                         GENL_SET_ERR_MSG(info,
7886                                                          "priority has invalid key");
7887                                         GOTO(report_err, rc = -EINVAL);
7888                                 }
7889
7890                                 num = nla_get_s64(route_prop);
7891                                 if (num < 0) {
7892                                         GENL_SET_ERR_MSG(info,
7893                                                          "invalid priority, must not be negative");
7894                                         GOTO(report_err, rc = -EINVAL);
7895                                 }
7896                                 priority = num;
7897                         } else if (nla_strcmp(route_prop,
7898                                               "health_sensitivity") == 0) {
7899                                 route_prop = nla_next(route_prop, &rem2);
7900                                 if (nla_type(route_prop) !=
7901                                     LN_SCALAR_ATTR_INT_VALUE) {
7902                                         GENL_SET_ERR_MSG(info,
7903                                                          "sensitivity has invalid key");
7904                                         GOTO(report_err, rc = -EINVAL);
7905                                 }
7906
7907                                 num = nla_get_s64(route_prop);
7908                                 if (num < 1) {
7909                                         GENL_SET_ERR_MSG(info,
7910                                                          "invalid health sensitivity, must be 1 or greater");
7911                                         GOTO(report_err, rc = -EINVAL);
7912                                 }
7913                                 sensitivity = num;
7914                         }
7915                 }
7916
7917                 if (net_id == LNET_NET_ANY) {
7918                         GENL_SET_ERR_MSG(info,
7919                                          "missing mandatory parameter: network");
7920                         GOTO(report_err, rc = -ENODEV);
7921                 }
7922
7923                 if (LNET_NID_IS_ANY(&gw_nid)) {
7924                         GENL_SET_ERR_MSG(info,
7925                                          "missing mandatory parameter: gateway");
7926                         GOTO(report_err, rc = -ENODEV);
7927                 }
7928
7929                 if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE) {
7930                         /* Convert the user-supplied real time to monotonic.
7931                          * NB: "when" is always in the past
7932                          */
7933                         when = ktime_get_seconds() -
7934                                 (ktime_get_real_seconds() - when);
7935
7936                         mutex_unlock(&the_lnet.ln_api_mutex);
7937                         rc = lnet_notify(NULL, &gw_nid, alive, false, when);
7938                         mutex_lock(&the_lnet.ln_api_mutex);
7939                         if (rc < 0)
7940                                 GOTO(report_err, rc);
7941                         else if (the_lnet.ln_state != LNET_STATE_RUNNING)
7942                                 GOTO(report_err, rc = -ENETDOWN);
7943                 } else if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
7944                         rc = lnet_add_route(net_id, hops, &gw_nid, priority,
7945                                             sensitivity);
7946                         if (rc < 0) {
7947                                 switch (rc) {
7948                                 case -EINVAL:
7949                                         GENL_SET_ERR_MSG(info,
7950                                                          "invalid settings for route creation");
7951                                         break;
7952                                 case -EHOSTUNREACH:
7953                                         GENL_SET_ERR_MSG(info,
7954                                                          "No interface configured on the same net as gateway");
7955                                         break;
7956                                 case -ESHUTDOWN:
7957                                         GENL_SET_ERR_MSG(info,
7958                                                          "Network is down");
7959                                         break;
7960                                 case -EEXIST:
7961                                         GENL_SET_ERR_MSG(info,
7962                                                          "Route already exists or the specified network is local");
7963                                         break;
7964                                 default:
7965                                         GENL_SET_ERR_MSG(info,
7966                                                          "failed to create route");
7967                                         break;
7968                                 }
7969                                 GOTO(report_err, rc);
7970                         }
7971                 } else if (!(info->nlhdr->nlmsg_flags & NLM_F_CREATE)) {
7972                         rc = lnet_del_route(net_id, &gw_nid);
7973                         if (rc < 0) {
7974                                 GENL_SET_ERR_MSG(info,
7975                                                  "failed to delete route");
7976                                 GOTO(report_err, rc);
7977                         }
7978                 }
7979         }
7980 report_err:
7981         mutex_unlock(&the_lnet.ln_api_mutex);
7982
7983         return rc;
7984 }
7985
7986 static inline struct lnet_genl_ping_list *
7987 lnet_ping_dump_ctx(struct netlink_callback *cb)
7988 {
7989         return (struct lnet_genl_ping_list *)cb->args[0];
7990 }
7991
7992 static int lnet_ping_show_done(struct netlink_callback *cb)
7993 {
7994         struct lnet_genl_ping_list *plist = lnet_ping_dump_ctx(cb);
7995
7996         if (plist) {
7997                 genradix_free(&plist->lgpl_failed);
7998                 genradix_free(&plist->lgpl_list);
7999                 LIBCFS_FREE(plist, sizeof(*plist));
8000                 cb->args[0] = 0;
8001         }
8002
8003         return 0;
8004 }
8005
8006 /* LNet ping ->start() handler for GET requests */
8007 static int lnet_ping_show_start(struct netlink_callback *cb)
8008 {
8009         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
8010 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
8011         struct netlink_ext_ack *extack = NULL;
8012 #endif
8013         struct lnet_genl_ping_list *plist;
8014         int msg_len = genlmsg_len(gnlh);
8015         struct nlattr *params, *top;
8016         int rem, rc = 0;
8017
8018 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
8019         extack = cb->extack;
8020 #endif
8021         if (the_lnet.ln_refcount == 0) {
8022                 NL_SET_ERR_MSG(extack, "Network is down");
8023                 return -ENETDOWN;
8024         }
8025
8026         if (!msg_len) {
8027                 NL_SET_ERR_MSG(extack, "Ping needs NID targets");
8028                 return -ENOENT;
8029         }
8030
8031         LIBCFS_ALLOC(plist, sizeof(*plist));
8032         if (!plist) {
8033                 NL_SET_ERR_MSG(extack, "failed to setup ping list");
8034                 return -ENOMEM;
8035         }
8036         genradix_init(&plist->lgpl_list);
8037         plist->lgpl_timeout = cfs_time_seconds(DEFAULT_PEER_TIMEOUT);
8038         plist->lgpl_src_nid = LNET_ANY_NID;
8039         plist->lgpl_index = 0;
8040         plist->lgpl_list_count = 0;
8041         cb->args[0] = (long)plist;
8042
8043         params = genlmsg_data(gnlh);
8044         nla_for_each_attr(top, params, msg_len, rem) {
8045                 struct nlattr *nids;
8046                 int rem2;
8047
8048                 switch (nla_type(top)) {
8049                 case LN_SCALAR_ATTR_VALUE:
8050                         if (nla_strcmp(top, "timeout") == 0) {
8051                                 s64 timeout;
8052
8053                                 top = nla_next(top, &rem);
8054                                 if (nla_type(top) != LN_SCALAR_ATTR_INT_VALUE) {
8055                                         NL_SET_ERR_MSG(extack,
8056                                                        "invalid timeout param");
8057                                         GOTO(report_err, rc = -EINVAL);
8058                                 }
8059
8060                                 /* If timeout is negative then set default of
8061                                  * 3 minutes
8062                                  */
8063                                 timeout = nla_get_s64(top);
8064                                 if (timeout > 0 &&
8065                                     timeout < (DEFAULT_PEER_TIMEOUT * MSEC_PER_SEC))
8066                                         plist->lgpl_timeout =
8067                                                 nsecs_to_jiffies(timeout * NSEC_PER_MSEC);
8068                         } else if (nla_strcmp(top, "source") == 0) {
8069                                 char nidstr[LNET_NIDSTR_SIZE + 1];
8070
8071                                 top = nla_next(top, &rem);
8072                                 if (nla_type(top) != LN_SCALAR_ATTR_VALUE) {
8073                                         NL_SET_ERR_MSG(extack,
8074                                                        "invalid source param");
8075                                         GOTO(report_err, rc = -EINVAL);
8076                                 }
8077
8078                                 rc = nla_strscpy(nidstr, top, sizeof(nidstr));
8079                                 if (rc < 0) {
8080                                         NL_SET_ERR_MSG(extack,
8081                                                        "failed to parse source nid");
8082                                         GOTO(report_err, rc);
8083                                 }
8084
8085                                 rc = libcfs_strnid(&plist->lgpl_src_nid,
8086                                                    strim(nidstr));
8087                                 if (rc < 0) {
8088                                         NL_SET_ERR_MSG(extack,
8089                                                        "invalid source nid");
8090                                         GOTO(report_err, rc);
8091                                 }
8092                                 rc = 0;
8093                         }
8094                         break;
8095                 case LN_SCALAR_ATTR_LIST:
8096                         nla_for_each_nested(nids, top, rem2) {
8097                                 char nid[LNET_NIDSTR_SIZE + 1];
8098                                 struct lnet_processid *id;
8099
8100                                 if (nla_type(nids) != LN_SCALAR_ATTR_VALUE)
8101                                         continue;
8102
8103                                 memset(nid, 0, sizeof(nid));
8104                                 rc = nla_strscpy(nid, nids, sizeof(nid));
8105                                 if (rc < 0) {
8106                                         NL_SET_ERR_MSG(extack,
8107                                                        "failed to get NID");
8108                                         GOTO(report_err, rc);
8109                                 }
8110
8111                                 id = genradix_ptr_alloc(&plist->lgpl_list,
8112                                                         plist->lgpl_list_count++,
8113                                                         GFP_KERNEL);
8114                                 if (!id) {
8115                                         NL_SET_ERR_MSG(extack,
8116                                                        "failed to allocate NID");
8117                                         GOTO(report_err, rc = -ENOMEM);
8118                                 }
8119
8120                                 rc = libcfs_strid(id, strim(nid));
8121                                 if (rc < 0) {
8122                                         NL_SET_ERR_MSG(extack, "cannot parse NID");
8123                                         GOTO(report_err, rc);
8124                                 }
8125                                 rc = 0;
8126                         }
8127                         fallthrough;
8128                 default:
8129                         break;
8130                 }
8131         }
8132 report_err:
8133         if (rc < 0)
8134                 lnet_ping_show_done(cb);
8135
8136         return rc;
8137 }
8138
8139 static const struct ln_key_list ping_err_props_list = {
8140         .lkl_maxattr                    = LNET_ERR_ATTR_MAX,
8141         .lkl_list                       = {
8142                 [LNET_ERR_ATTR_HDR]             = {
8143                         .lkp_value              = "manage",
8144                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8145                         .lkp_data_type          = NLA_NUL_STRING,
8146                 },
8147                 [LNET_ERR_ATTR_TYPE]            = {
8148                         .lkp_value              = "ping",
8149                         .lkp_data_type          = NLA_STRING,
8150                 },
8151                 [LNET_ERR_ATTR_ERRNO]           = {
8152                         .lkp_value              = "errno",
8153                         .lkp_data_type          = NLA_S16,
8154                 },
8155                 [LNET_ERR_ATTR_DESCR]           = {
8156                         .lkp_value              = "descr",
8157                         .lkp_data_type          = NLA_STRING,
8158                 },
8159         },
8160 };
8161
8162 static const struct ln_key_list ping_props_list = {
8163         .lkl_maxattr                    = LNET_PING_ATTR_MAX,
8164         .lkl_list                       = {
8165                 [LNET_PING_ATTR_HDR]            = {
8166                         .lkp_value              = "ping",
8167                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8168                         .lkp_data_type          = NLA_NUL_STRING,
8169                 },
8170                 [LNET_PING_ATTR_PRIMARY_NID]    = {
8171                         .lkp_value              = "primary nid",
8172                         .lkp_data_type          = NLA_STRING
8173                 },
8174                 [LNET_PING_ATTR_ERRNO]          = {
8175                         .lkp_value              = "errno",
8176                         .lkp_data_type          = NLA_S16
8177                 },
8178                 [LNET_PING_ATTR_MULTIRAIL]      = {
8179                         .lkp_value              = "Multi-Rail",
8180                         .lkp_data_type          = NLA_FLAG
8181                 },
8182                 [LNET_PING_ATTR_PEER_NI_LIST]   = {
8183                         .lkp_value              = "peer_ni",
8184                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8185                         .lkp_data_type          = NLA_NESTED
8186                 },
8187         },
8188 };
8189
8190 static const struct ln_key_list ping_peer_ni_list = {
8191         .lkl_maxattr                    = LNET_PING_PEER_NI_ATTR_MAX,
8192         .lkl_list                       = {
8193                 [LNET_PING_PEER_NI_ATTR_NID]    = {
8194                         .lkp_value              = "nid",
8195                         .lkp_data_type          = NLA_STRING
8196                 },
8197         },
8198 };
8199
8200 static int lnet_ping_show_dump(struct sk_buff *msg,
8201                                struct netlink_callback *cb)
8202 {
8203         struct lnet_genl_ping_list *plist = lnet_ping_dump_ctx(cb);
8204 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
8205         struct netlink_ext_ack *extack = NULL;
8206 #endif
8207         int portid = NETLINK_CB(cb->skb).portid;
8208         int seq = cb->nlh->nlmsg_seq;
8209         int idx = plist->lgpl_index;
8210         int rc = 0, i = 0;
8211
8212 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
8213         extack = cb->extack;
8214 #endif
8215         if (!plist->lgpl_index) {
8216                 const struct ln_key_list *all[] = {
8217                         &ping_props_list, &ping_peer_ni_list, NULL
8218                 };
8219
8220                 rc = lnet_genl_send_scalar_list(msg, portid, seq,
8221                                                 &lnet_family,
8222                                                 NLM_F_CREATE | NLM_F_MULTI,
8223                                                 LNET_CMD_PING, all);
8224                 if (rc < 0) {
8225                         NL_SET_ERR_MSG(extack, "failed to send key table");
8226                         GOTO(send_error, rc);
8227                 }
8228
8229                 genradix_init(&plist->lgpl_failed);
8230         }
8231
8232         while (idx < plist->lgpl_list_count) {
8233                 struct lnet_nid primary_nid = LNET_ANY_NID;
8234                 struct lnet_genl_ping_list peers;
8235                 struct lnet_processid *id;
8236                 struct nlattr *nid_list;
8237                 struct lnet_peer *lp;
8238                 bool mr_flag = false;
8239                 unsigned int count;
8240                 void *hdr = NULL;
8241
8242                 id = genradix_ptr(&plist->lgpl_list, idx++);
8243
8244                 rc = lnet_ping(id, &plist->lgpl_src_nid, plist->lgpl_timeout,
8245                                &peers, lnet_interfaces_max);
8246                 if (rc < 0) {
8247                         struct lnet_fail_ping *fail;
8248
8249                         fail = genradix_ptr_alloc(&plist->lgpl_failed,
8250                                                   plist->lgpl_failed_count++,
8251                                                   GFP_KERNEL);
8252                         if (!fail) {
8253                                 NL_SET_ERR_MSG(extack,
8254                                                "failed to allocate failed NID");
8255                                 GOTO(send_error, rc);
8256                         }
8257                         memset(fail->lfp_msg, '\0', sizeof(fail->lfp_msg));
8258                         snprintf(fail->lfp_msg, sizeof(fail->lfp_msg),
8259                                  "failed to ping %s",
8260                                  libcfs_nidstr(&id->nid));
8261                         fail->lfp_id = *id;
8262                         fail->lfp_errno = rc;
8263                         goto cant_reach;
8264                 }
8265
8266                 mutex_lock(&the_lnet.ln_api_mutex);
8267                 lp = lnet_find_peer(&id->nid);
8268                 if (lp) {
8269                         primary_nid = lp->lp_primary_nid;
8270                         mr_flag = lnet_peer_is_multi_rail(lp);
8271                         lnet_peer_decref_locked(lp);
8272                 }
8273                 mutex_unlock(&the_lnet.ln_api_mutex);
8274
8275                 hdr = genlmsg_put(msg, portid, seq, &lnet_family,
8276                                   NLM_F_MULTI, LNET_CMD_PING);
8277                 if (!hdr) {
8278                         NL_SET_ERR_MSG(extack, "failed to send values");
8279                         genlmsg_cancel(msg, hdr);
8280                         GOTO(send_error, rc = -EMSGSIZE);
8281                 }
8282
8283                 if (i++ == 0)
8284                         nla_put_string(msg, LNET_PING_ATTR_HDR, "");
8285
8286                 nla_put_string(msg, LNET_PING_ATTR_PRIMARY_NID,
8287                                libcfs_nidstr(&primary_nid));
8288                 if (mr_flag)
8289                         nla_put_flag(msg, LNET_PING_ATTR_MULTIRAIL);
8290
8291                 nid_list = nla_nest_start(msg, LNET_PING_ATTR_PEER_NI_LIST);
8292                 for (count = 0; count < rc; count++) {
8293                         struct lnet_processid *result;
8294                         struct nlattr *nid_attr;
8295                         char *idstr;
8296
8297                         result = genradix_ptr(&peers.lgpl_list, count);
8298                         if (nid_is_lo0(&result->nid))
8299                                 continue;
8300
8301                         nid_attr = nla_nest_start(msg, count + 1);
8302                         if (id->pid == LNET_PID_LUSTRE)
8303                                 idstr = libcfs_nidstr(&result->nid);
8304                         else
8305                                 idstr = libcfs_idstr(result);
8306                         nla_put_string(msg, LNET_PING_PEER_NI_ATTR_NID, idstr);
8307                         nla_nest_end(msg, nid_attr);
8308                 }
8309                 nla_nest_end(msg, nid_list);
8310                 genlmsg_end(msg, hdr);
8311 cant_reach:
8312                 genradix_free(&peers.lgpl_list);
8313         }
8314
8315         if (plist->lgpl_failed_count) {
8316                 int flags = NLM_F_CREATE | NLM_F_REPLACE | NLM_F_MULTI;
8317                 const struct ln_key_list *fail[] = {
8318                         &ping_err_props_list, NULL
8319                 };
8320
8321                 rc = lnet_genl_send_scalar_list(msg, portid, seq, &lnet_family,
8322                                                 flags, LNET_CMD_PING, fail);
8323                 if (rc < 0) {
8324                         NL_SET_ERR_MSG(extack,
8325                                        "failed to send new key table");
8326                         GOTO(send_error, rc);
8327                 }
8328
8329                 for (i = 0; i < plist->lgpl_failed_count; i++) {
8330                         struct lnet_fail_ping *fail;
8331                         void *hdr;
8332
8333                         fail = genradix_ptr(&plist->lgpl_failed, i);
8334
8335                         hdr = genlmsg_put(msg, portid, seq, &lnet_family,
8336                                           NLM_F_MULTI, LNET_CMD_PING);
8337                         if (!hdr) {
8338                                 NL_SET_ERR_MSG(extack,
8339                                                "failed to send failed values");
8340                                 genlmsg_cancel(msg, hdr);
8341                                 GOTO(send_error, rc = -EMSGSIZE);
8342                         }
8343
8344                         if (i == 0)
8345                                 nla_put_string(msg, LNET_ERR_ATTR_HDR, "");
8346
8347                         nla_put_string(msg, LNET_ERR_ATTR_TYPE, "\n");
8348                         nla_put_s16(msg, LNET_ERR_ATTR_ERRNO,
8349                                     fail->lfp_errno);
8350                         nla_put_string(msg, LNET_ERR_ATTR_DESCR,
8351                                        fail->lfp_msg);
8352                         genlmsg_end(msg, hdr);
8353                 }
8354         }
8355         genradix_free(&plist->lgpl_list);
8356         rc = 0; /* don't treat it as an error */
8357
8358         plist->lgpl_index = idx;
8359 send_error:
8360         return lnet_nl_send_error(cb->skb, portid, seq, rc);
8361 }
8362
8363 #ifndef HAVE_NETLINK_CALLBACK_START
8364 static int lnet_old_ping_show_dump(struct sk_buff *msg,
8365                                    struct netlink_callback *cb)
8366 {
8367         if (!cb->args[0]) {
8368                 int rc = lnet_ping_show_start(cb);
8369
8370                 if (rc < 0)
8371                         return lnet_nl_send_error(cb->skb,
8372                                                   NETLINK_CB(cb->skb).portid,
8373                                                   cb->nlh->nlmsg_seq,
8374                                                   rc);
8375         }
8376
8377         return lnet_ping_show_dump(msg, cb);
8378 }
8379 #endif
8380
8381 static const struct ln_key_list discover_err_props_list = {
8382         .lkl_maxattr                    = LNET_ERR_ATTR_MAX,
8383         .lkl_list                       = {
8384                 [LNET_ERR_ATTR_HDR]             = {
8385                         .lkp_value              = "manage",
8386                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8387                         .lkp_data_type          = NLA_NUL_STRING,
8388                 },
8389                 [LNET_ERR_ATTR_TYPE]            = {
8390                         .lkp_value              = "discover",
8391                         .lkp_data_type          = NLA_STRING,
8392                 },
8393                 [LNET_ERR_ATTR_ERRNO]           = {
8394                         .lkp_value              = "errno",
8395                         .lkp_data_type          = NLA_S16,
8396                 },
8397                 [LNET_ERR_ATTR_DESCR]           = {
8398                         .lkp_value              = "descr",
8399                         .lkp_data_type          = NLA_STRING,
8400                 },
8401         },
8402 };
8403
8404 static const struct ln_key_list discover_props_list = {
8405         .lkl_maxattr                    = LNET_PING_ATTR_MAX,
8406         .lkl_list                       = {
8407                 [LNET_PING_ATTR_HDR]            = {
8408                         .lkp_value              = "discover",
8409                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8410                         .lkp_data_type          = NLA_NUL_STRING,
8411                 },
8412                 [LNET_PING_ATTR_PRIMARY_NID]    = {
8413                         .lkp_value              = "primary nid",
8414                         .lkp_data_type          = NLA_STRING
8415                 },
8416                 [LNET_PING_ATTR_ERRNO]          = {
8417                         .lkp_value              = "errno",
8418                         .lkp_data_type          = NLA_S16
8419                 },
8420                 [LNET_PING_ATTR_MULTIRAIL]      = {
8421                         .lkp_value              = "Multi-Rail",
8422                         .lkp_data_type          = NLA_FLAG
8423                 },
8424                 [LNET_PING_ATTR_PEER_NI_LIST]   = {
8425                         .lkp_value              = "peer_ni",
8426                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8427                         .lkp_data_type          = NLA_NESTED
8428                 },
8429         },
8430 };
8431
8432 static int lnet_ping_cmd(struct sk_buff *skb, struct genl_info *info)
8433 {
8434         const struct ln_key_list *all[] = {
8435                 &discover_props_list, &ping_peer_ni_list, NULL
8436         };
8437         struct nlmsghdr *nlh = nlmsg_hdr(skb);
8438         struct genlmsghdr *gnlh = nlmsg_data(nlh);
8439         struct nlattr *params = genlmsg_data(gnlh);
8440         struct lnet_genl_ping_list dlists;
8441         int msg_len, rem, rc = 0, i;
8442         bool clear_hdr = false;
8443         struct sk_buff *reply;
8444         struct nlattr *attr;
8445         void *hdr = NULL;
8446
8447         msg_len = genlmsg_len(gnlh);
8448         if (!msg_len) {
8449                 GENL_SET_ERR_MSG(info, "no configuration");
8450                 return -ENOMSG;
8451         }
8452
8453         if (!(info->nlhdr->nlmsg_flags & NLM_F_CREATE)) {
8454                 GENL_SET_ERR_MSG(info, "only NLM_F_CREATE setting is allowed");
8455                 return -EINVAL;
8456         }
8457
8458         reply = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8459         if (!reply) {
8460                 GENL_SET_ERR_MSG(info,
8461                                  "fail to allocate reply");
8462                 return -ENOMEM;
8463         }
8464
8465         genradix_init(&dlists.lgpl_failed);
8466         dlists.lgpl_failed_count = 0;
8467         genradix_init(&dlists.lgpl_list);
8468         dlists.lgpl_list_count = 0;
8469
8470         rc = lnet_genl_send_scalar_list(reply, info->snd_portid,
8471                                         info->snd_seq, &lnet_family,
8472                                         NLM_F_CREATE | NLM_F_MULTI,
8473                                         LNET_CMD_PING, all);
8474         if (rc < 0) {
8475                 GENL_SET_ERR_MSG(info,
8476                                  "failed to send key table");
8477                 GOTO(report_err, rc);
8478         }
8479
8480         nla_for_each_attr(attr, params, msg_len, rem) {
8481                 struct nlattr *nids;
8482                 int rem2;
8483
8484                 /* We only care about the NID list to discover with */
8485                 if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
8486                         continue;
8487
8488                 nla_for_each_nested(nids, attr, rem2) {
8489                         char nid[LNET_NIDSTR_SIZE + 1];
8490                         struct lnet_processid id;
8491                         struct nlattr *nid_list;
8492                         struct lnet_peer *lp;
8493                         ssize_t len;
8494
8495                         if (nla_type(nids) != LN_SCALAR_ATTR_VALUE)
8496                                 continue;
8497
8498                         memset(nid, 0, sizeof(nid));
8499                         rc = nla_strscpy(nid, nids, sizeof(nid));
8500                         if (rc < 0) {
8501                                 GENL_SET_ERR_MSG(info,
8502                                                  "failed to get NID");
8503                                 GOTO(report_err, rc);
8504                         }
8505
8506                         len = libcfs_strid(&id, strim(nid));
8507                         if (len < 0) {
8508                                 struct lnet_fail_ping *fail;
8509
8510                                 fail = genradix_ptr_alloc(&dlists.lgpl_failed,
8511                                                           dlists.lgpl_failed_count++,
8512                                                           GFP_KERNEL);
8513                                 if (!fail) {
8514                                         GENL_SET_ERR_MSG(info,
8515                                                          "failed to allocate improper NID");
8516                                         GOTO(report_err, rc = -ENOMEM);
8517                                 }
8518                                 memset(fail->lfp_msg, '\0', sizeof(fail->lfp_msg));
8519                                 snprintf(fail->lfp_msg, sizeof(fail->lfp_msg),
8520                                          "cannot parse NID '%s'", strim(nid));
8521                                 fail->lfp_id = id;
8522                                 fail->lfp_errno = len;
8523                                 continue;
8524                         }
8525
8526                         if (LNET_NID_IS_ANY(&id.nid))
8527                                 continue;
8528
8529                         rc = lnet_discover(&id,
8530                                            info->nlhdr->nlmsg_flags & NLM_F_EXCL,
8531                                            &dlists);
8532                         if (rc < 0) {
8533                                 struct lnet_fail_ping *fail;
8534
8535                                 fail = genradix_ptr_alloc(&dlists.lgpl_failed,
8536                                                           dlists.lgpl_failed_count++,
8537                                                           GFP_KERNEL);
8538                                 if (!fail) {
8539                                         GENL_SET_ERR_MSG(info,
8540                                                          "failed to allocate failed NID");
8541                                         GOTO(report_err, rc = -ENOMEM);
8542                                 }
8543                                 memset(fail->lfp_msg, '\0', sizeof(fail->lfp_msg));
8544                                 snprintf(fail->lfp_msg, sizeof(fail->lfp_msg),
8545                                          "failed to discover %s",
8546                                          libcfs_nidstr(&id.nid));
8547                                 fail->lfp_id = id;
8548                                 fail->lfp_errno = rc;
8549                                 continue;
8550                         }
8551
8552                         /* create the genetlink message header */
8553                         hdr = genlmsg_put(reply, info->snd_portid, info->snd_seq,
8554                                           &lnet_family, NLM_F_MULTI, LNET_CMD_PING);
8555                         if (!hdr) {
8556                                 GENL_SET_ERR_MSG(info,
8557                                                  "failed to allocate hdr");
8558                                 GOTO(report_err, rc = -ENOMEM);
8559                         }
8560
8561                         if (!clear_hdr) {
8562                                 nla_put_string(reply, LNET_PING_ATTR_HDR, "");
8563                                 clear_hdr = true;
8564                         }
8565
8566                         lp = lnet_find_peer(&id.nid);
8567                         if (lp) {
8568                                 nla_put_string(reply, LNET_PING_ATTR_PRIMARY_NID,
8569                                                libcfs_nidstr(&lp->lp_primary_nid));
8570                                 if (lnet_peer_is_multi_rail(lp))
8571                                         nla_put_flag(reply, LNET_PING_ATTR_MULTIRAIL);
8572                                 lnet_peer_decref_locked(lp);
8573                         }
8574
8575                         nid_list = nla_nest_start(reply, LNET_PING_ATTR_PEER_NI_LIST);
8576                         for (i = 0; i < dlists.lgpl_list_count; i++) {
8577                                 struct lnet_processid *found;
8578                                 struct nlattr *nid_attr;
8579                                 char *idstr;
8580
8581                                 found = genradix_ptr(&dlists.lgpl_list, i);
8582                                 if (nid_is_lo0(&found->nid))
8583                                         continue;
8584
8585                                 nid_attr = nla_nest_start(reply, i + 1);
8586                                 if (id.pid == LNET_PID_LUSTRE)
8587                                         idstr = libcfs_nidstr(&found->nid);
8588                                 else
8589                                         idstr = libcfs_idstr(found);
8590                                 nla_put_string(reply, LNET_PING_PEER_NI_ATTR_NID, idstr);
8591                                 nla_nest_end(reply, nid_attr);
8592                         }
8593                         nla_nest_end(reply, nid_list);
8594
8595                         genlmsg_end(reply, hdr);
8596                 }
8597         }
8598
8599         if (dlists.lgpl_failed_count) {
8600                 int flags = NLM_F_CREATE | NLM_F_REPLACE | NLM_F_MULTI;
8601                 const struct ln_key_list *fail[] = {
8602                         &discover_err_props_list, NULL
8603                 };
8604
8605                 rc = lnet_genl_send_scalar_list(reply, info->snd_portid,
8606                                                 info->snd_seq, &lnet_family,
8607                                                 flags, LNET_CMD_PING, fail);
8608                 if (rc < 0) {
8609                         GENL_SET_ERR_MSG(info,
8610                                          "failed to send new key table");
8611                         GOTO(report_err, rc);
8612                 }
8613
8614                 for (i = 0; i < dlists.lgpl_failed_count; i++) {
8615                         struct lnet_fail_ping *fail;
8616
8617                         hdr = genlmsg_put(reply, info->snd_portid, info->snd_seq,
8618                                           &lnet_family, NLM_F_MULTI, LNET_CMD_PING);
8619                         if (!hdr) {
8620                                 GENL_SET_ERR_MSG(info,
8621                                                  "failed to send failed values");
8622                                 GOTO(report_err, rc = -ENOMSG);
8623                         }
8624
8625                         fail = genradix_ptr(&dlists.lgpl_failed, i);
8626                         if (i == 0)
8627                                 nla_put_string(reply, LNET_ERR_ATTR_HDR, "");
8628
8629                         nla_put_string(reply, LNET_ERR_ATTR_TYPE, "\n");
8630                         nla_put_s16(reply, LNET_ERR_ATTR_ERRNO,
8631                                     fail->lfp_errno);
8632                         nla_put_string(reply, LNET_ERR_ATTR_DESCR,
8633                                        fail->lfp_msg);
8634                         genlmsg_end(reply, hdr);
8635                 }
8636         }
8637
8638         nlh = nlmsg_put(reply, info->snd_portid, info->snd_seq, NLMSG_DONE, 0,
8639                         NLM_F_MULTI);
8640         if (!nlh) {
8641                 genlmsg_cancel(reply, hdr);
8642                 GENL_SET_ERR_MSG(info,
8643                                  "failed to finish message");
8644                 GOTO(report_err, rc = -EMSGSIZE);
8645         }
8646
8647 report_err:
8648         genradix_free(&dlists.lgpl_failed);
8649         genradix_free(&dlists.lgpl_list);
8650
8651         if (rc < 0) {
8652                 genlmsg_cancel(reply, hdr);
8653                 nlmsg_free(reply);
8654         } else {
8655                 rc = genlmsg_reply(reply, info);
8656         }
8657
8658         return rc;
8659 }
8660
8661 #define lnet_peer_dist_show_done        lnet_peer_ni_show_done
8662
8663 static int lnet_peer_dist_show_start(struct netlink_callback *cb)
8664 {
8665         struct genlmsghdr *gnlh = nlmsg_data(cb->nlh);
8666 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
8667         struct netlink_ext_ack *extack = NULL;
8668 #endif
8669         struct lnet_genl_processid_list *plist;
8670         int msg_len = genlmsg_len(gnlh);
8671         struct nlattr *params, *top;
8672         int rem, rc = 0;
8673
8674 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
8675         extack = cb->extack;
8676 #endif
8677         mutex_lock(&the_lnet.ln_api_mutex);
8678         if (the_lnet.ln_state != LNET_STATE_RUNNING) {
8679                 NL_SET_ERR_MSG(extack, "Network is down");
8680                 mutex_unlock(&the_lnet.ln_api_mutex);
8681                 return -ENETDOWN;
8682         }
8683
8684         msg_len = genlmsg_len(gnlh);
8685         if (!msg_len) {
8686                 NL_SET_ERR_MSG(extack, "Missing NID argument(s)");
8687                 mutex_unlock(&the_lnet.ln_api_mutex);
8688                 return -ENOENT;
8689         }
8690
8691         CFS_ALLOC_PTR(plist);
8692         if (!plist) {
8693                 NL_SET_ERR_MSG(extack, "No memory for peer NID list");
8694                 mutex_unlock(&the_lnet.ln_api_mutex);
8695                 return -ENOMEM;
8696         }
8697
8698         genradix_init(&plist->lgpl_list);
8699         plist->lgpl_count = 0;
8700         plist->lgpl_index = 0;
8701         cb->args[0] = (long)plist;
8702
8703         params = genlmsg_data(gnlh);
8704         nla_for_each_attr(top, params, msg_len, rem) {
8705                 struct nlattr *nids;
8706                 int rem2;
8707
8708                 if (nla_type(top) != LN_SCALAR_ATTR_LIST)
8709                         continue;
8710
8711                 nla_for_each_nested(nids, top, rem2) {
8712                         char nidstr[LNET_NIDSTR_SIZE + 1];
8713                         struct lnet_processid *id;
8714
8715                         if (nla_type(nids) != LN_SCALAR_ATTR_VALUE)
8716                                 continue;
8717
8718                         memset(nidstr, 0, sizeof(nidstr));
8719                         rc = nla_strscpy(nidstr, nids, sizeof(nidstr));
8720                         if (rc < 0) {
8721                                 NL_SET_ERR_MSG(extack,
8722                                                "failed to get NID");
8723                                 GOTO(report_err, rc);
8724                         }
8725
8726                         id = genradix_ptr_alloc(&plist->lgpl_list,
8727                                                 plist->lgpl_count++,
8728                                                 GFP_KERNEL);
8729                         if (!id) {
8730                                 NL_SET_ERR_MSG(extack, "failed to allocate NID");
8731                                 GOTO(report_err, rc = -ENOMEM);
8732                         }
8733
8734                         rc = libcfs_strid(id, strim(nidstr));
8735                         if (rc < 0) {
8736                                 NL_SET_ERR_MSG(extack, "invalid NID");
8737                                 GOTO(report_err, rc);
8738                         }
8739                         rc = 0;
8740                 }
8741         }
8742 report_err:
8743         mutex_unlock(&the_lnet.ln_api_mutex);
8744
8745         if (rc < 0)
8746                 lnet_peer_dist_show_done(cb);
8747
8748         return rc;
8749 }
8750
8751 static const struct ln_key_list peer_dist_props_list = {
8752         .lkl_maxattr                    = LNET_PEER_DIST_ATTR_MAX,
8753         .lkl_list                       = {
8754                 [LNET_PEER_DIST_ATTR_HDR]       = {
8755                         .lkp_value              = "peer",
8756                         .lkp_key_format         = LNKF_SEQUENCE | LNKF_MAPPING,
8757                         .lkp_data_type          = NLA_NUL_STRING,
8758                 },
8759                 [LNET_PEER_DIST_ATTR_NID]       = {
8760                         .lkp_value              = "nid",
8761                         .lkp_data_type          = NLA_STRING
8762                 },
8763                 [LNET_PEER_DIST_ATTR_DIST]      = {
8764                         .lkp_value              = "distance",
8765                         .lkp_data_type          = NLA_U32
8766                 },
8767                 [LNET_PEER_DIST_ATTR_ORDER]     = {
8768                         .lkp_value              = "order",
8769                         .lkp_data_type          = NLA_U32
8770                 },
8771         },
8772 };
8773
8774 static int lnet_peer_dist_show_dump(struct sk_buff *msg,
8775                                     struct netlink_callback *cb)
8776 {
8777         struct lnet_genl_processid_list *plist = lnet_peer_dump_ctx(cb);
8778 #ifdef HAVE_NL_PARSE_WITH_EXT_ACK
8779         struct netlink_ext_ack *extack = NULL;
8780 #endif
8781         int portid = NETLINK_CB(cb->skb).portid;
8782         int seq = cb->nlh->nlmsg_seq;
8783         int idx = plist->lgpl_index;
8784         int rc = 0;
8785
8786 #ifdef HAVE_NL_DUMP_WITH_EXT_ACK
8787         extack = cb->extack;
8788 #endif
8789         if (!idx) {
8790                 const struct ln_key_list *all[] = {
8791                         &peer_dist_props_list, NULL
8792                 };
8793
8794                 rc = lnet_genl_send_scalar_list(msg, portid, seq,
8795                                                 &lnet_family,
8796                                                 NLM_F_CREATE | NLM_F_MULTI,
8797                                                 LNET_CMD_PEER_DIST, all);
8798                 if (rc < 0) {
8799                         NL_SET_ERR_MSG(extack, "failed to send key table");
8800                         GOTO(send_error, rc);
8801                 }
8802         }
8803
8804         while (idx < plist->lgpl_count) {
8805                 struct lnet_processid *id;
8806                 void *hdr;
8807                 u32 order;
8808                 int dist;
8809
8810                 id = genradix_ptr(&plist->lgpl_list, idx++);
8811                 if (nid_is_lo0(&id->nid))
8812                         continue;
8813
8814                 dist = LNetDist(&id->nid, &id->nid, &order);
8815                 if (dist < 0) {
8816                         if (dist == -EHOSTUNREACH)
8817                                 continue;
8818
8819                         rc = dist;
8820                         return rc;
8821                 }
8822
8823                 hdr = genlmsg_put(msg, portid, seq, &lnet_family,
8824                                   NLM_F_MULTI, LNET_CMD_PEER_DIST);
8825                 if (!hdr) {
8826                         NL_SET_ERR_MSG(extack, "failed to send values");
8827                         genlmsg_cancel(msg, hdr);
8828                         GOTO(send_error, rc = -EMSGSIZE);
8829                 }
8830
8831                 if (idx == 1)
8832                         nla_put_string(msg, LNET_PEER_DIST_ATTR_HDR, "");
8833
8834                 nla_put_string(msg, LNET_PEER_DIST_ATTR_NID,
8835                                libcfs_nidstr(&id->nid));
8836                 nla_put_u32(msg, LNET_PEER_DIST_ATTR_DIST, dist);
8837                 nla_put_u32(msg, LNET_PEER_DIST_ATTR_ORDER, order);
8838
8839                 genlmsg_end(msg, hdr);
8840         }
8841
8842         plist->lgpl_index = idx;
8843 send_error:
8844         return lnet_nl_send_error(cb->skb, portid, seq, rc);
8845 }
8846
8847 #ifndef HAVE_NETLINK_CALLBACK_START
8848 static int lnet_old_peer_dist_show_dump(struct sk_buff *msg,
8849                                         struct netlink_callback *cb)
8850 {
8851         if (!cb->args[0]) {
8852                 int rc = lnet_peer_dist_show_start(cb);
8853
8854                 if (rc < 0)
8855                         return lnet_nl_send_error(cb->skb,
8856                                                   NETLINK_CB(cb->skb).portid,
8857                                                   cb->nlh->nlmsg_seq,
8858                                                   rc);
8859         }
8860
8861         return lnet_peer_dist_show_dump(msg, cb);
8862 }
8863 #endif
8864
8865 static const struct genl_multicast_group lnet_mcast_grps[] = {
8866         { .name =       "ip2net",       },
8867         { .name =       "net",          },
8868         { .name =       "peer",         },
8869         { .name =       "route",        },
8870         { .name =       "ping",         },
8871         { .name =       "discover",     },
8872         { .name =       "cpt-of-nid",   },
8873 };
8874
8875 static const struct genl_ops lnet_genl_ops[] = {
8876         {
8877                 .cmd            = LNET_CMD_CONFIGURE,
8878                 .flags          = GENL_ADMIN_PERM,
8879                 .doit           = lnet_net_conf_cmd,
8880         },
8881         {
8882                 .cmd            = LNET_CMD_NETS,
8883                 .flags          = GENL_ADMIN_PERM,
8884 #ifdef HAVE_NETLINK_CALLBACK_START
8885                 .start          = lnet_net_show_start,
8886                 .dumpit         = lnet_net_show_dump,
8887 #else
8888                 .dumpit         = lnet_old_net_show_dump,
8889 #endif
8890                 .done           = lnet_net_show_done,
8891                 .doit           = lnet_net_cmd,
8892         },
8893         {
8894                 .cmd            = LNET_CMD_PEERS,
8895                 .flags          = GENL_ADMIN_PERM,
8896 #ifdef HAVE_NETLINK_CALLBACK_START
8897                 .start          = lnet_peer_ni_show_start,
8898                 .dumpit         = lnet_peer_ni_show_dump,
8899 #else
8900                 .dumpit         = lnet_old_peer_ni_show_dump,
8901 #endif
8902                 .done           = lnet_peer_ni_show_done,
8903                 .doit           = lnet_peer_ni_cmd,
8904         },
8905         {
8906                 .cmd            = LNET_CMD_ROUTES,
8907                 .flags          = GENL_ADMIN_PERM,
8908 #ifdef HAVE_NETLINK_CALLBACK_START
8909                 .start          = lnet_route_show_start,
8910                 .dumpit         = lnet_route_show_dump,
8911 #else
8912                 .dumpit         = lnet_old_route_show_dump,
8913 #endif
8914                 .done           = lnet_route_show_done,
8915                 .doit           = lnet_route_cmd,
8916         },
8917         {
8918                 .cmd            = LNET_CMD_PING,
8919                 .flags          = GENL_ADMIN_PERM,
8920 #ifdef HAVE_NETLINK_CALLBACK_START
8921                 .start          = lnet_ping_show_start,
8922                 .dumpit         = lnet_ping_show_dump,
8923 #else
8924                 .dumpit         = lnet_old_ping_show_dump,
8925 #endif
8926                 .done           = lnet_ping_show_done,
8927                 .doit           = lnet_ping_cmd,
8928         },
8929         {
8930                 .cmd            = LNET_CMD_CPT_OF_NID,
8931 #ifdef HAVE_NETLINK_CALLBACK_START
8932                 .start          = lnet_cpt_of_nid_show_start,
8933                 .dumpit         = lnet_cpt_of_nid_show_dump,
8934 #else
8935                 .dumpit         = lnet_old_cpt_of_nid_show_dump,
8936 #endif
8937                 .done           = lnet_cpt_of_nid_show_done,
8938         },
8939         {
8940                 .cmd            = LNET_CMD_PEER_DIST,
8941 #ifdef HAVE_NETLINK_CALLBACK_START
8942                 .start          = lnet_peer_dist_show_start,
8943                 .dumpit         = lnet_peer_dist_show_dump,
8944 #else
8945                 .dumpit         = lnet_old_peer_dist_show_dump,
8946 #endif
8947                 .done           = lnet_peer_dist_show_done,
8948         },
8949 };
8950
8951 static struct genl_family lnet_family = {
8952         .name           = LNET_GENL_NAME,
8953         .version        = LNET_GENL_VERSION,
8954         .module         = THIS_MODULE,
8955         .parallel_ops   = true,
8956         .netnsok        = true,
8957         .ops            = lnet_genl_ops,
8958         .n_ops          = ARRAY_SIZE(lnet_genl_ops),
8959         .mcgrps         = lnet_mcast_grps,
8960         .n_mcgrps       = ARRAY_SIZE(lnet_mcast_grps),
8961 #ifdef GENL_FAMILY_HAS_RESV_START_OP
8962         .resv_start_op  = __LNET_CMD_MAX_PLUS_ONE,
8963 #endif
8964 };
8965
8966 void LNetDebugPeer(struct lnet_processid *id)
8967 {
8968         lnet_debug_peer(&id->nid);
8969 }
8970 EXPORT_SYMBOL(LNetDebugPeer);
8971
8972 /**
8973  * Determine if the specified peer \a nid is on the local node.
8974  *
8975  * \param nid   peer nid to check
8976  *
8977  * \retval true         If peer NID is on the local node.
8978  * \retval false        If peer NID is not on the local node.
8979  */
8980 bool LNetIsPeerLocal(struct lnet_nid *nid)
8981 {
8982         struct lnet_net *net;
8983         struct lnet_ni *ni;
8984         int cpt;
8985
8986         cpt = lnet_net_lock_current();
8987         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
8988                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
8989                         if (nid_same(&ni->ni_nid, nid)) {
8990                                 lnet_net_unlock(cpt);
8991                                 return true;
8992                         }
8993                 }
8994         }
8995         lnet_net_unlock(cpt);
8996
8997         return false;
8998 }
8999 EXPORT_SYMBOL(LNetIsPeerLocal);
9000
9001 /**
9002  * Retrieve the struct lnet_process_id ID of LNet interface at \a index.
9003  * Note that all interfaces share a same PID, as requested by LNetNIInit().
9004  *
9005  * @index       Index of the interface to look up.
9006  * @id          On successful return, this location will hold the
9007  *              struct lnet_process_id ID of the interface.
9008  * @large_nids  Report large NIDs if this is true.
9009  *
9010  * RETURN       0 If an interface exists at \a index.
9011  *              -ENOENT If no interface has been found.
9012  */
9013 int
9014 LNetGetId(unsigned int index, struct lnet_processid *id, bool large_nids)
9015 {
9016         struct lnet_ni   *ni;
9017         struct lnet_net  *net;
9018         int               cpt;
9019         int               rc = -ENOENT;
9020
9021         LASSERT(the_lnet.ln_refcount > 0);
9022
9023         cpt = lnet_net_lock_current();
9024
9025         list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
9026                 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
9027                         if (!large_nids && !nid_is_nid4(&ni->ni_nid))
9028                                 continue;
9029
9030                         if (index-- != 0)
9031                                 continue;
9032
9033                         id->nid = ni->ni_nid;
9034                         id->pid = the_lnet.ln_pid;
9035                         rc = 0;
9036                         break;
9037                 }
9038         }
9039
9040         lnet_net_unlock(cpt);
9041         return rc;
9042 }
9043 EXPORT_SYMBOL(LNetGetId);
9044
9045 struct ping_data {
9046         int rc;
9047         int replied;
9048         int pd_unlinked;
9049         struct lnet_handle_md mdh;
9050         struct completion completion;
9051 };
9052
9053 static void
9054 lnet_ping_event_handler(struct lnet_event *event)
9055 {
9056         struct ping_data *pd = event->md_user_ptr;
9057
9058         CDEBUG(D_NET, "ping event (%d %d)%s\n",
9059                event->type, event->status,
9060                event->unlinked ? " unlinked" : "");
9061
9062         if (event->status) {
9063                 if (!pd->rc)
9064                         pd->rc = event->status;
9065         } else if (event->type == LNET_EVENT_REPLY) {
9066                 pd->replied = 1;
9067                 pd->rc = event->mlength;
9068         }
9069
9070         if (event->unlinked)
9071                 pd->pd_unlinked = 1;
9072
9073         if (event->unlinked ||
9074             (event->type == LNET_EVENT_SEND && event->status))
9075                 complete(&pd->completion);
9076 }
9077
9078 static int lnet_ping(struct lnet_processid *id, struct lnet_nid *src_nid,
9079                      signed long timeout, struct lnet_genl_ping_list *plist,
9080                      int n_ids)
9081 {
9082         int id_bytes = sizeof(struct lnet_ni_status); /* For 0@lo */
9083         struct lnet_md md = { NULL };
9084         struct ping_data pd = { 0 };
9085         struct lnet_ping_buffer *pbuf;
9086         struct lnet_processid pid;
9087         struct lnet_ping_iter pi;
9088         int i = 0;
9089         u32 *st;
9090         int nob;
9091         int rc;
9092         int rc2;
9093
9094         genradix_init(&plist->lgpl_list);
9095
9096         /* n_ids limit is arbitrary */
9097         if (n_ids <= 0 || LNET_NID_IS_ANY(&id->nid))
9098                 return -EINVAL;
9099
9100         /* if the user buffer has more space than the lnet_interfaces_max
9101          * then only fill it up to lnet_interfaces_max
9102          */
9103         if (n_ids > lnet_interfaces_max)
9104                 n_ids = lnet_interfaces_max;
9105
9106         if (id->pid == LNET_PID_ANY)
9107                 id->pid = LNET_PID_LUSTRE;
9108
9109         id_bytes += n_ids * sizeof(struct lnet_nid);
9110         pbuf = lnet_ping_buffer_alloc(id_bytes, GFP_NOFS);
9111         if (!pbuf)
9112                 return -ENOMEM;
9113
9114         /* initialize md content */
9115         md.start     = &pbuf->pb_info;
9116         md.length    = id_bytes;
9117         md.threshold = 2; /* GET/REPLY */
9118         md.max_size  = 0;
9119         md.options   = LNET_MD_TRUNCATE;
9120         md.user_ptr  = &pd;
9121         md.handler   = lnet_ping_event_handler;
9122
9123         init_completion(&pd.completion);
9124
9125         rc = LNetMDBind(&md, LNET_UNLINK, &pd.mdh);
9126         if (rc != 0) {
9127                 CERROR("Can't bind MD: %d\n", rc);
9128                 goto fail_ping_buffer_decref;
9129         }
9130
9131         rc = LNetGet(src_nid, pd.mdh, id, LNET_RESERVED_PORTAL,
9132                      LNET_PROTO_PING_MATCHBITS, 0, false);
9133         if (rc != 0) {
9134                 /* Don't CERROR; this could be deliberate! */
9135                 rc2 = LNetMDUnlink(pd.mdh);
9136                 LASSERT(rc2 == 0);
9137
9138                 /* NB must wait for the UNLINK event below... */
9139         }
9140
9141         /* Ensure completion in finite time... */
9142         wait_for_completion_timeout(&pd.completion, timeout);
9143         if (!pd.pd_unlinked) {
9144                 LNetMDUnlink(pd.mdh);
9145                 wait_for_completion(&pd.completion);
9146         }
9147
9148         if (!pd.replied) {
9149                 rc = pd.rc ?: -EIO;
9150                 goto fail_ping_buffer_decref;
9151         }
9152
9153         nob = pd.rc;
9154         LASSERT(nob >= 0 && nob <= id_bytes);
9155
9156         rc = -EPROTO;           /* if I can't parse... */
9157
9158         if (nob < LNET_PING_INFO_HDR_SIZE) {
9159                 CERROR("%s: ping info too short %d\n",
9160                        libcfs_idstr(id), nob);
9161                 goto fail_ping_buffer_decref;
9162         }
9163
9164         if (pbuf->pb_info.pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) {
9165                 lnet_swap_pinginfo(pbuf);
9166         } else if (pbuf->pb_info.pi_magic != LNET_PROTO_PING_MAGIC) {
9167                 CERROR("%s: Unexpected magic %08x\n",
9168                        libcfs_idstr(id), pbuf->pb_info.pi_magic);
9169                 goto fail_ping_buffer_decref;
9170         }
9171
9172         if ((pbuf->pb_info.pi_features & LNET_PING_FEAT_NI_STATUS) == 0) {
9173                 CERROR("%s: ping w/o NI status: 0x%x\n",
9174                        libcfs_idstr(id), pbuf->pb_info.pi_features);
9175                 goto fail_ping_buffer_decref;
9176         }
9177
9178         /* Test if smaller than lnet_pinginfo with just one pi_ni status info.
9179          * That one might contain size when large nids are used.
9180          */
9181         if (nob < offsetof(struct lnet_ping_info, pi_ni[1])) {
9182                 CERROR("%s: Short reply %d(%lu min)\n",
9183                        libcfs_idstr(id), nob,
9184                        offsetof(struct lnet_ping_info, pi_ni[1]));
9185                 goto fail_ping_buffer_decref;
9186         }
9187
9188         if (ping_info_count_entries(pbuf) < n_ids) {
9189                 n_ids = ping_info_count_entries(pbuf);
9190                 id_bytes = lnet_ping_info_size(&pbuf->pb_info);
9191         }
9192
9193         if (nob < id_bytes) {
9194                 CERROR("%s: Short reply %d(%d expected)\n",
9195                        libcfs_idstr(id), nob, id_bytes);
9196                 goto fail_ping_buffer_decref;
9197         }
9198
9199         for (st = ping_iter_first(&pi, pbuf, &pid.nid);
9200              st;
9201              st = ping_iter_next(&pi, &pid.nid)) {
9202                 id = genradix_ptr_alloc(&plist->lgpl_list, i++, GFP_KERNEL);
9203                 if (!id) {
9204                         rc = -ENOMEM;
9205                         goto fail_ping_buffer_decref;
9206                 }
9207
9208                 id->pid = pbuf->pb_info.pi_pid;
9209                 id->nid = pid.nid;
9210         }
9211         rc = i;
9212 fail_ping_buffer_decref:
9213         lnet_ping_buffer_decref(pbuf);
9214         return rc;
9215 }
9216
9217 static int
9218 lnet_discover(struct lnet_processid *pid, u32 force,
9219               struct lnet_genl_ping_list *dlist)
9220 {
9221         struct lnet_peer_ni *lpni;
9222         struct lnet_peer_ni *p;
9223         struct lnet_peer *lp;
9224         int cpt;
9225         int rc;
9226
9227         if (LNET_NID_IS_ANY(&pid->nid))
9228                 return -EINVAL;
9229
9230         if (pid->pid == LNET_PID_ANY)
9231                 pid->pid = LNET_PID_LUSTRE;
9232
9233         cpt = lnet_net_lock_current();
9234         lpni = lnet_peerni_by_nid_locked(&pid->nid, NULL, cpt);
9235         if (IS_ERR(lpni)) {
9236                 rc = PTR_ERR(lpni);
9237                 goto out;
9238         }
9239
9240         /*
9241          * Clearing the NIDS_UPTODATE flag ensures the peer will
9242          * be discovered, provided discovery has not been disabled.
9243          */
9244         lp = lpni->lpni_peer_net->lpn_peer;
9245         spin_lock(&lp->lp_lock);
9246         lp->lp_state &= ~LNET_PEER_NIDS_UPTODATE;
9247         /* If the force flag is set, force a PING and PUSH as well. */
9248         if (force)
9249                 lp->lp_state |= LNET_PEER_FORCE_PING | LNET_PEER_FORCE_PUSH;
9250         spin_unlock(&lp->lp_lock);
9251         rc = lnet_discover_peer_locked(lpni, cpt, true);
9252         if (rc)
9253                 goto out_decref;
9254
9255         /* The lpni (or lp) for this NID may have changed and our ref is
9256          * the only thing keeping the old one around. Release the ref
9257          * and lookup the lpni again
9258          */
9259         lnet_peer_ni_decref_locked(lpni);
9260         lpni = lnet_peer_ni_find_locked(&pid->nid);
9261         if (!lpni) {
9262                 rc = -ENOENT;
9263                 goto out;
9264         }
9265         lp = lpni->lpni_peer_net->lpn_peer;
9266
9267         dlist->lgpl_list_count = 0;
9268         p = NULL;
9269         while ((p = lnet_get_next_peer_ni_locked(lp, NULL, p)) != NULL) {
9270                 struct lnet_processid *id;
9271
9272                 id = genradix_ptr_alloc(&dlist->lgpl_list,
9273                                         dlist->lgpl_list_count++, GFP_ATOMIC);
9274                 if (!id) {
9275                         rc = -ENOMEM;
9276                         goto out_decref;
9277                 }
9278                 id->pid = pid->pid;
9279                 id->nid = p->lpni_nid;
9280         }
9281         rc = dlist->lgpl_list_count;
9282
9283 out_decref:
9284         lnet_peer_ni_decref_locked(lpni);
9285 out:
9286         lnet_net_unlock(cpt);
9287
9288         return rc;
9289 }
9290
9291 /**
9292  * Retrieve peer discovery status.
9293  *
9294  * \retval 1 if lnet_peer_discovery_disabled is 0
9295  * \retval 0 if lnet_peer_discovery_disabled is 1
9296  */
9297 int
9298 LNetGetPeerDiscoveryStatus(void)
9299 {
9300         return !lnet_peer_discovery_disabled;
9301 }
9302 EXPORT_SYMBOL(LNetGetPeerDiscoveryStatus);