lnet/klnds/gnilnd/gnilnd_modparams.c

   1 /*
   2  * Copyright (C) 2004 Cluster File Systems, Inc.
   3  *
   4  * Copyright (C) 2009-2012 Cray, Inc.
   5  *
   6  *   Derived from work by: Eric Barton <eric@bartonsoftware.com>
   7  *   Author: Nic Henke <nic@cray.com>
   8  *
   9  *   This file is part of Lustre, http://www.lustre.org.
  10  *
  11  *   Lustre is free software; you can redistribute it and/or
  12  *   modify it under the terms of version 2 of the GNU General Public
  13  *   License as published by the Free Software Foundation.
  14  *
  15  *   Lustre is distributed in the hope that it will be useful,
  16  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  *   GNU General Public License for more details.
  19  *
  20  *   You should have received a copy of the GNU General Public License
  21  *   along with Lustre; if not, write to the Free Software
  22  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23  *
  24  */
  25
  26 #include "gnilnd.h"
  27
  28 static int credits = 256;
  29 CFS_MODULE_PARM(credits, "i", int, 0444,
  30                 "# concurrent sends");
  31
  32 static int eager_credits = 256 * 1024;
  33 CFS_MODULE_PARM(eager_credits, "i", int, 0444,
  34                 "# eager buffers");
  35
  36 static int peer_credits = 16;
  37 CFS_MODULE_PARM(peer_credits, "i", int, 0444,
  38                 "# LNet peer credits");
  39
  40 /* NB - we'll not actually limit sends to this, we just size the mailbox buffer
  41  * such that at most we'll have concurrent_sends * max_immediate messages
  42  * in the mailbox */
  43 static int concurrent_sends = 0;
  44 CFS_MODULE_PARM(concurrent_sends, "i", int, 0444,
  45                 "# concurrent HW sends to 1 peer");
  46
  47 /* default for 2k nodes @ 16 peer credits */
  48 static int fma_cq_size = 32768;
  49 CFS_MODULE_PARM(fma_cq_size, "i", int, 0444,
  50                 "size of the completion queue");
  51
  52 static int timeout = GNILND_BASE_TIMEOUT;
  53 /* can't change @ runtime because LNet gets NI data at startup from
  54  * this value */
  55 CFS_MODULE_PARM(timeout, "i", int, 0444,
  56                 "communications timeout (seconds)");
  57
  58 /* time to wait between datagram timeout and sending of next dgram */
  59 static int min_reconnect_interval = GNILND_MIN_RECONNECT_TO;
  60 CFS_MODULE_PARM(min_reconnect_interval, "i", int, 0644,
  61                 "minimum connection retry interval (seconds)");
  62
  63 /* if this goes longer than timeout, we'll timeout the TX before
  64  * the dgram */
  65 static int max_reconnect_interval = GNILND_MAX_RECONNECT_TO;
  66 CFS_MODULE_PARM(max_reconnect_interval, "i", int, 0644,
  67                 "maximum connection retry interval (seconds)");
  68
  69 static int max_immediate = (2<<10);
  70 CFS_MODULE_PARM(max_immediate, "i", int, 0644,
  71                 "immediate/RDMA breakpoint");
  72
  73 static int checksum = GNILND_CHECKSUM_DEFAULT;
  74 CFS_MODULE_PARM(checksum, "i", int, 0644,
  75                 "0: None, 1: headers, 2: short msg, 3: all traffic");
  76
  77 static int checksum_dump = 0;
  78 CFS_MODULE_PARM(checksum_dump, "i", int, 0644,
  79                 "0: None, 1: dump log on failure, 2: payload data to D_INFO log");
  80
  81 static int bte_dlvr_mode = GNILND_RDMA_DLVR_OPTION;
  82 CFS_MODULE_PARM(bte_dlvr_mode, "i", int, 0644,
  83                 "enable hashing for BTE (RDMA) transfers");
  84
  85 static int bte_relaxed_ordering = 1;
  86 CFS_MODULE_PARM(bte_relaxed_ordering, "i", int, 0644,
  87                 "enable relaxed ordering (PASSPW) for BTE (RDMA) transfers");
  88
  89 #ifdef CONFIG_MK1OM
  90 static int ptag = GNI_PTAG_LND_KNC;
  91 #else
  92 static int ptag = GNI_PTAG_LND;
  93 #endif
  94 CFS_MODULE_PARM(ptag, "i", int, 0444,
  95                 "ptag for Gemini CDM");
  96
  97 static int max_retransmits = 1024;
  98 CFS_MODULE_PARM(max_retransmits, "i", int, 0444,
  99                 "max retransmits for FMA");
 100
 101 static int nwildcard = 4;
 102 CFS_MODULE_PARM(nwildcard, "i", int, 0444,
 103                 "# wildcard datagrams to post per net (interface)");
 104
 105 static int nice = -20;
 106 CFS_MODULE_PARM(nice, "i", int, 0444,
 107                 "nice value for kgnilnd threads, default -20");
 108
 109 static int rdmaq_intervals = 4;
 110 CFS_MODULE_PARM(rdmaq_intervals, "i", int, 0644,
 111                 "# intervals per second for rdmaq throttling, default 4, 0 to disable");
 112
 113 static int loops = 100;
 114 CFS_MODULE_PARM(loops, "i", int, 0644,
 115                 "# of loops before scheduler is friendly, default 100");
 116
 117 static int hash_size = 503;
 118 CFS_MODULE_PARM(hash_size, "i", int, 0444,
 119                 "prime number for peer/conn hash sizing, default 503");
 120
 121 static int peer_health = 0;
 122 CFS_MODULE_PARM(peer_health, "i", int, 0444,
 123                 "Disable peer timeout for LNet peer health, default off, > 0 to enable");
 124
 125 static int peer_timeout = -1;
 126 CFS_MODULE_PARM(peer_timeout, "i", int, 0444,
 127                 "Peer timeout used for peer_health, default based on gnilnd timeout, > -1 to manually set");
 128
 129 static int vmap_cksum = 0;
 130 CFS_MODULE_PARM(vmap_cksum, "i", int, 0644,
 131                 "use vmap for all kiov checksumming, default off");
 132
 133 static int mbox_per_block = GNILND_FMABLK;
 134 CFS_MODULE_PARM(mbox_per_block, "i", int, 0644,
 135                 "mailboxes per block");
 136
 137 static int nphys_mbox = 0;
 138 CFS_MODULE_PARM(nphys_mbox, "i", int, 0444,
 139                 "# mbox to preallocate from physical memory, default 0");
 140
 141 static int mbox_credits = GNILND_MBOX_CREDITS;
 142 CFS_MODULE_PARM(mbox_credits, "i", int, 0644,
 143                 "number of credits per mailbox");
 144
 145 static int sched_threads = GNILND_SCHED_THREADS;
 146 CFS_MODULE_PARM(sched_threads, "i", int, 0444,
 147                 "number of threads for moving data");
 148
 149 static int net_hash_size = 11;
 150 CFS_MODULE_PARM(net_hash_size, "i", int, 0444,
 151                 "prime number for net hash sizing, default 11");
 152
 153 static int hardware_timeout = GNILND_HARDWARE_TIMEOUT;
 154 CFS_MODULE_PARM(hardware_timeout, "i", int, 0444,
 155                 "maximum time for traffic to get from one node to another");
 156
 157 static int mdd_timeout = GNILND_MDD_TIMEOUT;
 158 CFS_MODULE_PARM(mdd_timeout, "i", int, 0644,
 159                 "maximum time (in minutes) for mdd to be held");
 160
 161 static int sched_timeout = GNILND_SCHED_TIMEOUT;
 162 CFS_MODULE_PARM(sched_timeout, "i", int, 0644,
 163                 "scheduler aliveness in seconds max time");
 164
 165 static int sched_nice = GNILND_SCHED_NICE;
 166 CFS_MODULE_PARM(sched_nice, "i", int, 0444,
 167                 "scheduler's nice setting, default compute 0 service -20");
 168
 169 static int reverse_rdma = GNILND_REVERSE_RDMA;
 170 CFS_MODULE_PARM(reverse_rdma, "i", int, 0644,
 171                 "Normal 0: Reverse GET: 1 Reverse Put: 2 Reverse Both: 3");
 172
 173 static int dgram_timeout = GNILND_DGRAM_TIMEOUT;
 174 CFS_MODULE_PARM(dgram_timeout, "i", int, 0644,
 175                 "dgram thread aliveness seconds max time");
 176
 177 static int efault_lbug = 0;
 178 CFS_MODULE_PARM(efault_lbug, "i", int, 0644,
 179                 "If a compute receives an EFAULT in"
 180                 " a message should it LBUG. 0 off 1 on");
 181
 182 kgn_tunables_t kgnilnd_tunables = {
 183         .kgn_min_reconnect_interval = &min_reconnect_interval,
 184         .kgn_max_reconnect_interval = &max_reconnect_interval,
 185         .kgn_credits                = &credits,
 186         .kgn_peer_credits           = &peer_credits,
 187         .kgn_concurrent_sends       = &concurrent_sends,
 188         .kgn_fma_cq_size            = &fma_cq_size,
 189         .kgn_timeout                = &timeout,
 190         .kgn_max_immediate          = &max_immediate,
 191         .kgn_checksum               = &checksum,
 192         .kgn_checksum_dump          = &checksum_dump,
 193         .kgn_bte_dlvr_mode          = &bte_dlvr_mode,
 194         .kgn_bte_relaxed_ordering   = &bte_relaxed_ordering,
 195         .kgn_ptag                   = &ptag,
 196         .kgn_max_retransmits        = &max_retransmits,
 197         .kgn_nwildcard              = &nwildcard,
 198         .kgn_nice                   = &nice,
 199         .kgn_rdmaq_intervals        = &rdmaq_intervals,
 200         .kgn_loops                  = &loops,
 201         .kgn_peer_hash_size         = &hash_size,
 202         .kgn_peer_health            = &peer_health,
 203         .kgn_peer_timeout           = &peer_timeout,
 204         .kgn_vmap_cksum             = &vmap_cksum,
 205         .kgn_mbox_per_block         = &mbox_per_block,
 206         .kgn_nphys_mbox             = &nphys_mbox,
 207         .kgn_mbox_credits           = &mbox_credits,
 208         .kgn_sched_threads          = &sched_threads,
 209         .kgn_net_hash_size          = &net_hash_size,
 210         .kgn_hardware_timeout       = &hardware_timeout,
 211         .kgn_mdd_timeout            = &mdd_timeout,
 212         .kgn_sched_timeout          = &sched_timeout,
 213         .kgn_sched_nice             = &sched_nice,
 214         .kgn_reverse_rdma           = &reverse_rdma,
 215         .kgn_dgram_timeout          = &dgram_timeout,
 216         .kgn_eager_credits          = &eager_credits,
 217         .kgn_efault_lbug            = &efault_lbug
 218 };
 219
 220 #if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
 221 static struct ctl_table kgnilnd_ctl_table[] = {
 222         {
 223                 INIT_CTL_NAME
 224                 .procname = "min_reconnect_interval",
 225                 .data     = &min_reconnect_interval,
 226                 .maxlen   = sizeof(int),
 227                 .mode     = 0644,
 228                 .proc_handler = &proc_dointvec
 229         },
 230         {
 231                 INIT_CTL_NAME
 232                 .procname = "max_reconnect_interval",
 233                 .data     = &max_reconnect_interval,
 234                 .maxlen   = sizeof(int),
 235                 .mode     = 0644,
 236                 .proc_handler = &proc_dointvec
 237         },
 238         {
 239                 INIT_CTL_NAME
 240                 .procname = "credits",
 241                 .data     = &credits,
 242                 .maxlen   = sizeof(int),
 243                 .mode     = 0444,
 244                 .proc_handler = &proc_dointvec
 245         },
 246         {
 247                 INIT_CTL_NAME
 248                 .procname = "peer_credits",
 249                 .data     = &peer_credits,
 250                 .maxlen   = sizeof(int),
 251                 .mode     = 0444,
 252                 .proc_handler = &proc_dointvec
 253         },
 254         {
 255                 INIT_CTL_NAME
 256                 .procname = "fma_cq_size",
 257                 .data     = &fma_cq_size,
 258                 .maxlen   = sizeof(int),
 259                 .mode     = 0444,
 260                 .proc_handler = &proc_dointvec
 261         },
 262         {
 263                 INIT_CTL_NAME
 264                 .procname = "timeout",
 265                 .data     = &timeout,
 266                 .maxlen   = sizeof(int),
 267                 .mode     = 0444,
 268                 .proc_handler = &proc_dointvec
 269         },
 270         {
 271                 INIT_CTL_NAME
 272                 .procname = "max_immediate",
 273                 .data     = &max_immediate,
 274                 .maxlen   = sizeof(int),
 275                 .mode     = 0444,
 276                 .proc_handler = &proc_dointvec
 277         },
 278         {
 279                 INIT_CTL_NAME
 280                 .procname = "checksum",
 281                 .data     = &checksum,
 282                 .maxlen   = sizeof(int),
 283                 .mode     = 0644,
 284                 .proc_handler = &proc_dointvec
 285         },
 286         {
 287                 INIT_CTL_NAME
 288                 .procname = "bte_dlvr_mode",
 289                 .data     = &bte_dlvr_mode,
 290                 .maxlen   = sizeof(int),
 291                 .mode     = 0644,
 292                 .proc_handler = &proc_dointvec
 293         },
 294         {
 295                 INIT_CTL_NAME
 296                 .procname = "ptag",
 297                 .data     = &ptag,
 298                 .maxlen   = sizeof(int),
 299                 .mode     = 0444,
 300                 .proc_handler = &proc_dointvec
 301         },
 302         {
 303                 INIT_CTL_NAME
 304                 .procname = "nwildcard",
 305                 .data     = &nwildcard,
 306                 .maxlen   = sizeof(int),
 307                 .mode     = 0444,
 308                 .proc_handler = &proc_dointvec
 309         },
 310         {
 311                 INIT_CTL_NAME
 312                 .procname = "bte_relaxed_ordering",
 313                 .data     = &bte_relaxed_ordering,
 314                 .maxlen   = sizeof(int),
 315                 .mode     = 0644,
 316                 .proc_handler = &proc_dointvec
 317         },
 318         {
 319                 INIT_CTL_NAME
 320                 .procname = "checksum_dump",
 321                 .data     = &checksum_dump,
 322                 .maxlen   = sizeof(int),
 323                 .mode     = 0644,
 324                 .proc_handler = &proc_dointvec
 325         },
 326         {
 327                 INIT_CTL_NAME
 328                 .procname = "nice",
 329                 .data     = &nice,
 330                 .maxlen   = sizeof(int),
 331                 .mode     = 0444,
 332                 .proc_handler = &proc_dointvec
 333         },
 334         {
 335                 INIT_CTL_NAME
 336                 .procname = "rdmaq_intervals",
 337                 .data     = &rdmaq_intervals,
 338                 .maxlen   = sizeof(int),
 339                 .mode     = 0644,
 340                 .proc_handler = &proc_dointvec
 341         },
 342         {
 343                 INIT_CTL_NAME
 344                 .procname = "loops",
 345                 .data     = &loops,
 346                 .maxlen   = sizeof(int),
 347                 .mode     = 0644,
 348                 .proc_handler = &proc_dointvec
 349         },
 350         {
 351                 INIT_CTL_NAME
 352                 .procname = "hash_size",
 353                 .data     = &hash_size,
 354                 .maxlen   = sizeof(int),
 355                 .mode     = 0444,
 356                 .proc_handler = &proc_dointvec
 357         },
 358         {
 359                 INIT_CTL_NAME
 360                 .procname = "peer_health",
 361                 .data     = &peer_health,
 362                 .maxlen   = sizeof(int),
 363                 .mode     = 0444,
 364                 .proc_handler = &proc_dointvec
 365         },
 366         {
 367                 INIT_CTL_NAME
 368                 .procname = "vmap_cksum",
 369                 .data     = &vmap_cksum,
 370                 .maxlen   = sizeof(int),
 371                 .mode     = 0644,
 372                 .proc_handler = &proc_dointvec
 373         },
 374         {
 375                 INIT_CTL_NAME
 376                 .procname = "mbox_per_block",
 377                 .data     = &mbox_per_block,
 378                 .maxlen   = sizeof(int),
 379                 .mode     = 0644,
 380                 .proc_handler = &proc_dointvec
 381         },
 382         {
 383                 INIT_CTL_NAME
 384                 .procname = "mbox_credits"
 385                 .data     = &mbox_credits,
 386                 .maxlen   = sizeof(int),
 387                 .mode     = 0644,
 388                 .proc_handler = &proc_dointvec
 389         },
 390         {
 391                 INIT_CTL_NAME
 392                 .procname = "sched_threads"
 393                 .data     = &sched_threads,
 394                 .maxlen   = sizeof(int),
 395                 .mode     = 0444,
 396                 .proc_handler = &proc_dointvec
 397         },
 398         {
 399                 INIT_CTL_NAME
 400                 .procname = "net_hash_size",
 401                 .data     = &net_hash_size,
 402                 .maxlen   = sizeof(int),
 403                 .mode     = 0444,
 404                 .proc_handler = &proc_dointvec
 405         },
 406         {
 407                 INIT_CTL_NAME
 408                 .procname = "hardware_timeout",
 409                 .data     = &hardware_timeout,
 410                 .maxlen   = sizeof(int),
 411                 .mode     = 0444,
 412                 .proc_handler = &proc_dointvec
 413         },
 414         {
 415                 INIT_CTL_NAME
 416                 .procname = "mdd_timeout",
 417                 .data     = &mdd_timeout,
 418                 .maxlen   = sizeof(int),
 419                 .mode     = 0644,
 420                 .proc_handler = &proc_dointvec
 421         },
 422         {
 423                 INIT_CTL_NAME
 424                 .procname = "max_retransmits"
 425                 .data     = &max_retransmits,
 426                 .maxlen   = sizeof(int),
 427                 .mode     = 0444,
 428                 .proc_handler = &proc_dointvec
 429         },
 430         {
 431                 INIT_CTL_NAME
 432                 .procname = "concurrent_sends",
 433                 .data     = &concurrent_sends,
 434                 .maxlen   = sizeof(int),
 435                 .mode     = 0444,
 436                 .proc_handler = &proc_dointvec
 437         },
 438         {
 439                 INIT_CTL_NAME
 440                 .procname = "nphys_mbox",
 441                 .data     = &nphys_mbox,
 442                 .maxlen   = sizeof(int),
 443                 .mode     = 0444,
 444                 .proc_handler = &proc_dointvec
 445         },
 446         {
 447                 INIT_CTL_NAME
 448                 .procname = "sched_timeout",
 449                 .data     = &sched_timeout,
 450                 .maxlen   = sizeof(int),
 451                 .mode     = 0644,
 452                 .proc_handler = &proc_dointvec
 453         },
 454         {
 455                 INIT_CTL_NAME
 456                 .procname = "sched_nice",
 457                 .data     = &sched_nice,
 458                 .maxlen   = sizeof(int),
 459                 .mode     = 0444,
 460                 .proc_handler = &proc_dointvec
 461         },
 462         {
 463                 INIT_CTL_NAME
 464                 .procname = "reverse_rdma",
 465                 .data     = &reverse_rdma,
 466                 .maxlen   = sizeof(int),
 467                 .mode     = 0644,
 468                 .proc_handler = &proc_dointvec
 469         },
 470                 INIT_CTL_NAME
 471                 .procname = "dgram_timeout"
 472                 .data     = &dgram_timeout,
 473                 .maxlen   = sizeof(int),
 474                 .mode     = 0644,
 475                 .proc_handler = &proc_dointvec
 476         },
 477         {
 478                 INIT_CTL_NAME
 479                 .procname = "peer_timeout"
 480                 .data     = &peer_timeout,
 481                 .maxlen   = sizeof(int),
 482                 .mode     = 0444,
 483                 .proc_handler = &proc_dointvec
 484         },
 485         {
 486                 INIT_CTL_NAME
 487                 .procname = "eager_credits",
 488                 .data     = &eager_credits,
 489                 .maxlen   = sizeof(int),
 490                 .mode     = 0644,
 491                 .proc_handler = &proc_dointvec
 492         },
 493         {
 494                 INIT_CTL_NAME
 495                 .procname = "efault_lbug"
 496                 .data     = &efault_lbug,
 497                 .maxlen   = sizeof(int),
 498                 .mode     = 0644,
 499                 .proc_handler = &proc_dointvec
 500         },
 501         { 0 }
 502 };
 503
 504 static struct ctl_table kgnilnd_top_ctl_table[] = {
 505         {
 506                 INIT_CTL_NAME
 507                 .procname = "gnilnd",
 508                 .data     = NULL,
 509                 .maxlen   = 0,
 510                 .mode     = 0555,
 511                 .child    = kgnilnd_ctl_table
 512         },
 513         { 0 }
 514 };
 515 #endif
 516
 517 int
 518 kgnilnd_tunables_init()
 519 {
 520         int rc = 0;
 521
 522 #if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
 523         kgnilnd_tunables.kgn_sysctl =
 524                 cfs_register_sysctl_table(kgnilnd_top_ctl_table, 0);
 525
 526         if (kgnilnd_tunables.kgn_sysctl == NULL)
 527                 CWARN("Can't setup /proc tunables\n");
 528 #endif
 529         switch (*kgnilnd_tunables.kgn_checksum) {
 530         default:
 531                 CERROR("Invalid checksum module parameter: %d\n",
 532                        *kgnilnd_tunables.kgn_checksum);
 533                 rc = -EINVAL;
 534                 GOTO(out, rc);
 535         case GNILND_CHECKSUM_OFF:
 536                 /* no checksumming */
 537                 break;
 538         case GNILND_CHECKSUM_SMSG_HEADER:
 539                 LCONSOLE_INFO("SMSG header only checksumming enabled\n");
 540                 break;
 541         case GNILND_CHECKSUM_SMSG:
 542                 LCONSOLE_INFO("SMSG checksumming enabled\n");
 543                 break;
 544         case GNILND_CHECKSUM_SMSG_BTE:
 545                 LCONSOLE_INFO("SMSG + BTE checksumming enabled\n");
 546                 break;
 547         }
 548
 549         if (*kgnilnd_tunables.kgn_max_immediate > GNILND_MAX_IMMEDIATE) {
 550                 LCONSOLE_ERROR("kgnilnd module parameter 'max_immediate' too large %d > %d\n",
 551                 *kgnilnd_tunables.kgn_max_immediate, GNILND_MAX_IMMEDIATE);
 552                 rc = -EINVAL;
 553                 GOTO(out, rc);
 554         }
 555
 556         if (*kgnilnd_tunables.kgn_mbox_per_block < 1) {
 557                 *kgnilnd_tunables.kgn_mbox_per_block = 1;
 558         }
 559
 560         if (*kgnilnd_tunables.kgn_concurrent_sends == 0) {
 561                 *kgnilnd_tunables.kgn_concurrent_sends = *kgnilnd_tunables.kgn_peer_credits;
 562         } else if (*kgnilnd_tunables.kgn_concurrent_sends > *kgnilnd_tunables.kgn_peer_credits) {
 563                 LCONSOLE_ERROR("kgnilnd parameter 'concurrent_sends' too large: %d > %d (peer_credits)\n",
 564                                *kgnilnd_tunables.kgn_concurrent_sends, *kgnilnd_tunables.kgn_peer_credits);
 565                 rc = -EINVAL;
 566         }
 567 out:
 568         return rc;
 569 }
 570
 571 void
 572 kgnilnd_tunables_fini()
 573 {
 574 #if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
 575         if (kgnilnd_tunables.kgn_sysctl != NULL)
 576                 cfs_unregister_sysctl_table(kgnilnd_tunables.kgn_sysctl);
 577 #endif
 578 }