lnet/klnds/gnilnd/gnilnd_modparams.c

   1 /*
   2  * Copyright (C) 2004 Cluster File Systems, Inc.
   3  *
   4  * Copyright (C) 2009-2012 Cray, Inc.
   5  *
   6  *   Derived from work by: Eric Barton <eric@bartonsoftware.com>
   7  *   Author: Nic Henke <nic@cray.com>
   8  *
   9  *   This file is part of Lustre, http://www.lustre.org.
  10  *
  11  *   Lustre is free software; you can redistribute it and/or
  12  *   modify it under the terms of version 2 of the GNU General Public
  13  *   License as published by the Free Software Foundation.
  14  *
  15  *   Lustre is distributed in the hope that it will be useful,
  16  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  *   GNU General Public License for more details.
  19  *
  20  *   You should have received a copy of the GNU General Public License
  21  *   along with Lustre; if not, write to the Free Software
  22  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23  *
  24  */
  25
  26 #include "gnilnd.h"
  27
  28 static int credits = 256;
  29 CFS_MODULE_PARM(credits, "i", int, 0444,
  30                 "# concurrent sends");
  31
  32 static int peer_credits = 16;
  33 CFS_MODULE_PARM(peer_credits, "i", int, 0444,
  34                 "# LNet peer credits");
  35
  36 /* NB - we'll not actually limit sends to this, we just size the mailbox buffer
  37  * such that at most we'll have concurrent_sends * max_immediate messages
  38  * in the mailbox */
  39 static int concurrent_sends = 0;
  40 CFS_MODULE_PARM(concurrent_sends, "i", int, 0444,
  41                 "# concurrent HW sends to 1 peer");
  42
  43 /* default for 2k nodes @ 16 peer credits */
  44 static int fma_cq_size = 32768;
  45 CFS_MODULE_PARM(fma_cq_size, "i", int, 0444,
  46                 "size of the completion queue");
  47
  48 static int timeout = GNILND_BASE_TIMEOUT;
  49 /* can't change @ runtime because LNet gets NI data at startup from
  50  * this value */
  51 CFS_MODULE_PARM(timeout, "i", int, 0444,
  52                 "communications timeout (seconds)");
  53
  54 /* time to wait between datagram timeout and sending of next dgram */
  55 static int min_reconnect_interval = GNILND_MIN_RECONNECT_TO;
  56 CFS_MODULE_PARM(min_reconnect_interval, "i", int, 0644,
  57                 "minimum connection retry interval (seconds)");
  58
  59 /* if this goes longer than timeout, we'll timeout the TX before
  60  * the dgram */
  61 static int max_reconnect_interval = GNILND_MAX_RECONNECT_TO;
  62 CFS_MODULE_PARM(max_reconnect_interval, "i", int, 0644,
  63                 "maximum connection retry interval (seconds)");
  64
  65 static int max_immediate = (2<<10);
  66 CFS_MODULE_PARM(max_immediate, "i", int, 0644,
  67                 "immediate/RDMA breakpoint");
  68
  69 #ifdef CONFIG_CRAY_GEMINI
  70 static int checksum = GNILND_CHECKSUM_SMSG_BTE;
  71 #else
  72 static int checksum = 0;
  73 #endif
  74 CFS_MODULE_PARM(checksum, "i", int, 0644,
  75                 "0: None, 1: headers, 2: short msg, 3: all traffic");
  76
  77 static int checksum_dump = 0;
  78 CFS_MODULE_PARM(checksum_dump, "i", int, 0644,
  79                 "0: None, 1: dump log on failure, 2: payload data to D_INFO log");
  80
  81 static int bte_hash = 1;
  82 CFS_MODULE_PARM(bte_hash, "i", int, 0644,
  83                 "enable hashing for BTE (RDMA) transfers");
  84
  85 static int bte_adapt = 1;
  86 CFS_MODULE_PARM(bte_adapt, "i", int, 0644,
  87                 "enable adaptive request and response for BTE (RDMA) transfers");
  88
  89 static int bte_relaxed_ordering = 1;
  90 CFS_MODULE_PARM(bte_relaxed_ordering, "i", int, 0644,
  91                 "enable relaxed ordering (PASSPW) for BTE (RDMA) transfers");
  92
  93 static int ptag = GNI_PTAG_LND;
  94 CFS_MODULE_PARM(ptag, "i", int, 0444,
  95                 "ptag for Gemini CDM");
  96
  97 static int max_retransmits = 1024;
  98 CFS_MODULE_PARM(max_retransmits, "i", int, 0644,
  99                 "max retransmits for FMA");
 100
 101 static int nwildcard = 4;
 102 CFS_MODULE_PARM(nwildcard, "i", int, 0444,
 103                 "# wildcard datagrams to post per net (interface)");
 104
 105 static int nice = -20;
 106 CFS_MODULE_PARM(nice, "i", int, 0444,
 107                 "nice value for kgnilnd threads, default -20");
 108
 109 static int rdmaq_intervals = 4;
 110 CFS_MODULE_PARM(rdmaq_intervals, "i", int, 0644,
 111                 "# intervals per second for rdmaq throttling, default 4, 0 to disable");
 112
 113 static int loops = 100;
 114 CFS_MODULE_PARM(loops, "i", int, 0644,
 115                 "# of loops before scheduler is friendly, default 100");
 116
 117 static int hash_size = 503;
 118 CFS_MODULE_PARM(hash_size, "i", int, 0444,
 119                 "prime number for peer/conn hash sizing, default 503");
 120
 121 static int peer_health = 0;
 122 CFS_MODULE_PARM(peer_health, "i", int, 0444,
 123                 "Disable peer timeout for LNet peer health, default off, > 0 to enable");
 124
 125 static int vmap_cksum = 0;
 126 CFS_MODULE_PARM(vmap_cksum, "i", int, 0644,
 127                 "use vmap for all kiov checksumming, default off");
 128
 129 static int mbox_per_block = GNILND_FMABLK;
 130 CFS_MODULE_PARM(mbox_per_block, "i", int, 0644,
 131                 "mailboxes per block");
 132
 133 static int nphys_mbox = 0;
 134 CFS_MODULE_PARM(nphys_mbox, "i", int, 0444,
 135                 "# mbox to preallocate from physical memory, default 0");
 136
 137 static int mbox_credits = GNILND_MBOX_CREDITS;
 138 CFS_MODULE_PARM(mbox_credits, "i", int, 0644,
 139                 "number of credits per mailbox");
 140
 141 static int sched_threads = GNILND_SCHED_THREADS;
 142 CFS_MODULE_PARM(sched_threads, "i", int, 0444,
 143                 "number of threads for moving data");
 144
 145 static int net_hash_size = 11;
 146 CFS_MODULE_PARM(net_hash_size, "i", int, 0444,
 147                 "prime number for net hash sizing, default 11");
 148
 149 static int hardware_timeout = GNILND_HARDWARE_TIMEOUT;
 150 CFS_MODULE_PARM(hardware_timeout, "i", int, 0444,
 151                 "maximum time for traffic to get from one node to another");
 152
 153 static int mdd_timeout = GNILND_MDD_TIMEOUT;
 154 CFS_MODULE_PARM(mdd_timeout, "i", int, 0644,
 155                 "maximum time (in minutes) for mdd to be held");
 156
 157 kgn_tunables_t kgnilnd_tunables = {
 158         .kgn_min_reconnect_interval = &min_reconnect_interval,
 159         .kgn_max_reconnect_interval = &max_reconnect_interval,
 160         .kgn_credits                = &credits,
 161         .kgn_peer_credits           = &peer_credits,
 162         .kgn_concurrent_sends       = &concurrent_sends,
 163         .kgn_fma_cq_size            = &fma_cq_size,
 164         .kgn_timeout                = &timeout,
 165         .kgn_max_immediate          = &max_immediate,
 166         .kgn_checksum               = &checksum,
 167         .kgn_checksum_dump          = &checksum_dump,
 168         .kgn_bte_hash               = &bte_hash,
 169         .kgn_bte_adapt              = &bte_adapt,
 170         .kgn_bte_relaxed_ordering   = &bte_relaxed_ordering,
 171         .kgn_ptag                   = &ptag,
 172         .kgn_max_retransmits        = &max_retransmits,
 173         .kgn_nwildcard              = &nwildcard,
 174         .kgn_nice                   = &nice,
 175         .kgn_rdmaq_intervals        = &rdmaq_intervals,
 176         .kgn_loops                  = &loops,
 177         .kgn_peer_hash_size         = &hash_size,
 178         .kgn_peer_health            = &peer_health,
 179         .kgn_vmap_cksum             = &vmap_cksum,
 180         .kgn_mbox_per_block         = &mbox_per_block,
 181         .kgn_nphys_mbox             = &nphys_mbox,
 182         .kgn_mbox_credits           = &mbox_credits,
 183         .kgn_sched_threads          = &sched_threads,
 184         .kgn_net_hash_size          = &net_hash_size,
 185         .kgn_hardware_timeout       = &hardware_timeout,
 186         .kgn_mdd_timeout            = &mdd_timeout
 187 };
 188
 189 #if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
 190 static cfs_sysctl_table_t kgnilnd_ctl_table[] = {
 191         {
 192                 INIT_CTL_NAME(2)
 193                 .procname = "min_reconnect_interval",
 194                 .data     = &min_reconnect_interval,
 195                 .maxlen   = sizeof(int),
 196                 .mode     = 0644,
 197                 .proc_handler = &proc_dointvec
 198         },
 199         {
 200                 INIT_CTL_NAME(3)
 201                 .procname = "max_reconnect_interval",
 202                 .data     = &max_reconnect_interval,
 203                 .maxlen   = sizeof(int),
 204                 .mode     = 0644,
 205                 .proc_handler = &proc_dointvec
 206         },
 207         {
 208                 INIT_CTL_NAME(5)
 209                 .procname = "credits",
 210                 .data     = &credits,
 211                 .maxlen   = sizeof(int),
 212                 .mode     = 0444,
 213                 .proc_handler = &proc_dointvec
 214         },
 215         {
 216                 INIT_CTL_NAME(6)
 217                 .procname = "peer_credits",
 218                 .data     = &peer_credits,
 219                 .maxlen   = sizeof(int),
 220                 .mode     = 0444,
 221                 .proc_handler = &proc_dointvec
 222         },
 223         {
 224                 INIT_CTL_NAME(7)
 225                 .procname = "fma_cq_size",
 226                 .data     = &fma_cq_size,
 227                 .maxlen   = sizeof(int),
 228                 .mode     = 0444,
 229                 .proc_handler = &proc_dointvec
 230         },
 231         {
 232                 INIT_CTL_NAME(8)
 233                 .procname = "timeout",
 234                 .data     = &timeout,
 235                 .maxlen   = sizeof(int),
 236                 .mode     = 0444,
 237                 .proc_handler = &proc_dointvec
 238         },
 239         {
 240                 INIT_CTL_NAME(9)
 241                 .procname = "max_immediate",
 242                 .data     = &max_immediate,
 243                 .maxlen   = sizeof(int),
 244                 .mode     = 0444,
 245                 .proc_handler = &proc_dointvec
 246         },
 247         {
 248                 INIT_CTL_NAME(10)
 249                 .procname = "checksum",
 250                 .data     = &checksum,
 251                 .maxlen   = sizeof(int),
 252                 .mode     = 0644,
 253                 .proc_handler = &proc_dointvec
 254         },
 255         {
 256                 INIT_CTL_NAME(11)
 257                 .procname = "bte_hash",
 258                 .data     = &bte_hash,
 259                 .maxlen   = sizeof(int),
 260                 .mode     = 0644,
 261                 .proc_handler = &proc_dointvec
 262         },
 263         {
 264                 INIT_CTL_NAME(12)
 265                 .procname = "bte_adapt",
 266                 .data     = &bte_adapt,
 267                 .maxlen   = sizeof(int),
 268                 .mode     = 0644,
 269                 .proc_handler = &proc_dointvec
 270         },
 271         {
 272                 INIT_CTL_NAME(13)
 273                 .procname = "ptag",
 274                 .data     = &ptag,
 275                 .maxlen   = sizeof(int),
 276                 .mode     = 0444,
 277                 .proc_handler = &proc_dointvec
 278         },
 279         {
 280                 INIT_CTL_NAME(14)
 281                 .procname = "nwildcard",
 282                 .data     = &nwildcard,
 283                 .maxlen   = sizeof(int),
 284                 .mode     = 0444,
 285                 .proc_handler = &proc_dointvec
 286         },
 287         {
 288                 INIT_CTL_NAME(15)
 289                 .procname = "bte_relaxed_ordering",
 290                 .data     = &bte_relaxed_ordering,
 291                 .maxlen   = sizeof(int),
 292                 .mode     = 0644,
 293                 .proc_handler = &proc_dointvec
 294         },
 295         {
 296                 INIT_CTL_NAME(16)
 297                 .procname = "checksum_dump",
 298                 .data     = &checksum_dump,
 299                 .maxlen   = sizeof(int),
 300                 .mode     = 0644,
 301                 .proc_handler = &proc_dointvec
 302         },
 303         {
 304                 INIT_CTL_NAME(17)
 305                 .procname = "nice",
 306                 .data     = &nice,
 307                 .maxlen   = sizeof(int),
 308                 .mode     = 0444,
 309                 .proc_handler = &proc_dointvec
 310         },
 311         {
 312                 INIT_CTL_NAME(18)
 313                 .procname = "rdmaq_intervals",
 314                 .data     = &rdmaq_intervals,
 315                 .maxlen   = sizeof(int),
 316                 .mode     = 0644,
 317                 .proc_handler = &proc_dointvec
 318         },
 319         {
 320                 INIT_CTL_NAME(19)
 321                 .procname = "loops",
 322                 .data     = &loops,
 323                 .maxlen   = sizeof(int),
 324                 .mode     = 0644,
 325                 .proc_handler = &proc_dointvec
 326         },
 327         {
 328                 INIT_CTL_NAME(20)
 329                 .procname = "hash_size",
 330                 .data     = &hash_size,
 331                 .maxlen   = sizeof(int),
 332                 .mode     = 0444,
 333                 .proc_handler = &proc_dointvec
 334         },
 335         {
 336                 INIT_CTL_NAME(21)
 337                 .procname = "peer_health",
 338                 .data     = &peer_health,
 339                 .maxlen   = sizeof(int),
 340                 .mode     = 0444,
 341                 .proc_handler = &proc_dointvec
 342         },
 343         {
 344                 INIT_CTL_NAME(22)
 345                 .procname = "vmap_cksum",
 346                 .data     = &vmap_cksum,
 347                 .maxlen   = sizeof(int),
 348                 .mode     = 0644,
 349                 .proc_handler = &proc_dointvec
 350         },
 351         {
 352                 INIT_CTL_NAME(23)
 353                 .procname = "mbox_per_block",
 354                 .data     = &mbox_per_block,
 355                 .maxlen   = sizeof(int),
 356                 .mode     = 0644,
 357                 .proc_handler = &proc_dointvec
 358         },
 359         {
 360                 INIT_CTL_NAME(24)
 361                 .procname = "mbox_credits"
 362                 .data     = &mbox_credits,
 363                 .maxlen   = sizeof(int),
 364                 .mode     = 0644,
 365                 .proc_handler = &proc_dointvec
 366         },
 367         {
 368                 INIT_CTL_NAME(25)
 369                 .procname = "sched_threads"
 370                 .data     = &sched_threads,
 371                 .maxlen   = sizeof(int),
 372                 .mode     = 0444,
 373                 .proc_handler = &proc_dointvec
 374         },
 375         {
 376                 INIT_CTL_NAME(26)
 377                 .procname = "net_hash_size",
 378                 .data     = &net_hash_size,
 379                 .maxlen   = sizeof(int),
 380                 .mode     = 0444,
 381                 .proc_handler = &proc_dointvec
 382         },
 383         {
 384                 INIT_CTL_NAME(27)
 385                 .procname = "hardware_timeout",
 386                 .data     = &hardware_timeout,
 387                 .maxlen   = sizeof(int),
 388                 .mode     = 0444,
 389                 .proc_handler = &proc_dointvec
 390         },
 391         {
 392                 INIT_CTL_NAME(28)
 393                 .procname = "mdd_timeout",
 394                 .data     = &mdd_timeout,
 395                 .maxlen   = sizeof(int),
 396                 .mode     = 0644,
 397                 .proc_handler = &proc_dointvec
 398         },
 399         {
 400                 INIT_CTL_NAME(29)
 401                 .procname = "max_retransmits"
 402                 .data     = &max_retransmits,
 403                 .maxlen   = sizeof(int),
 404                 .mode     = 0444,
 405                 .proc_handler = &proc_dointvec
 406         },
 407         {
 408                 INIT_CTL_NAME(30)
 409                 .procname = "concurrent_sends",
 410                 .data     = &concurrent_sends,
 411                 .maxlen   = sizeof(int),
 412                 .mode     = 0444,
 413                 .proc_handler = &proc_dointvec
 414         },
 415         {
 416                 INIT_CTL_NAME(31)
 417                 .procname = "nphys_mbox",
 418                 .data     = &nphys_mbox,
 419                 .maxlen   = sizeof(int),
 420                 .mode     = 0444,
 421                 .proc_handler = &proc_dointvec
 422         },
 423         {0}
 424 };
 425
 426 static cfs_sysctl_table_t kgnilnd_top_ctl_table[] = {
 427         {
 428                 INIT_CTL_NAME(202)
 429                 .procname = "gnilnd",
 430                 .data     = NULL,
 431                 .maxlen   = 0,
 432                 .mode     = 0555,
 433                 .child    = kgnilnd_ctl_table
 434         },
 435         {       INIT_CTL_NAME(0)   }
 436 };
 437 #endif
 438
 439 int
 440 kgnilnd_tunables_init()
 441 {
 442         int rc = 0;
 443
 444 #if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
 445         kgnilnd_tunables.kgn_sysctl =
 446                 cfs_register_sysctl_table(kgnilnd_top_ctl_table, 0);
 447
 448         if (kgnilnd_tunables.kgn_sysctl == NULL)
 449                 CWARN("Can't setup /proc tunables\n");
 450 #endif
 451         switch (*kgnilnd_tunables.kgn_checksum) {
 452         default:
 453                 CERROR("Invalid checksum module parameter: %d\n",
 454                        *kgnilnd_tunables.kgn_checksum);
 455                 rc = -EINVAL;
 456                 GOTO(out, rc);
 457         case GNILND_CHECKSUM_OFF:
 458                 /* no checksumming */
 459                 break;
 460         case GNILND_CHECKSUM_SMSG_HEADER:
 461                 LCONSOLE_INFO("SMSG header only checksumming enabled\n");
 462                 break;
 463         case GNILND_CHECKSUM_SMSG:
 464                 LCONSOLE_INFO("SMSG checksumming enabled\n");
 465                 break;
 466         case GNILND_CHECKSUM_SMSG_BTE:
 467                 LCONSOLE_INFO("SMSG + BTE checksumming enabled\n");
 468                 break;
 469         }
 470
 471         if (*kgnilnd_tunables.kgn_max_immediate > GNILND_MAX_IMMEDIATE) {
 472                 LCONSOLE_ERROR("kgnilnd module parameter 'max_immediate' too large %d > %d\n",
 473                 *kgnilnd_tunables.kgn_max_immediate, GNILND_MAX_IMMEDIATE);
 474                 rc = -EINVAL;
 475                 GOTO(out, rc);
 476         }
 477
 478         if (*kgnilnd_tunables.kgn_mbox_per_block < 1) {
 479                 *kgnilnd_tunables.kgn_mbox_per_block = 1;
 480         }
 481
 482         if (*kgnilnd_tunables.kgn_concurrent_sends == 0) {
 483                 *kgnilnd_tunables.kgn_concurrent_sends = *kgnilnd_tunables.kgn_peer_credits;
 484         } else if (*kgnilnd_tunables.kgn_concurrent_sends > *kgnilnd_tunables.kgn_peer_credits) {
 485                 LCONSOLE_ERROR("kgnilnd parameter 'concurrent_sends' too large: %d > %d (peer_credits)\n",
 486                                *kgnilnd_tunables.kgn_concurrent_sends, *kgnilnd_tunables.kgn_peer_credits);
 487                 rc = -EINVAL;
 488         }
 489 out:
 490         return rc;
 491 }
 492
 493 void
 494 kgnilnd_tunables_fini()
 495 {
 496 #if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
 497         if (kgnilnd_tunables.kgn_sysctl != NULL)
 498                 cfs_unregister_sysctl_table(kgnilnd_tunables.kgn_sysctl);
 499 #endif
 500 }