lnet/klnds/gnilnd/gnilnd_sysctl.c

   1 /*
   2  * Copyright (C) 2012 Cray, Inc.
   3  *
   4  *   Author: Nic Henke <nic@cray.com>
   5  *   Author: James Shimek <jshimek@cray.com>
   6  *
   7  *   This file is part of Lustre, http://www.lustre.org.
   8  *
   9  *   Lustre is free software; you can redistribute it and/or
  10  *   modify it under the terms of version 2 of the GNU General Public
  11  *   License as published by the Free Software Foundation.
  12  *
  13  *   Lustre is distributed in the hope that it will be useful,
  14  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  *   GNU General Public License for more details.
  17  *
  18  *   You should have received a copy of the GNU General Public License
  19  *   along with Lustre; if not, write to the Free Software
  20  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21  */
  22
  23 /* this code liberated and modified from Lustre */
  24
  25 #define DEBUG_SUBSYSTEM S_LND
  26
  27 #include "gnilnd.h"
  28
  29 #define GNILND_RCA_INJ_STRLEN 16
  30 typedef struct kgn_sysctl_data {
  31         int                     ksd_pause_trigger;
  32         int                     ksd_quiesce_secs;
  33         int                     ksd_rdmaq_override;
  34         char                    ksd_rca_inject[GNILND_RCA_INJ_STRLEN];
  35 } kgn_sysctl_data_t;
  36
  37 static kgn_sysctl_data_t        kgnilnd_sysctl;
  38
  39 #if defined(CONFIG_SYSCTL)
  40
  41 static cfs_sysctl_table_header_t *kgnilnd_table_header = NULL;
  42 #ifndef HAVE_SYSCTL_UNNUMBERED
  43
  44 enum {
  45         GNILND_VERSION = 1,
  46         GNILND_THREAD_PAUSE,
  47         GNILND_HW_QUIESCE,
  48         GNILND_STACK_RESET,
  49         GNILND_RDMAQ_OVERRIDE,
  50         GNILND_RCA_INJECT,
  51 };
  52 #else
  53 #define GNILND_VERSION             CTL_UNNUMBERED
  54 #define GNILND_THREAD_PAUSE        CTL_UNNUMBERED
  55 #define GNILND_HW_QUIESCE          CTL_UNNUMBERED
  56 #define GNILND_STACK_RESET         CTL_UNNUMBERED
  57 #define GNILND_RDMAQ_OVERRIDE      CTL_UNNUMBERED
  58 #define GNILND_RCA_INJECT          CTL_UNNUMBERED
  59 #endif
  60
  61 static int LL_PROC_PROTO(proc_toggle_thread_pause)
  62 {
  63         int  old_val = kgnilnd_sysctl.ksd_pause_trigger;
  64         int  rc = 0;
  65         ENTRY;
  66
  67         rc = ll_proc_dointvec(table, write, filp, buffer, lenp, ppos);
  68         if (!write) {
  69                 /* read */
  70                 RETURN(rc);
  71         }
  72
  73         if (kgnilnd_data.kgn_init != GNILND_INIT_ALL) {
  74                 rc = -EINVAL;
  75                 RETURN(rc);
  76         }
  77
  78         if (old_val != kgnilnd_sysctl.ksd_pause_trigger) {
  79                 down(&kgnilnd_data.kgn_quiesce_sem);
  80                 CDEBUG(D_NET, "setting quiesce_trigger %d\n", old_val);
  81                 kgnilnd_data.kgn_quiesce_trigger = kgnilnd_sysctl.ksd_pause_trigger;
  82                 kgnilnd_quiesce_wait("admin sysctl");
  83                 up(&kgnilnd_data.kgn_quiesce_sem);
  84         }
  85
  86         RETURN(rc);
  87 }
  88
  89 static int LL_PROC_PROTO(proc_hw_quiesce)
  90 {
  91         int              rc = 0;
  92         kgn_device_t    *dev;
  93         ENTRY;
  94
  95         rc = ll_proc_dointvec(table, write, filp, buffer, lenp, ppos);
  96         if (!write) {
  97                 /* read */
  98                 RETURN(rc);
  99         }
 100
 101         if (kgnilnd_data.kgn_init != GNILND_INIT_ALL) {
 102                 rc = -EINVAL;
 103                 RETURN(rc);
 104         }
 105
 106
 107         /* only device 0 gets the handle, see kgnilnd_dev_init */
 108         dev = &kgnilnd_data.kgn_devices[0];
 109
 110         LASSERTF(dev != NULL, "dev 0 is NULL\n");
 111
 112         kgnilnd_quiesce_end_callback(dev->gnd_handle,
 113                                      kgnilnd_sysctl.ksd_quiesce_secs * MSEC_PER_SEC);
 114
 115         RETURN(rc);
 116 }
 117
 118 int LL_PROC_PROTO(proc_trigger_stack_reset)
 119 {
 120         int              rc = 0;
 121         int                i = 1;
 122        kgn_device_t    *dev;
 123         ENTRY;
 124
 125         if (!write) {
 126                 /* read */
 127                 rc = ll_proc_dointvec(table, write, filp, buffer, lenp, ppos);
 128                 RETURN(rc);
 129         }
 130
 131         /* only device 0 gets the handle, see kgnilnd_dev_init */
 132         dev = &kgnilnd_data.kgn_devices[0];
 133
 134         LASSERTF(dev != NULL, "dev 0 is NULL\n");
 135
 136         kgnilnd_critical_error(dev->gnd_err_handle);
 137
 138         /* Wait for the reset to complete.  This prevents any races in testing
 139          * where we'd immediately try to send traffic again */
 140        while (kgnilnd_data.kgn_needs_reset != 0) {
 141                i++;
 142                LCONSOLE((((i) & (-i)) == i) ? D_WARNING : D_NET,
 143                                "Waiting for stack reset request to clear\n");
 144                cfs_pause(cfs_time_seconds(1 * i));
 145        }
 146
 147         RETURN(rc);
 148 }
 149
 150 static int LL_PROC_PROTO(proc_toggle_rdmaq_override)
 151 {
 152         int  old_val = kgnilnd_sysctl.ksd_rdmaq_override;
 153         int  rc = 0;
 154         ENTRY;
 155
 156         rc = ll_proc_dointvec(table, write, filp, buffer, lenp, ppos);
 157         if (!write) {
 158                 /* read */
 159                 RETURN(rc);
 160         }
 161
 162         if (kgnilnd_data.kgn_init != GNILND_INIT_ALL) {
 163                 rc = -EINVAL;
 164                 RETURN(rc);
 165         }
 166
 167         if (old_val != kgnilnd_sysctl.ksd_rdmaq_override) {
 168                 long    new_mb = kgnilnd_sysctl.ksd_rdmaq_override * (long)(1024*1024);
 169                 LCONSOLE_INFO("changing RDMAQ override to %d mbytes/sec\n",
 170                               kgnilnd_sysctl.ksd_rdmaq_override);
 171                 /* override proc is mbytes, but we calc in bytes */
 172                 kgnilnd_data.kgn_rdmaq_override = new_mb;
 173                 smp_wmb();
 174         }
 175
 176         RETURN(rc);
 177 }
 178
 179 /* /proc/sys entry point for injecting up/down nid event
 180  * <up|down> <nid>
 181  */
 182 static int LL_PROC_PROTO(proc_rca_inject)
 183 {
 184         int             rc;
 185         int             nid;
 186         int             node_down;
 187         char            command[10];
 188         ENTRY;
 189
 190         rc = ll_proc_dostring(table, write, filp, buffer, lenp, ppos);
 191
 192         if (!write) {
 193                 /* read */
 194                 RETURN(rc);
 195         }
 196
 197         if (kgnilnd_data.kgn_init != GNILND_INIT_ALL) {
 198                 rc = -EINVAL;
 199                 RETURN(rc);
 200         }
 201
 202         /* convert to nid, up/down values */
 203         rc = sscanf(kgnilnd_sysctl.ksd_rca_inject, "%s %d", command, &nid);
 204         CDEBUG(D_INFO, "command %s, nid %d\n", command, nid);
 205
 206         if (rc != 2) {
 207                 CDEBUG(D_ERROR, "invalid parameter\n");
 208                 RETURN(rc);
 209         } else {
 210                 switch (command[0]) {
 211                 case 'd': /* down */
 212                         node_down = 1;
 213                         CDEBUG(D_INFO, "take node %d down\n", nid);
 214                         break;
 215                 case 'u': /* up */
 216                         node_down = 0;
 217                         CDEBUG(D_INFO, "bring node %d up\n", nid);
 218                         break;
 219                 default:
 220                         CDEBUG(D_ERROR, "invalid command %s\n", command);
 221                         RETURN(-EINVAL);
 222                 }
 223         }
 224
 225         CDEBUG(D_INFO, "proc_rca_inject: reporting node_down %d, nid %d\n",
 226                       node_down, nid);
 227         rc = kgnilnd_report_node_state(nid, node_down);
 228
 229         if (rc) {
 230                 rc = -EINVAL;
 231         }
 232
 233         RETURN(rc);
 234 }
 235
 236 static cfs_sysctl_table_t kgnilnd_table[] = {
 237         /*
 238          * NB No .strategy entries have been provided since sysctl(8) prefers
 239          * to go via /proc for portability.
 240          */
 241         {
 242                 INIT_CTL_NAME(GNILND_VERSION)
 243                 .procname = "version",
 244                 .data     = KGNILND_BUILD_REV,
 245                 .maxlen   = sizeof(KGNILND_BUILD_REV),
 246                 .mode     = 0444,
 247                 .proc_handler = &proc_dostring
 248         },
 249         {
 250                 INIT_CTL_NAME(GNILND_THREAD_PAUSE)
 251                 .procname = "thread_pause",
 252                 .data     = &kgnilnd_sysctl.ksd_pause_trigger,
 253                 .maxlen   = sizeof(int),
 254                 .mode     = 0644,
 255                 .proc_handler = &proc_toggle_thread_pause,
 256         },
 257         {
 258                 INIT_CTL_NAME(GNILND_HW_QUIESCE)
 259                 .procname = "hw_quiesce",
 260                 .data     = &kgnilnd_sysctl.ksd_quiesce_secs,
 261                 .maxlen   = sizeof(__u32),
 262                 .mode     = 0644,
 263                 .proc_handler = &proc_hw_quiesce,
 264         },
 265         {
 266                 INIT_CTL_NAME(GNILND_STACK_RESET)
 267                 .procname = "stack_reset",
 268                 .data     = NULL,
 269                 .maxlen   = sizeof(int),
 270                 .mode     = 0600,
 271                 .proc_handler = &proc_trigger_stack_reset,
 272         },
 273         {
 274                 INIT_CTL_NAME(GNILND_RDMAQ_OVERRIDE)
 275                 .procname = "rdmaq_override",
 276                 .data     = &kgnilnd_sysctl.ksd_rdmaq_override,
 277                 .maxlen   = sizeof(int),
 278                 .mode     = 0644,
 279                 .proc_handler = &proc_toggle_rdmaq_override,
 280         },
 281         {
 282                 INIT_CTL_NAME(GNILND_RCA_INJECT)
 283                 .procname = "rca_inject",
 284                 .data     = kgnilnd_sysctl.ksd_rca_inject,
 285                 .maxlen   = GNILND_RCA_INJ_STRLEN,
 286                 .mode     = 0644,
 287                 .proc_handler = &proc_rca_inject,
 288         },
 289         {       INIT_CTL_NAME(0)   }
 290 };
 291
 292 static cfs_sysctl_table_t kgnilnd_top_table[2] = {
 293         {
 294                 INIT_CTL_NAME(CTL_GNILND)
 295                 .procname = "kgnilnd",
 296                 .data     = NULL,
 297                 .maxlen   = 0,
 298                 .mode     = 0555,
 299                 .child    = kgnilnd_table
 300         },
 301         {       INIT_CTL_NAME(0)   }
 302 };
 303
 304 void kgnilnd_insert_sysctl(void)
 305 {
 306         if (kgnilnd_table_header == NULL)
 307                 kgnilnd_table_header = cfs_register_sysctl_table(kgnilnd_top_table, 0);
 308 }
 309
 310 void kgnilnd_remove_sysctl(void)
 311 {
 312         if (kgnilnd_table_header != NULL)
 313                 cfs_unregister_sysctl_table(kgnilnd_table_header);
 314
 315         kgnilnd_table_header = NULL;
 316 }
 317
 318 #else
 319 void kgnilnd_insert_sysctl(void) {}
 320 void kgnilnd_remove_sysctl(void) {}
 321 #endif