lustre/obdclass/jobid.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2014, Intel Corporation.
  27  *
  28  * Copyright 2017 Cray Inc, all rights reserved.
  29  * Author: Ben Evans.
  30  *
  31  * Store PID->JobID mappings
  32  */
  33
  34 #define DEBUG_SUBSYSTEM S_RPC
  35 #include <linux/user_namespace.h>
  36 #ifdef HAVE_UIDGID_HEADER
  37 #include <linux/uidgid.h>
  38 #endif
  39 #include <linux/utsname.h>
  40
  41 #include <libcfs/libcfs.h>
  42 #include <obd_support.h>
  43 #include <obd_class.h>
  44 #include <lustre_net.h>
  45
  46 static struct cfs_hash *jobid_hash;
  47 static struct cfs_hash_ops jobid_hash_ops;
  48 spinlock_t jobid_hash_lock;
  49
  50 #define RESCAN_INTERVAL 30
  51 #define DELETE_INTERVAL 300
  52
  53 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
  54 char obd_jobid_name[LUSTRE_JOBID_SIZE] = "%e.%u";
  55
  56 /**
  57  * Structure to store a single PID->JobID mapping
  58  */
  59 struct jobid_pid_map {
  60         struct hlist_node       jp_hash;
  61         time64_t                jp_time;
  62         spinlock_t              jp_lock; /* protects jp_jobid */
  63         char                    jp_jobid[LUSTRE_JOBID_SIZE];
  64         unsigned int            jp_joblen;
  65         atomic_t                jp_refcount;
  66         pid_t                   jp_pid;
  67 };
  68
  69 /*
  70  * Get jobid of current process by reading the environment variable
  71  * stored in between the "env_start" & "env_end" of task struct.
  72  *
  73  * If some job scheduler doesn't store jobid in the "env_start/end",
  74  * then an upcall could be issued here to get the jobid by utilizing
  75  * the userspace tools/API. Then, the jobid must be cached.
  76  */
  77 int jobid_get_from_environ(char *jobid_var, char *jobid, int *jobid_len)
  78 {
  79         static bool printed;
  80         int rc;
  81
  82         rc = cfs_get_environ(jobid_var, jobid, jobid_len);
  83         if (!rc)
  84                 goto out;
  85
  86         if (unlikely(rc == -EOVERFLOW && !printed)) {
  87                 /* For the PBS_JOBID and LOADL_STEP_ID keys (which are
  88                  * variable length strings instead of just numbers), it
  89                  * might make sense to keep the unique parts for JobID,
  90                  * instead of just returning an error.  That means a
  91                  * larger temp buffer for cfs_get_environ(), then
  92                  * truncating the string at some separator to fit into
  93                  * the specified jobid_len.  Fix later if needed. */
  94                 LCONSOLE_WARN("jobid: '%s' value too large (%d)\n",
  95                               obd_jobid_var, *jobid_len);
  96                 printed = true;
  97                 rc = 0;
  98         }
  99         if (rc) {
 100                 CDEBUG((rc == -ENOENT || rc == -EINVAL ||
 101                         rc == -EDEADLK) ? D_INFO : D_ERROR,
 102                        "jobid: get '%s' failed: rc = %d\n",
 103                        obd_jobid_var, rc);
 104         }
 105
 106 out:
 107         return rc;
 108 }
 109
 110 /*
 111  * jobid_should_free_item
 112  *
 113  * Each item is checked to see if it should be released
 114  * Removed from hash table by caller
 115  * Actually freed in jobid_put_locked
 116  *
 117  * Returns 1 if item is to be freed, 0 if it is to be kept
 118  */
 119
 120 static int jobid_should_free_item(void *obj, void *data)
 121 {
 122         char *jobid = data;
 123         struct jobid_pid_map *pidmap = obj;
 124         int rc = 0;
 125
 126         if (obj == NULL)
 127                 return 0;
 128
 129         spin_lock(&pidmap->jp_lock);
 130         if (jobid == NULL)
 131                 rc = 1;
 132         else if (jobid[0] == '\0')
 133                 rc = 1;
 134         else if (ktime_get_real_seconds() - pidmap->jp_time > DELETE_INTERVAL)
 135                 rc = 1;
 136         else if (strcmp(pidmap->jp_jobid, jobid) == 0)
 137                 rc = 1;
 138         spin_unlock(&pidmap->jp_lock);
 139
 140         return rc;
 141 }
 142
 143 /*
 144  * jobid_name_is_valid
 145  *
 146  * Checks if the jobid is a Lustre process
 147  *
 148  * Returns true if jobid is valid
 149  * Returns false if jobid looks like it's a Lustre process
 150  */
 151 static bool jobid_name_is_valid(char *jobid)
 152 {
 153         const char *const lustre_reserved[] = { "ll_ping", "ptlrpc",
 154                                                 "ldlm", "ll_sa", NULL };
 155         int i;
 156
 157         if (jobid[0] == '\0')
 158                 return false;
 159
 160         for (i = 0; lustre_reserved[i] != NULL; i++) {
 161                 if (strncmp(jobid, lustre_reserved[i],
 162                             strlen(lustre_reserved[i])) == 0)
 163                         return false;
 164         }
 165         return true;
 166 }
 167
 168 /*
 169  * jobid_get_from_cache()
 170  *
 171  * Returns contents of jobid_var from process environment for current PID.
 172  * This will be cached for some time to avoid overhead scanning environment.
 173  *
 174  * Return: -ENOMEM if allocating a new pidmap fails
 175  *         -ENOENT if no entry could be found
 176  *         +ve string length for success (something was returned in jobid)
 177  */
 178 static int jobid_get_from_cache(char *jobid, size_t joblen)
 179 {
 180         static time64_t last_expire;
 181         bool expire_cache = false;
 182         pid_t pid = current_pid();
 183         struct jobid_pid_map *pidmap = NULL;
 184         time64_t now = ktime_get_real_seconds();
 185         int rc = 0;
 186         ENTRY;
 187
 188         LASSERT(jobid_hash != NULL);
 189
 190         /* scan hash periodically to remove old PID entries from cache */
 191         spin_lock(&jobid_hash_lock);
 192         if (unlikely(last_expire + DELETE_INTERVAL <= now)) {
 193                 expire_cache = true;
 194                 last_expire = now;
 195         }
 196         spin_unlock(&jobid_hash_lock);
 197
 198         if (expire_cache)
 199                 cfs_hash_cond_del(jobid_hash, jobid_should_free_item,
 200                                   "intentionally_bad_jobid");
 201
 202         /* first try to find PID in the hash and use that value */
 203         pidmap = cfs_hash_lookup(jobid_hash, &pid);
 204         if (pidmap == NULL) {
 205                 struct jobid_pid_map *pidmap2;
 206
 207                 OBD_ALLOC_PTR(pidmap);
 208                 if (pidmap == NULL)
 209                         GOTO(out, rc = -ENOMEM);
 210
 211                 pidmap->jp_pid = pid;
 212                 pidmap->jp_time = 0;
 213                 pidmap->jp_jobid[0] = '\0';
 214                 spin_lock_init(&pidmap->jp_lock);
 215                 INIT_HLIST_NODE(&pidmap->jp_hash);
 216                 /*
 217                  * @pidmap might be reclaimed just after it is added into
 218                  * hash list, init @jp_refcount as 1 to make sure memory
 219                  * could be not freed during access.
 220                  */
 221                 atomic_set(&pidmap->jp_refcount, 1);
 222
 223                 /*
 224                  * Add the newly created map to the hash, on key collision we
 225                  * lost a racing addition and must destroy our newly allocated
 226                  * map.  The object which exists in the hash will be returned.
 227                  */
 228                 pidmap2 = cfs_hash_findadd_unique(jobid_hash, &pid,
 229                                                   &pidmap->jp_hash);
 230                 if (unlikely(pidmap != pidmap2)) {
 231                         CDEBUG(D_INFO, "jobid: duplicate found for PID=%u\n",
 232                                pid);
 233                         OBD_FREE_PTR(pidmap);
 234                         pidmap = pidmap2;
 235                 }
 236         }
 237
 238         /*
 239          * If pidmap is old (this is always true for new entries) refresh it.
 240          * If obd_jobid_var is not found, cache empty entry and try again
 241          * later, to avoid repeat lookups for PID if obd_jobid_var missing.
 242          */
 243         spin_lock(&pidmap->jp_lock);
 244         if (pidmap->jp_time + RESCAN_INTERVAL <= now) {
 245                 char env_jobid[LUSTRE_JOBID_SIZE] = "";
 246                 int env_len = sizeof(env_jobid);
 247
 248                 pidmap->jp_time = now;
 249
 250                 spin_unlock(&pidmap->jp_lock);
 251                 rc = jobid_get_from_environ(obd_jobid_var, env_jobid, &env_len);
 252
 253                 CDEBUG(D_INFO, "jobid: PID mapping established: %d->%s\n",
 254                        pidmap->jp_pid, env_jobid);
 255                 spin_lock(&pidmap->jp_lock);
 256                 if (!rc) {
 257                         pidmap->jp_joblen = env_len;
 258                         strlcpy(pidmap->jp_jobid, env_jobid,
 259                                 sizeof(pidmap->jp_jobid));
 260                         rc = 0;
 261                 } else if (rc == -ENOENT) {
 262                         /* It might have been deleted, clear out old entry */
 263                         pidmap->jp_joblen = 0;
 264                         pidmap->jp_jobid[0] = '\0';
 265                 }
 266         }
 267
 268         /*
 269          * Regardless of how pidmap was found, if it contains a valid entry
 270          * use that for now.  If there was a technical error (e.g. -ENOMEM)
 271          * use the old cached value until it can be looked up again properly.
 272          * If a cached missing entry was found, return -ENOENT.
 273          */
 274         if (pidmap->jp_joblen) {
 275                 strlcpy(jobid, pidmap->jp_jobid, joblen);
 276                 joblen = pidmap->jp_joblen;
 277                 rc = 0;
 278         } else if (!rc) {
 279                 rc = -ENOENT;
 280         }
 281         spin_unlock(&pidmap->jp_lock);
 282
 283         cfs_hash_put(jobid_hash, &pidmap->jp_hash);
 284
 285         EXIT;
 286 out:
 287         return rc < 0 ? rc : joblen;
 288 }
 289
 290 /*
 291  * jobid_interpret_string()
 292  *
 293  * Interpret the jobfmt string to expand specified fields, like coredumps do:
 294  *   %e = executable
 295  *   %g = gid
 296  *   %h = hostname
 297  *   %j = jobid from environment
 298  *   %p = pid
 299  *   %u = uid
 300  *
 301  * Unknown escape strings are dropped.  Other characters are copied through,
 302  * excluding whitespace (to avoid making jobid parsing difficult).
 303  *
 304  * Return: -EOVERFLOW if the expanded string does not fit within @joblen
 305  *         0 for success
 306  */
 307 static int jobid_interpret_string(const char *jobfmt, char *jobid,
 308                                   ssize_t joblen)
 309 {
 310         char c;
 311
 312         while ((c = *jobfmt++) && joblen > 1) {
 313                 char f;
 314                 int l;
 315
 316                 if (isspace(c)) /* Don't allow embedded spaces */
 317                         continue;
 318
 319                 if (c != '%') {
 320                         *jobid = c;
 321                         joblen--;
 322                         jobid++;
 323                         continue;
 324                 }
 325
 326                 switch ((f = *jobfmt++)) {
 327                 case 'e': /* executable name */
 328                         l = snprintf(jobid, joblen, "%s", current_comm());
 329                         break;
 330                 case 'g': /* group ID */
 331                         l = snprintf(jobid, joblen, "%u",
 332                                      from_kgid(&init_user_ns, current_fsgid()));
 333                         break;
 334                 case 'h': /* hostname */
 335                         l = snprintf(jobid, joblen, "%s",
 336                                      init_utsname()->nodename);
 337                         break;
 338                 case 'j': /* jobid stored in process environment */
 339                         l = jobid_get_from_cache(jobid, joblen);
 340                         if (l < 0)
 341                                 l = 0;
 342                         break;
 343                 case 'p': /* process ID */
 344                         l = snprintf(jobid, joblen, "%u", current_pid());
 345                         break;
 346                 case 'u': /* user ID */
 347                         l = snprintf(jobid, joblen, "%u",
 348                                      from_kuid(&init_user_ns, current_fsuid()));
 349                         break;
 350                 case '\0': /* '%' at end of format string */
 351                         l = 0;
 352                         goto out;
 353                 default: /* drop unknown %x format strings */
 354                         l = 0;
 355                         break;
 356                 }
 357                 jobid += l;
 358                 joblen -= l;
 359         }
 360         /*
 361          * This points at the end of the buffer, so long as jobid is always
 362          * incremented the same amount as joblen is decremented.
 363          */
 364 out:
 365         jobid[joblen - 1] = '\0';
 366
 367         return joblen < 0 ? -EOVERFLOW : 0;
 368 }
 369
 370 /*
 371  * Hash initialization, copied from server-side job stats bucket sizes
 372  */
 373 #define HASH_JOBID_BKT_BITS 5
 374 #define HASH_JOBID_CUR_BITS 7
 375 #define HASH_JOBID_MAX_BITS 12
 376
 377 int jobid_cache_init(void)
 378 {
 379         int rc = 0;
 380         ENTRY;
 381
 382         if (jobid_hash)
 383                 return 0;
 384
 385         spin_lock_init(&jobid_hash_lock);
 386         jobid_hash = cfs_hash_create("JOBID_HASH", HASH_JOBID_CUR_BITS,
 387                                      HASH_JOBID_MAX_BITS, HASH_JOBID_BKT_BITS,
 388                                      0, CFS_HASH_MIN_THETA, CFS_HASH_MAX_THETA,
 389                                      &jobid_hash_ops, CFS_HASH_DEFAULT);
 390         if (!jobid_hash)
 391                 rc = -ENOMEM;
 392
 393         RETURN(rc);
 394 }
 395 EXPORT_SYMBOL(jobid_cache_init);
 396
 397 void jobid_cache_fini(void)
 398 {
 399         struct cfs_hash *tmp_hash;
 400         ENTRY;
 401
 402         spin_lock(&jobid_hash_lock);
 403         tmp_hash = jobid_hash;
 404         jobid_hash = NULL;
 405         spin_unlock(&jobid_hash_lock);
 406
 407         if (tmp_hash != NULL) {
 408                 cfs_hash_cond_del(tmp_hash, jobid_should_free_item, NULL);
 409                 cfs_hash_putref(tmp_hash);
 410         }
 411
 412         EXIT;
 413 }
 414 EXPORT_SYMBOL(jobid_cache_fini);
 415
 416 /*
 417  * Hash operations for pid<->jobid
 418  */
 419 static unsigned jobid_hashfn(struct cfs_hash *hs, const void *key,
 420                              unsigned mask)
 421 {
 422         return cfs_hash_djb2_hash(key, sizeof(pid_t), mask);
 423 }
 424
 425 static void *jobid_key(struct hlist_node *hnode)
 426 {
 427         struct jobid_pid_map *pidmap;
 428
 429         pidmap = hlist_entry(hnode, struct jobid_pid_map, jp_hash);
 430         return &pidmap->jp_pid;
 431 }
 432
 433 static int jobid_keycmp(const void *key, struct hlist_node *hnode)
 434 {
 435         const pid_t *pid_key1;
 436         const pid_t *pid_key2;
 437
 438         LASSERT(key != NULL);
 439         pid_key1 = (pid_t *)key;
 440         pid_key2 = (pid_t *)jobid_key(hnode);
 441
 442         return *pid_key1 == *pid_key2;
 443 }
 444
 445 static void *jobid_object(struct hlist_node *hnode)
 446 {
 447         return hlist_entry(hnode, struct jobid_pid_map, jp_hash);
 448 }
 449
 450 static void jobid_get(struct cfs_hash *hs, struct hlist_node *hnode)
 451 {
 452         struct jobid_pid_map *pidmap;
 453
 454         pidmap = hlist_entry(hnode, struct jobid_pid_map, jp_hash);
 455
 456         atomic_inc(&pidmap->jp_refcount);
 457 }
 458
 459 static void jobid_put_locked(struct cfs_hash *hs, struct hlist_node *hnode)
 460 {
 461         struct jobid_pid_map *pidmap;
 462
 463         if (hnode == NULL)
 464                 return;
 465
 466         pidmap = hlist_entry(hnode, struct jobid_pid_map, jp_hash);
 467         LASSERT(atomic_read(&pidmap->jp_refcount) > 0);
 468         if (atomic_dec_and_test(&pidmap->jp_refcount)) {
 469                 CDEBUG(D_INFO, "Freeing: %d->%s\n",
 470                        pidmap->jp_pid, pidmap->jp_jobid);
 471
 472                 OBD_FREE_PTR(pidmap);
 473         }
 474 }
 475
 476 static struct cfs_hash_ops jobid_hash_ops = {
 477         .hs_hash        = jobid_hashfn,
 478         .hs_keycmp      = jobid_keycmp,
 479         .hs_key         = jobid_key,
 480         .hs_object      = jobid_object,
 481         .hs_get         = jobid_get,
 482         .hs_put         = jobid_put_locked,
 483         .hs_put_locked  = jobid_put_locked,
 484 };
 485
 486 /**
 487  * Generate the job identifier string for this process for tracking purposes.
 488  *
 489  * Fill in @jobid string based on the value of obd_jobid_var:
 490  * JOBSTATS_DISABLE:      none
 491  * JOBSTATS_NODELOCAL:    content of obd_jobid_node (jobid_interpret_string())
 492  * JOBSTATS_PROCNAME_UID: process name/UID
 493  * anything else:         look up obd_jobid_var in the processes environment
 494  *
 495  * Return -ve error number, 0 on success.
 496  */
 497 int lustre_get_jobid(char *jobid, size_t joblen)
 498 {
 499         int rc = 0;
 500         ENTRY;
 501
 502         if (unlikely(joblen < 2)) {
 503                 if (joblen == 1)
 504                         jobid[0] = '\0';
 505                 RETURN(-EINVAL);
 506         }
 507
 508         if (strcmp(obd_jobid_var, JOBSTATS_DISABLE) == 0) {
 509                 /* Jobstats isn't enabled */
 510                 memset(jobid, 0, joblen);
 511         } else if (strcmp(obd_jobid_var, JOBSTATS_NODELOCAL) == 0) {
 512                 /* Whole node dedicated to single job */
 513                 rc = jobid_interpret_string(obd_jobid_name, jobid, joblen);
 514         } else if (strcmp(obd_jobid_var, JOBSTATS_PROCNAME_UID) == 0) {
 515                 rc = jobid_interpret_string("%e.%u", jobid, joblen);
 516         } else if (jobid_name_is_valid(current_comm())) {
 517                 /*
 518                  * obd_jobid_var holds the jobid environment variable name.
 519                  * Skip initial check if obd_jobid_name already uses "%j",
 520                  * otherwise try just "%j" first, then fall back to whatever
 521                  * is in obd_jobid_name if obd_jobid_var is not found.
 522                  */
 523                 rc = -EAGAIN;
 524                 if (!strnstr(obd_jobid_name, "%j", joblen))
 525                         rc = jobid_get_from_cache(jobid, joblen);
 526
 527                 /* fall back to jobid_node if jobid_var not in environment */
 528                 if (rc < 0) {
 529                         int rc2 = jobid_interpret_string(obd_jobid_name,
 530                                                          jobid, joblen);
 531                         if (!rc2)
 532                                 rc = 0;
 533                 }
 534         }
 535
 536         RETURN(rc);
 537 }
 538 EXPORT_SYMBOL(lustre_get_jobid);
 539
 540 /*
 541  * lustre_jobid_clear
 542  *
 543  * Search cache for JobID given by @find_jobid.
 544  * If any entries in the hash table match the value, they are removed
 545  */
 546 void lustre_jobid_clear(const char *find_jobid)
 547 {
 548         char jobid[LUSTRE_JOBID_SIZE];
 549         char *end;
 550
 551         if (jobid_hash == NULL)
 552                 return;
 553
 554         strlcpy(jobid, find_jobid, sizeof(jobid));
 555         /* trim \n off the end of the incoming jobid */
 556         end = strchr(jobid, '\n');
 557         if (end && *end == '\n')
 558                 *end = '\0';
 559
 560         CDEBUG(D_INFO, "Clearing Jobid: %s\n", jobid);
 561         cfs_hash_cond_del(jobid_hash, jobid_should_free_item, jobid);
 562
 563         CDEBUG(D_INFO, "%d items remain in jobID table\n",
 564                atomic_read(&jobid_hash->hs_count));
 565 }