lustre/ldlm/ldlm_test.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (c) 2002 Cluster File Systems, Inc. <info@clusterfs.com>
   5  * Copyright (c) 2002 Lawrence Livermore National Laboratory
   6  *  Author: James Newsome <newsome2@llnl.gov>
   7  *
   8  *   This file is part of Lustre, http://www.lustre.org.
   9  *
  10  *   Lustre is free software; you can redistribute it and/or
  11  *   modify it under the terms of version 2 of the GNU General Public
  12  *   License as published by the Free Software Foundation.
  13  *
  14  *   Lustre is distributed in the hope that it will be useful,
  15  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  *   GNU General Public License for more details.
  18  *
  19  *   You should have received a copy of the GNU General Public License
  20  *   along with Lustre; if not, write to the Free Software
  21  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22  */
  23
  24 #define DEBUG_SUBSYSTEM S_LDLM
  25
  26 #include <asm/atomic.h>
  27 #include <linux/types.h>
  28 #include <linux/random.h>
  29
  30 #include <linux/lustre_dlm.h>
  31 #include <linux/obd.h>
  32
  33 struct ldlm_test_thread {
  34         struct obd_device *obddev;
  35         struct ldlm_namespace *t_ns;
  36         struct list_head t_link;
  37         __u32 t_flags;
  38         wait_queue_head_t t_ctl_waitq;
  39 };
  40
  41 struct ldlm_test_lock {
  42         struct list_head l_link;
  43         struct lustre_handle l_lockh;
  44 };
  45
  46 static unsigned int max_locks;
  47 static unsigned int num_resources;
  48 static unsigned int num_extents;
  49
  50 static spinlock_t ctl_lock = SPIN_LOCK_UNLOCKED;
  51 /* protect these with the ctl_lock */
  52 static LIST_HEAD(ctl_threads);
  53 static int regression_running = 0;
  54 static LIST_HEAD(lock_list);
  55 static int num_locks = 0;
  56
  57 /* cumulative stats for regression test */
  58 static atomic_t locks_requested = ATOMIC_INIT(0);
  59 static atomic_t converts_requested = ATOMIC_INIT(0);
  60 static atomic_t locks_granted = ATOMIC_INIT(0);
  61 static atomic_t locks_matched = ATOMIC_INIT(0);
  62
  63 /* making this a global avoids the problem of having pointers
  64  * to garbage after the test exits.
  65  */
  66 static struct lustre_handle regress_connh;
  67
  68 static int ldlm_do_decrement(void);
  69 static int ldlm_do_enqueue(struct ldlm_test_thread *thread);
  70 static int ldlm_do_convert(void);
  71
  72 /*
  73  * blocking ast for regression test.
  74  * Just cancels lock
  75  */
  76 static int ldlm_test_blocking_ast(struct ldlm_lock *lock,
  77                                   struct ldlm_lock_desc *new,
  78                                   void *data, __u32 data_len)
  79 {
  80         int rc;
  81         struct lustre_handle lockh;
  82         ENTRY;
  83
  84         LDLM_DEBUG(lock, "We're blocking. Cancelling lock");
  85         ldlm_lock2handle(lock, &lockh);
  86         rc = ldlm_cli_cancel(&lockh);
  87         if (rc < 0) {
  88                 CERROR("ldlm_cli_cancel: %d\n", rc);
  89                 LBUG();
  90         }
  91
  92         RETURN(0);
  93 }
  94
  95 /* blocking ast for basic tests. noop */
  96 static int ldlm_blocking_ast(struct ldlm_lock *lock,
  97                              struct ldlm_lock_desc *new,
  98                              void *data, __u32 data_len)
  99 {
 100         ENTRY;
 101         CERROR("ldlm_blocking_ast: lock=%p, new=%p\n", lock, new);
 102         RETURN(0);
 103 }
 104
 105 /* Completion ast for regression test.
 106  * Does not sleep when blocked.
 107  */
 108 static int ldlm_test_completion_ast(struct ldlm_lock *lock, int flags)
 109 {
 110         struct ldlm_test_lock *lock_info;
 111         ENTRY;
 112
 113         if (flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
 114                       LDLM_FL_BLOCK_CONV)) {
 115
 116                 LDLM_DEBUG(lock, "client-side enqueue returned a blocked lock");
 117                 RETURN(0);
 118         }
 119
 120         if (lock->l_granted_mode != lock->l_req_mode)
 121                 CERROR("completion ast called with non-granted lock\n");
 122
 123         /* add to list of granted locks */
 124
 125         if (flags & LDLM_FL_WAIT_NOREPROC) {
 126                 atomic_inc(&locks_matched);
 127                 LDLM_DEBUG(lock, "lock matched");
 128         } else {
 129                 atomic_inc(&locks_granted);
 130                 LDLM_DEBUG(lock, "lock granted");
 131         }
 132
 133         OBD_ALLOC(lock_info, sizeof(*lock_info));
 134         if (lock_info == NULL) {
 135                 LBUG();
 136                 RETURN(-ENOMEM);
 137         }
 138
 139         ldlm_lock2handle(lock, &lock_info->l_lockh);
 140
 141         spin_lock(&ctl_lock);
 142         list_add_tail(&lock_info->l_link, &lock_list);
 143         num_locks++;
 144         spin_unlock(&ctl_lock);
 145
 146         RETURN(0);
 147 }
 148
 149 int ldlm_test_basics(struct obd_device *obddev)
 150 {
 151         struct ldlm_namespace *ns;
 152         struct ldlm_resource *res;
 153         __u64 res_id[RES_NAME_SIZE] = {1, 2, 3};
 154         ldlm_error_t err;
 155         struct ldlm_lock *lock1, *lock;
 156         int flags;
 157         ENTRY;
 158
 159         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 160         if (ns == NULL)
 161                 LBUG();
 162
 163         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_CR, NULL, 0);
 164         if (lock1 == NULL)
 165                 LBUG();
 166         err = ldlm_lock_enqueue(lock1, NULL, 0, &flags,
 167                                 ldlm_completion_ast, ldlm_blocking_ast);
 168         if (err != ELDLM_OK)
 169                 LBUG();
 170
 171         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_EX, NULL, 0);
 172         if (lock == NULL)
 173                 LBUG();
 174         err = ldlm_lock_enqueue(lock, NULL, 0, &flags,
 175                                 ldlm_completion_ast, ldlm_blocking_ast);
 176         if (err != ELDLM_OK)
 177                 LBUG();
 178         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 179                 LBUG();
 180
 181         res = ldlm_resource_get(ns, NULL, res_id, LDLM_PLAIN, 1);
 182         if (res == NULL)
 183                 LBUG();
 184         ldlm_resource_dump(res);
 185
 186         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 187         if (res != NULL)
 188                 ldlm_reprocess_all(res);
 189
 190         ldlm_resource_dump(res);
 191         ldlm_namespace_free(ns);
 192
 193         RETURN(0);
 194 }
 195
 196 int ldlm_test_extents(struct obd_device *obddev)
 197 {
 198         struct ldlm_namespace *ns;
 199         struct ldlm_resource *res;
 200         struct ldlm_lock *lock, *lock1, *lock2;
 201         __u64 res_id[RES_NAME_SIZE] = {0, 0, 0};
 202         struct ldlm_extent ext1 = {4, 6}, ext2 = {6, 9}, ext3 = {10, 11};
 203         ldlm_error_t err;
 204         int flags;
 205         ENTRY;
 206
 207         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 208         if (ns == NULL)
 209                 LBUG();
 210
 211         flags = 0;
 212         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR, NULL,
 213                                  0);
 214         if (lock1 == NULL)
 215                 LBUG();
 216         err = ldlm_lock_enqueue(lock1, &ext1, sizeof(ext1), &flags, NULL, NULL);
 217         if (err != ELDLM_OK)
 218                 LBUG();
 219         if (!(flags & LDLM_FL_LOCK_CHANGED))
 220                 LBUG();
 221
 222         flags = 0;
 223         lock2 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR,
 224                                 NULL, 0);
 225         err = ldlm_lock_enqueue(lock2, &ext2, sizeof(ext2), &flags, NULL, NULL);
 226         if (err != ELDLM_OK)
 227                 LBUG();
 228         if (!(flags & LDLM_FL_LOCK_CHANGED))
 229                 LBUG();
 230
 231         flags = 0;
 232         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_EX, NULL, 0);
 233         if (lock == NULL)
 234                 LBUG();
 235         err = ldlm_lock_enqueue(lock, &ext3, sizeof(ext3), &flags,
 236                                 NULL, NULL);
 237         if (err != ELDLM_OK)
 238                 LBUG();
 239         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 240                 LBUG();
 241         if (flags & LDLM_FL_LOCK_CHANGED)
 242                 LBUG();
 243
 244         /* Convert/cancel blocking locks */
 245         flags = 0;
 246         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 247         if (res != NULL)
 248                 ldlm_reprocess_all(res);
 249
 250         ldlm_lock_cancel(lock2);
 251         if (res != NULL)
 252                 ldlm_reprocess_all(res);
 253
 254         /* Dump the results */
 255         res = ldlm_resource_get(ns, NULL, res_id, LDLM_EXTENT, 0);
 256         if (res == NULL)
 257                 LBUG();
 258         ldlm_resource_dump(res);
 259         ldlm_namespace_free(ns);
 260
 261         RETURN(0);
 262 }
 263
 264 static int ldlm_test_network(struct obd_device *obddev,
 265                              struct lustre_handle *connh)
 266 {
 267
 268         __u64 res_id[RES_NAME_SIZE] = {1, 2, 3};
 269         struct ldlm_extent ext = {4, 6};
 270         struct lustre_handle lockh1;
 271         struct ldlm_lock *lock;
 272         int flags = 0;
 273         ldlm_error_t err;
 274         ENTRY;
 275
 276         err = ldlm_cli_enqueue(connh, NULL, obddev->obd_namespace, NULL, res_id,
 277                                LDLM_EXTENT, &ext, sizeof(ext), LCK_PR, &flags,
 278                                ldlm_completion_ast, NULL, NULL, 0, &lockh1);
 279
 280         CERROR("ldlm_cli_enqueue: %d\n", err);
 281
 282         flags = 0;
 283         err = ldlm_cli_convert(&lockh1, LCK_EX, &flags);
 284         CERROR("ldlm_cli_convert: %d\n", err);
 285
 286         lock = ldlm_handle2lock(&lockh1);
 287         ldlm_lock_dump(lock);
 288         ldlm_lock_put(lock);
 289
 290         /* Need to decrement old mode. Don't bother incrementing new
 291          * mode since the test is done.
 292          */
 293         if (err == ELDLM_OK)
 294                 ldlm_lock_decref(&lockh1, LCK_PR);
 295
 296         RETURN(err);
 297 }
 298
 299 static int ldlm_do_decrement(void)
 300 {
 301         struct ldlm_test_lock *lock_info;
 302         struct ldlm_lock *lock;
 303         int rc = 0;
 304         ENTRY;
 305
 306         spin_lock(&ctl_lock);
 307         if(list_empty(&lock_list)) {
 308                 CERROR("lock_list is empty\n");
 309                 spin_unlock(&ctl_lock);
 310                 RETURN(0);
 311         }
 312
 313         /* delete from list */
 314         lock_info = list_entry(lock_list.next,
 315                         struct ldlm_test_lock, l_link);
 316         list_del(lock_list.next);
 317         num_locks--;
 318         spin_unlock(&ctl_lock);
 319
 320         /* decrement and free the info */
 321         lock = ldlm_handle2lock(&lock_info->l_lockh);
 322         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 323         ldlm_lock_put(lock);
 324
 325         OBD_FREE(lock_info, sizeof(*lock_info));
 326
 327         RETURN(rc);
 328 }
 329
 330 static int ldlm_do_enqueue(struct ldlm_test_thread *thread)
 331 {
 332         struct lustre_handle lockh;
 333         __u64 res_id[3] = {0};
 334         __u32 lock_mode;
 335         struct ldlm_extent ext;
 336         unsigned char random;
 337         int flags = 0, rc = 0;
 338         ENTRY;
 339
 340         /* Pick a random resource from 1 to num_resources */
 341         get_random_bytes(&random, sizeof(random));
 342         res_id[0] = random % num_resources;
 343
 344         /* Pick a random lock mode */
 345         get_random_bytes(&random, sizeof(random));
 346         lock_mode = random % LCK_NL + 1;
 347
 348         /* Pick a random extent */
 349         get_random_bytes(&random, sizeof(random));
 350         ext.start = random % num_extents;
 351         get_random_bytes(&random, sizeof(random));
 352         ext.end = random %
 353                 (num_extents - (int)ext.start) + ext.start;
 354
 355         LDLM_DEBUG_NOLOCK("about to enqueue with resource %d, mode %d,"
 356                           " extent %d -> %d",
 357                           (int)res_id[0],
 358                           lock_mode,
 359                           (int)ext.start,
 360                           (int)ext.end);
 361
 362         rc = ldlm_match_or_enqueue(&regress_connh,
 363                                    NULL,
 364                                    thread->obddev->obd_namespace,
 365                                    NULL, res_id, LDLM_EXTENT, &ext,
 366                                    sizeof(ext), lock_mode, &flags,
 367                                    ldlm_test_completion_ast,
 368                                    ldlm_test_blocking_ast,
 369                                    NULL, 0, &lockh);
 370
 371         atomic_inc(&locks_requested);
 372
 373         if (rc < 0) {
 374                 CERROR("ldlm_cli_enqueue: %d\n", rc);
 375                 LBUG();
 376         }
 377
 378         RETURN(rc);
 379 }
 380
 381 static int ldlm_do_convert(void)
 382 {
 383         __u32 lock_mode;
 384         unsigned char random;
 385         int flags = 0, rc = 0;
 386         struct ldlm_test_lock *lock_info;
 387         struct ldlm_lock *lock;
 388         ENTRY;
 389
 390         /* delete from list */
 391         spin_lock(&ctl_lock);
 392         lock_info = list_entry(lock_list.next, struct ldlm_test_lock, l_link);
 393         list_del(lock_list.next);
 394         num_locks--;
 395         spin_unlock(&ctl_lock);
 396
 397         /* Pick a random lock mode */
 398         get_random_bytes(&random, sizeof(random));
 399         lock_mode = random % LCK_NL + 1;
 400
 401         /* do the conversion */
 402         rc = ldlm_cli_convert(&lock_info->l_lockh , lock_mode, &flags);
 403         atomic_inc(&converts_requested);
 404
 405         if (rc < 0) {
 406                 CERROR("ldlm_cli_convert: %d\n", rc);
 407                 LBUG();
 408         }
 409
 410         /*
 411          *  Adjust reference counts.
 412          *  FIXME: This is technically a bit... wrong,
 413          *  since we don't know when/if the convert succeeded
 414          */
 415         ldlm_lock_addref(&lock_info->l_lockh, lock_mode);
 416         lock = ldlm_handle2lock(&lock_info->l_lockh);
 417         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 418         ldlm_lock_put(lock);
 419
 420         OBD_FREE(lock_info, sizeof(*lock_info));
 421
 422         RETURN(rc);
 423 }
 424
 425
 426
 427 static int ldlm_test_main(void *data)
 428 {
 429         struct ldlm_test_thread *thread = data;
 430         ENTRY;
 431
 432         lock_kernel();
 433         daemonize();
 434         spin_lock_irq(&current->sigmask_lock);
 435         sigfillset(&current->blocked);
 436         recalc_sigpending(current);
 437         spin_unlock_irq(&current->sigmask_lock);
 438
 439         sprintf(current->comm, "ldlm_test");
 440         unlock_kernel();
 441
 442         /* Record that the thread is running */
 443         thread->t_flags |= SVC_RUNNING;
 444         wake_up(&thread->t_ctl_waitq);
 445
 446         while (!(thread->t_flags & SVC_STOPPING)) {
 447                 unsigned char random;
 448                 unsigned char dec_chance, con_chance;
 449                 unsigned char chance_left = 100;
 450
 451                 spin_lock(&ctl_lock);
 452                 /* probability of decrementing increases linearly
 453                  * as more locks are held.
 454                  */
 455                 dec_chance = chance_left * num_locks / max_locks;
 456                 chance_left -= dec_chance;
 457
 458                 /* FIXME: conversions temporarily disabled
 459                  * until they are working correctly.
 460                  */
 461                 /* con_chance = chance_left * num_locks / max_locks; */
 462                 con_chance = 0;
 463                 chance_left -= con_chance;
 464                 spin_unlock(&ctl_lock);
 465
 466                 get_random_bytes(&random, sizeof(random));
 467
 468                 random = random % 100;
 469                 if (random < dec_chance)
 470                         ldlm_do_decrement();
 471                 else if (random < (dec_chance + con_chance))
 472                         ldlm_do_convert();
 473                 else
 474                         ldlm_do_enqueue(thread);
 475
 476                 LDLM_DEBUG_NOLOCK("locks requested: %d, "
 477                                   "conversions requested %d",
 478                                   atomic_read(&locks_requested),
 479                                   atomic_read(&converts_requested));
 480                 LDLM_DEBUG_NOLOCK("locks granted: %d, "
 481                                   "locks matched: %d",
 482                                   atomic_read(&locks_granted),
 483                                   atomic_read(&locks_matched));
 484
 485                 spin_lock(&ctl_lock);
 486                 LDLM_DEBUG_NOLOCK("lock references currently held: %d, ",
 487                                   num_locks);
 488                 spin_unlock(&ctl_lock);
 489
 490                 /*
 491                  * We don't sleep after a lock being blocked, so let's
 492                  * make sure other things can run.
 493                  */
 494                 schedule();
 495         }
 496
 497         thread->t_flags |= SVC_STOPPED;
 498         wake_up(&thread->t_ctl_waitq);
 499
 500         RETURN(0);
 501 }
 502
 503 static int ldlm_start_thread(struct obd_device *obddev,
 504                              struct lustre_handle *connh)
 505 {
 506         struct ldlm_test_thread *test;
 507         int rc;
 508         ENTRY;
 509
 510         OBD_ALLOC(test, sizeof(*test));
 511         if (test == NULL) {
 512                 LBUG();
 513                 RETURN(-ENOMEM);
 514         }
 515         init_waitqueue_head(&test->t_ctl_waitq);
 516
 517         test->obddev = obddev;
 518
 519         spin_lock(&ctl_lock);
 520         list_add(&test->t_link, &ctl_threads);
 521         spin_unlock(&ctl_lock);
 522
 523         rc = kernel_thread(ldlm_test_main, (void *)test,
 524                            CLONE_VM | CLONE_FS | CLONE_FILES);
 525         if (rc < 0) {
 526                 CERROR("cannot start thread\n");
 527                 RETURN(-EINVAL);
 528         }
 529         wait_event(test->t_ctl_waitq, test->t_flags & SVC_RUNNING);
 530
 531         RETURN(0);
 532 }
 533
 534 int ldlm_regression_start(struct obd_device *obddev,
 535                           struct lustre_handle *connh,
 536                           unsigned int threads, unsigned int max_locks_in,
 537                           unsigned int num_resources_in,
 538                           unsigned int num_extents_in)
 539 {
 540         int i, rc = 0;
 541         ENTRY;
 542
 543         spin_lock(&ctl_lock);
 544         if (regression_running) {
 545                 CERROR("You can't start the ldlm regression twice.\n");
 546                 spin_unlock(&ctl_lock);
 547                 RETURN(-EINVAL);
 548         }
 549         regression_running = 1;
 550         spin_unlock(&ctl_lock);
 551
 552         regress_connh = *connh;
 553         max_locks = max_locks_in;
 554         num_resources = num_resources_in;
 555         num_extents = num_extents_in;
 556
 557         LDLM_DEBUG_NOLOCK("regression test started: threads: %d, max_locks: "
 558                           "%d, num_res: %d, num_ext: %d\n",
 559                           threads, max_locks_in, num_resources_in,
 560                           num_extents_in);
 561
 562         for (i = 0; i < threads; i++) {
 563                 rc = ldlm_start_thread(obddev, connh);
 564                 if (rc < 0)
 565                         GOTO(cleanup, rc);
 566         }
 567
 568  cleanup:
 569         if (rc < 0)
 570                 ldlm_regression_stop();
 571         RETURN(rc);
 572 }
 573
 574 int ldlm_regression_stop(void)
 575 {
 576         ENTRY;
 577
 578         spin_lock(&ctl_lock);
 579         if (!regression_running) {
 580                 CERROR("The ldlm regression isn't started.\n");
 581                 spin_unlock(&ctl_lock);
 582                 RETURN(-EINVAL);
 583         }
 584
 585         while (!list_empty(&ctl_threads)) {
 586                 struct ldlm_test_thread *thread;
 587                 thread = list_entry(ctl_threads.next, struct ldlm_test_thread,
 588                                     t_link);
 589
 590                 thread->t_flags |= SVC_STOPPING;
 591
 592                 spin_unlock(&ctl_lock);
 593                 wake_up(&thread->t_ctl_waitq);
 594                 wait_event(thread->t_ctl_waitq, thread->t_flags & SVC_STOPPED);
 595                 spin_lock(&ctl_lock);
 596
 597                 list_del(&thread->t_link);
 598                 OBD_FREE(thread, sizeof(*thread));
 599         }
 600
 601         /* decrement all held locks */
 602         while (!list_empty(&lock_list)) {
 603                 struct ldlm_lock *lock;
 604                 struct ldlm_test_lock *lock_info =
 605                        list_entry(lock_list.next, struct ldlm_test_lock,
 606                                    l_link);
 607                 list_del(lock_list.next);
 608                 num_locks--;
 609
 610                 lock = ldlm_handle2lock(&lock_info->l_lockh);
 611                 ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 612                 ldlm_lock_put(lock);
 613
 614                 OBD_FREE(lock_info, sizeof(*lock_info));
 615         }
 616
 617         regression_running = 0;
 618         spin_unlock(&ctl_lock);
 619
 620         RETURN(0);
 621 }
 622
 623 int ldlm_test(struct obd_device *obddev, struct lustre_handle *connh)
 624 {
 625         int rc;
 626         rc = ldlm_test_basics(obddev);
 627         if (rc)
 628                 RETURN(rc);
 629
 630         rc = ldlm_test_extents(obddev);
 631         if (rc)
 632                 RETURN(rc);
 633
 634         rc = ldlm_test_network(obddev, connh);
 635         RETURN(rc);
 636 }