lustre/ldlm/ldlm_test.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (c) 2002 Cluster File Systems, Inc. <info@clusterfs.com>
   5  * Copyright (c) 2002 Lawrence Livermore National Laboratory
   6  *  Author: James Newsome <newsome2@llnl.gov>
   7  *
   8  *   This file is part of Lustre, http://www.lustre.org.
   9  *
  10  *   Lustre is free software; you can redistribute it and/or
  11  *   modify it under the terms of version 2 of the GNU General Public
  12  *   License as published by the Free Software Foundation.
  13  *
  14  *   Lustre is distributed in the hope that it will be useful,
  15  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  *   GNU General Public License for more details.
  18  *
  19  *   You should have received a copy of the GNU General Public License
  20  *   along with Lustre; if not, write to the Free Software
  21  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22  */
  23
  24 #define DEBUG_SUBSYSTEM S_LDLM
  25
  26 #include <asm/atomic.h>
  27 #include <linux/types.h>
  28 #include <linux/random.h>
  29
  30 #include <linux/lustre_dlm.h>
  31 #include <linux/obd.h>
  32
  33 struct ldlm_test_thread {
  34         struct obd_device *obddev;
  35         struct ldlm_namespace *t_ns;
  36         struct list_head t_link;
  37         __u32 t_flags;
  38         wait_queue_head_t t_ctl_waitq;
  39 };
  40
  41 struct ldlm_test_lock {
  42         struct list_head l_link;
  43         struct lustre_handle l_lockh;
  44 };
  45
  46 static unsigned int max_locks;
  47 static unsigned int num_resources;
  48 static unsigned int num_extents;
  49
  50 static spinlock_t ctl_lock = SPIN_LOCK_UNLOCKED;
  51 /* protect these with the ctl_lock */
  52 static LIST_HEAD(ctl_threads);
  53 static int regression_running = 0;
  54 static LIST_HEAD(lock_list);
  55 static int num_locks = 0;
  56
  57 /* cumulative stats for regression test */
  58 static atomic_t locks_requested = ATOMIC_INIT(0);
  59 static atomic_t converts_requested = ATOMIC_INIT(0);
  60 static atomic_t locks_granted = ATOMIC_INIT(0);
  61 static atomic_t locks_matched = ATOMIC_INIT(0);
  62
  63 /* making this a global avoids the problem of having pointers
  64  * to garbage after the test exits.
  65  */
  66 static struct lustre_handle regress_connh;
  67
  68 static int ldlm_do_decrement(void);
  69 static int ldlm_do_enqueue(struct ldlm_test_thread *thread);
  70 static int ldlm_do_convert(void);
  71
  72 /*
  73  * blocking ast for regression test.
  74  * Just cancels lock
  75  */
  76 static int ldlm_test_blocking_ast(struct ldlm_lock *lock,
  77                                   struct ldlm_lock_desc *new,
  78                                   void *data, int flag)
  79 {
  80         int rc;
  81         struct lustre_handle lockh;
  82         ENTRY;
  83
  84         switch (flag) {
  85         case LDLM_CB_BLOCKING:
  86                 LDLM_DEBUG(lock, "We're blocking. Cancelling lock");
  87                 ldlm_lock2handle(lock, &lockh);
  88                 rc = ldlm_cli_cancel(&lockh);
  89                 if (rc < 0) {
  90                         CERROR("ldlm_cli_cancel: %d\n", rc);
  91                         LBUG();
  92                 }
  93                 break;
  94         case LDLM_CB_CANCELING:
  95                 LDLM_DEBUG(lock, "this lock is being cancelled");
  96                 break;
  97         default:
  98                 LBUG();
  99         }
 100
 101         RETURN(0);
 102 }
 103
 104 /* blocking ast for basic tests. noop */
 105 static int ldlm_blocking_ast(struct ldlm_lock *lock,
 106                              struct ldlm_lock_desc *new,
 107                              void *data, int flag)
 108 {
 109         ENTRY;
 110         CERROR("ldlm_blocking_ast: lock=%p, new=%p, flag=%d\n", lock, new,
 111                flag);
 112         RETURN(0);
 113 }
 114
 115 /* Completion ast for regression test.
 116  * Does not sleep when blocked.
 117  */
 118 static int ldlm_test_completion_ast(struct ldlm_lock *lock, int flags, void *data)
 119 {
 120         struct ldlm_test_lock *lock_info;
 121         ENTRY;
 122
 123         if (flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
 124                      LDLM_FL_BLOCK_CONV)) {
 125                 LDLM_DEBUG(lock, "client-side enqueue returned a blocked lock");
 126                 RETURN(0);
 127         }
 128
 129         if (lock->l_granted_mode != lock->l_req_mode)
 130                 CERROR("completion ast called with non-granted lock\n");
 131
 132         /* add to list of granted locks */
 133
 134         if (flags & LDLM_FL_WAIT_NOREPROC) {
 135                 atomic_inc(&locks_matched);
 136                 LDLM_DEBUG(lock, "lock matched");
 137         } else {
 138                 atomic_inc(&locks_granted);
 139                 LDLM_DEBUG(lock, "lock granted");
 140         }
 141
 142         OBD_ALLOC(lock_info, sizeof(*lock_info));
 143         if (lock_info == NULL) {
 144                 LBUG();
 145                 RETURN(-ENOMEM);
 146         }
 147
 148         ldlm_lock2handle(lock, &lock_info->l_lockh);
 149
 150         spin_lock(&ctl_lock);
 151         list_add_tail(&lock_info->l_link, &lock_list);
 152         num_locks++;
 153         spin_unlock(&ctl_lock);
 154
 155         RETURN(0);
 156 }
 157
 158 int ldlm_test_basics(struct obd_device *obddev)
 159 {
 160         struct ldlm_namespace *ns;
 161         struct ldlm_resource *res;
 162         struct ldlm_res_id res_id = { .name = {1, 2, 3} };
 163         ldlm_error_t err;
 164         struct ldlm_lock *lock1, *lock;
 165         int flags;
 166         ENTRY;
 167
 168         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 169         if (ns == NULL)
 170                 LBUG();
 171
 172         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_CR, NULL, 0);
 173         if (lock1 == NULL)
 174                 LBUG();
 175         err = ldlm_lock_enqueue(ns, lock1, NULL, 0, &flags,
 176                                 ldlm_completion_ast, ldlm_blocking_ast);
 177         if (err != ELDLM_OK)
 178                 LBUG();
 179
 180         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_EX, NULL, 0);
 181         if (lock == NULL)
 182                 LBUG();
 183         err = ldlm_lock_enqueue(ns, lock, NULL, 0, &flags,
 184                                 ldlm_completion_ast, ldlm_blocking_ast);
 185         if (err != ELDLM_OK)
 186                 LBUG();
 187         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 188                 LBUG();
 189
 190         res = ldlm_resource_get(ns, NULL, res_id, LDLM_PLAIN, 1);
 191         if (res == NULL)
 192                 LBUG();
 193         ldlm_resource_dump(res);
 194
 195         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 196         if (res != NULL)
 197                 ldlm_reprocess_all(res);
 198
 199         ldlm_resource_dump(res);
 200         ldlm_namespace_free(ns, 0);
 201
 202         RETURN(0);
 203 }
 204
 205 int ldlm_test_extents(struct obd_device *obddev)
 206 {
 207         struct ldlm_namespace *ns;
 208         struct ldlm_resource *res;
 209         struct ldlm_lock *lock, *lock1, *lock2;
 210         struct ldlm_res_id res_id = { .name = {0} };
 211         struct ldlm_extent ext1 = {4, 6}, ext2 = {6, 9}, ext3 = {10, 11};
 212         ldlm_error_t err;
 213         int flags;
 214         ENTRY;
 215
 216         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 217         if (ns == NULL)
 218                 LBUG();
 219
 220         flags = 0;
 221         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR, NULL,
 222                                  0);
 223         if (lock1 == NULL)
 224                 LBUG();
 225         err = ldlm_lock_enqueue(ns, lock1, &ext1, sizeof(ext1), &flags, NULL,
 226                                 NULL);
 227         if (err != ELDLM_OK)
 228                 LBUG();
 229         if (!(flags & LDLM_FL_LOCK_CHANGED))
 230                 LBUG();
 231
 232         flags = 0;
 233         lock2 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR,
 234                                 NULL, 0);
 235         err = ldlm_lock_enqueue(ns, lock2, &ext2, sizeof(ext2), &flags, NULL,
 236                                 NULL);
 237         if (err != ELDLM_OK)
 238                 LBUG();
 239         if (!(flags & LDLM_FL_LOCK_CHANGED))
 240                 LBUG();
 241
 242         flags = 0;
 243         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_EX, NULL, 0);
 244         if (lock == NULL)
 245                 LBUG();
 246         err = ldlm_lock_enqueue(ns, lock, &ext3, sizeof(ext3), &flags,
 247                                 NULL, NULL);
 248         if (err != ELDLM_OK)
 249                 LBUG();
 250         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 251                 LBUG();
 252         if (flags & LDLM_FL_LOCK_CHANGED)
 253                 LBUG();
 254
 255         /* Convert/cancel blocking locks */
 256         flags = 0;
 257         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 258         if (res != NULL)
 259                 ldlm_reprocess_all(res);
 260
 261         ldlm_lock_cancel(lock2);
 262         if (res != NULL)
 263                 ldlm_reprocess_all(res);
 264
 265         /* Dump the results */
 266         res = ldlm_resource_get(ns, NULL, res_id, LDLM_EXTENT, 0);
 267         if (res == NULL)
 268                 LBUG();
 269         ldlm_resource_dump(res);
 270         ldlm_namespace_free(ns, 0);
 271
 272         RETURN(0);
 273 }
 274
 275 static int ldlm_test_network(struct obd_device *obddev,
 276                              struct lustre_handle *connh)
 277 {
 278         struct ldlm_res_id res_id = { .name = {1, 2, 3} };
 279         struct ldlm_extent ext = {4, 6};
 280         struct lustre_handle lockh1;
 281         struct ldlm_lock *lock;
 282         int flags = 0;
 283         ldlm_error_t err;
 284         ENTRY;
 285
 286         err = ldlm_cli_enqueue(connh, NULL, obddev->obd_namespace, NULL, res_id,
 287                                LDLM_EXTENT, &ext, sizeof(ext), LCK_PR, &flags,
 288                                ldlm_completion_ast, NULL, NULL, 0, &lockh1);
 289
 290         CERROR("ldlm_cli_enqueue: %d\n", err);
 291
 292         flags = 0;
 293         err = ldlm_cli_convert(&lockh1, LCK_EX, &flags);
 294         CERROR("ldlm_cli_convert: %d\n", err);
 295
 296         lock = ldlm_handle2lock(&lockh1);
 297         ldlm_lock_dump(D_OTHER, lock);
 298         ldlm_lock_put(lock);
 299
 300         /* Need to decrement old mode. Don't bother incrementing new
 301          * mode since the test is done.
 302          */
 303         if (err == ELDLM_OK)
 304                 ldlm_lock_decref(&lockh1, LCK_PR);
 305
 306         RETURN(err);
 307 }
 308
 309 static int ldlm_do_decrement(void)
 310 {
 311         struct ldlm_test_lock *lock_info;
 312         struct ldlm_lock *lock;
 313         int rc = 0;
 314         ENTRY;
 315
 316         spin_lock(&ctl_lock);
 317         if(list_empty(&lock_list)) {
 318                 CERROR("lock_list is empty\n");
 319                 spin_unlock(&ctl_lock);
 320                 RETURN(0);
 321         }
 322
 323         /* delete from list */
 324         lock_info = list_entry(lock_list.next,
 325                         struct ldlm_test_lock, l_link);
 326         list_del(lock_list.next);
 327         num_locks--;
 328         spin_unlock(&ctl_lock);
 329
 330         /* decrement and free the info */
 331         lock = ldlm_handle2lock(&lock_info->l_lockh);
 332         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 333         ldlm_lock_put(lock);
 334
 335         OBD_FREE(lock_info, sizeof(*lock_info));
 336
 337         RETURN(rc);
 338 }
 339
 340 static int ldlm_do_enqueue(struct ldlm_test_thread *thread)
 341 {
 342         struct lustre_handle lockh;
 343         struct ldlm_res_id res_id = { .name = {0} };
 344         __u32 lock_mode;
 345         struct ldlm_extent ext;
 346         unsigned char random;
 347         int flags = 0, rc = 0;
 348         ENTRY;
 349
 350         /* Pick a random resource from 1 to num_resources */
 351         get_random_bytes(&random, sizeof(random));
 352         res_id.name[0] = random % num_resources;
 353
 354         /* Pick a random lock mode */
 355         get_random_bytes(&random, sizeof(random));
 356         lock_mode = random % LCK_NL + 1;
 357
 358         /* Pick a random extent */
 359         get_random_bytes(&random, sizeof(random));
 360         ext.start = random % num_extents;
 361         get_random_bytes(&random, sizeof(random));
 362         ext.end = random %
 363                 (num_extents - (int)ext.start) + ext.start;
 364
 365         LDLM_DEBUG_NOLOCK("about to enqueue with resource "LPX64", mode %d,"
 366                           " extent "LPX64" -> "LPX64, res_id.name[0], lock_mode,
 367                           ext.start, ext.end);
 368
 369         rc = ldlm_match_or_enqueue(&regress_connh, NULL,
 370                                    thread->obddev->obd_namespace,
 371                                    NULL, res_id, LDLM_EXTENT, &ext,
 372                                    sizeof(ext), lock_mode, &flags,
 373                                    ldlm_test_completion_ast,
 374                                    ldlm_test_blocking_ast,
 375                                    NULL, 0, &lockh);
 376
 377         atomic_inc(&locks_requested);
 378
 379         if (rc < 0) {
 380                 CERROR("ldlm_cli_enqueue: %d\n", rc);
 381                 LBUG();
 382         }
 383
 384         RETURN(rc);
 385 }
 386
 387 static int ldlm_do_convert(void)
 388 {
 389         __u32 lock_mode;
 390         unsigned char random;
 391         int flags = 0, rc = 0;
 392         struct ldlm_test_lock *lock_info;
 393         struct ldlm_lock *lock;
 394         ENTRY;
 395
 396         /* delete from list */
 397         spin_lock(&ctl_lock);
 398         lock_info = list_entry(lock_list.next, struct ldlm_test_lock, l_link);
 399         list_del(lock_list.next);
 400         num_locks--;
 401         spin_unlock(&ctl_lock);
 402
 403         /* Pick a random lock mode */
 404         get_random_bytes(&random, sizeof(random));
 405         lock_mode = random % LCK_NL + 1;
 406
 407         /* do the conversion */
 408         rc = ldlm_cli_convert(&lock_info->l_lockh , lock_mode, &flags);
 409         atomic_inc(&converts_requested);
 410
 411         if (rc < 0) {
 412                 CERROR("ldlm_cli_convert: %d\n", rc);
 413                 LBUG();
 414         }
 415
 416         /*
 417          *  Adjust reference counts.
 418          *  FIXME: This is technically a bit... wrong,
 419          *  since we don't know when/if the convert succeeded
 420          */
 421         ldlm_lock_addref(&lock_info->l_lockh, lock_mode);
 422         lock = ldlm_handle2lock(&lock_info->l_lockh);
 423         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 424         ldlm_lock_put(lock);
 425
 426         OBD_FREE(lock_info, sizeof(*lock_info));
 427
 428         RETURN(rc);
 429 }
 430
 431
 432
 433 static int ldlm_test_main(void *data)
 434 {
 435         struct ldlm_test_thread *thread = data;
 436         unsigned long flags;
 437         ENTRY;
 438
 439         lock_kernel();
 440         daemonize();
 441 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0)
 442         sigfillset(&current->blocked);
 443         recalc_sigpending();
 444 #else
 445         spin_lock_irqsave(&current->sigmask_lock, flags);
 446         sigfillset(&current->blocked);
 447         recalc_sigpending(current);
 448         spin_unlock_irqrestore(&current->sigmask_lock, flags);
 449 #endif
 450
 451         sprintf(current->comm, "ldlm_test");
 452         unlock_kernel();
 453
 454         /* Record that the thread is running */
 455         thread->t_flags |= SVC_RUNNING;
 456         wake_up(&thread->t_ctl_waitq);
 457
 458         while (!(thread->t_flags & SVC_STOPPING)) {
 459                 unsigned char random;
 460                 unsigned char dec_chance, con_chance;
 461                 unsigned char chance_left = 100;
 462
 463                 spin_lock(&ctl_lock);
 464                 /* probability of decrementing increases linearly
 465                  * as more locks are held.
 466                  */
 467                 dec_chance = chance_left * num_locks / max_locks;
 468                 chance_left -= dec_chance;
 469
 470                 /* FIXME: conversions temporarily disabled
 471                  * until they are working correctly.
 472                  */
 473                 /* con_chance = chance_left * num_locks / max_locks; */
 474                 con_chance = 0;
 475                 chance_left -= con_chance;
 476                 spin_unlock(&ctl_lock);
 477
 478                 get_random_bytes(&random, sizeof(random));
 479
 480                 random = random % 100;
 481                 if (random < dec_chance)
 482                         ldlm_do_decrement();
 483                 else if (random < (dec_chance + con_chance))
 484                         ldlm_do_convert();
 485                 else
 486                         ldlm_do_enqueue(thread);
 487
 488                 LDLM_DEBUG_NOLOCK("locks requested: %d, "
 489                                   "conversions requested %d",
 490                                   atomic_read(&locks_requested),
 491                                   atomic_read(&converts_requested));
 492                 LDLM_DEBUG_NOLOCK("locks granted: %d, "
 493                                   "locks matched: %d",
 494                                   atomic_read(&locks_granted),
 495                                   atomic_read(&locks_matched));
 496
 497                 spin_lock(&ctl_lock);
 498                 LDLM_DEBUG_NOLOCK("lock references currently held: %d, ",
 499                                   num_locks);
 500                 spin_unlock(&ctl_lock);
 501
 502                 /*
 503                  * We don't sleep after a lock being blocked, so let's
 504                  * make sure other things can run.
 505                  */
 506                 schedule();
 507         }
 508
 509         thread->t_flags |= SVC_STOPPED;
 510         wake_up(&thread->t_ctl_waitq);
 511
 512         RETURN(0);
 513 }
 514
 515 static int ldlm_start_thread(struct obd_device *obddev,
 516                              struct lustre_handle *connh)
 517 {
 518         struct ldlm_test_thread *test;
 519         int rc;
 520         ENTRY;
 521
 522         OBD_ALLOC(test, sizeof(*test));
 523         if (test == NULL) {
 524                 LBUG();
 525                 RETURN(-ENOMEM);
 526         }
 527         init_waitqueue_head(&test->t_ctl_waitq);
 528
 529         test->obddev = obddev;
 530
 531         spin_lock(&ctl_lock);
 532         list_add(&test->t_link, &ctl_threads);
 533         spin_unlock(&ctl_lock);
 534
 535         rc = kernel_thread(ldlm_test_main, (void *)test,
 536                            CLONE_VM | CLONE_FS | CLONE_FILES);
 537         if (rc < 0) {
 538                 CERROR("cannot start thread\n");
 539                 RETURN(-EINVAL);
 540         }
 541         wait_event(test->t_ctl_waitq, test->t_flags & SVC_RUNNING);
 542
 543         RETURN(0);
 544 }
 545
 546 int ldlm_regression_start(struct obd_device *obddev,
 547                           struct lustre_handle *connh,
 548                           unsigned int threads, unsigned int max_locks_in,
 549                           unsigned int num_resources_in,
 550                           unsigned int num_extents_in)
 551 {
 552         int i, rc = 0;
 553         ENTRY;
 554
 555         spin_lock(&ctl_lock);
 556         if (regression_running) {
 557                 CERROR("You can't start the ldlm regression twice.\n");
 558                 spin_unlock(&ctl_lock);
 559                 RETURN(-EINVAL);
 560         }
 561         regression_running = 1;
 562         spin_unlock(&ctl_lock);
 563
 564         regress_connh = *connh;
 565         max_locks = max_locks_in;
 566         num_resources = num_resources_in;
 567         num_extents = num_extents_in;
 568
 569         LDLM_DEBUG_NOLOCK("regression test started: threads: %d, max_locks: "
 570                           "%d, num_res: %d, num_ext: %d\n",
 571                           threads, max_locks_in, num_resources_in,
 572                           num_extents_in);
 573
 574         for (i = 0; i < threads; i++) {
 575                 rc = ldlm_start_thread(obddev, connh);
 576                 if (rc < 0)
 577                         GOTO(cleanup, rc);
 578         }
 579
 580  cleanup:
 581         if (rc < 0)
 582                 ldlm_regression_stop();
 583         RETURN(rc);
 584 }
 585
 586 int ldlm_regression_stop(void)
 587 {
 588         ENTRY;
 589
 590         spin_lock(&ctl_lock);
 591         if (!regression_running) {
 592                 CERROR("The ldlm regression isn't started.\n");
 593                 spin_unlock(&ctl_lock);
 594                 RETURN(-EINVAL);
 595         }
 596
 597         while (!list_empty(&ctl_threads)) {
 598                 struct ldlm_test_thread *thread;
 599                 thread = list_entry(ctl_threads.next, struct ldlm_test_thread,
 600                                     t_link);
 601
 602                 thread->t_flags |= SVC_STOPPING;
 603
 604                 spin_unlock(&ctl_lock);
 605                 wake_up(&thread->t_ctl_waitq);
 606                 wait_event(thread->t_ctl_waitq, thread->t_flags & SVC_STOPPED);
 607                 spin_lock(&ctl_lock);
 608
 609                 list_del(&thread->t_link);
 610                 OBD_FREE(thread, sizeof(*thread));
 611         }
 612
 613         /* decrement all held locks */
 614         while (!list_empty(&lock_list)) {
 615                 struct ldlm_lock *lock;
 616                 struct ldlm_test_lock *lock_info =
 617                        list_entry(lock_list.next, struct ldlm_test_lock,
 618                                    l_link);
 619                 list_del(lock_list.next);
 620                 num_locks--;
 621
 622                 lock = ldlm_handle2lock(&lock_info->l_lockh);
 623                 ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 624                 ldlm_lock_put(lock);
 625
 626                 OBD_FREE(lock_info, sizeof(*lock_info));
 627         }
 628
 629         regression_running = 0;
 630         spin_unlock(&ctl_lock);
 631
 632         RETURN(0);
 633 }
 634
 635 int ldlm_test(struct obd_device *obddev, struct lustre_handle *connh)
 636 {
 637         int rc;
 638         rc = ldlm_test_basics(obddev);
 639         if (rc)
 640                 RETURN(rc);
 641
 642         rc = ldlm_test_extents(obddev);
 643         if (rc)
 644                 RETURN(rc);
 645
 646         rc = ldlm_test_network(obddev, connh);
 647         RETURN(rc);
 648 }