lustre/ldlm/ldlm_test.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (c) 2002 Cluster File Systems, Inc. <info@clusterfs.com>
   5  * Copyright (c) 2002 Lawrence Livermore National Laboratory
   6  *  Author: James Newsome <newsome2@llnl.gov>
   7  *
   8  *   This file is part of Lustre, http://www.lustre.org.
   9  *
  10  *   Lustre is free software; you can redistribute it and/or
  11  *   modify it under the terms of version 2 of the GNU General Public
  12  *   License as published by the Free Software Foundation.
  13  *
  14  *   Lustre is distributed in the hope that it will be useful,
  15  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  *   GNU General Public License for more details.
  18  *
  19  *   You should have received a copy of the GNU General Public License
  20  *   along with Lustre; if not, write to the Free Software
  21  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22  */
  23
  24 #define DEBUG_SUBSYSTEM S_LDLM
  25
  26 #include <asm/atomic.h>
  27 #include <linux/types.h>
  28 #include <linux/random.h>
  29
  30 #include <linux/lustre_dlm.h>
  31 #include <linux/obd.h>
  32
  33 struct ldlm_test_thread {
  34         struct obd_device *obddev;
  35         struct ldlm_namespace *t_ns;
  36         struct list_head t_link;
  37         __u32 t_flags;
  38         wait_queue_head_t t_ctl_waitq;
  39 };
  40
  41 struct ldlm_test_lock {
  42         struct list_head l_link;
  43         struct lustre_handle l_lockh;
  44 };
  45
  46 static unsigned int max_locks;
  47 static unsigned int num_resources;
  48 static unsigned int num_extents;
  49
  50 static spinlock_t ctl_lock = SPIN_LOCK_UNLOCKED;
  51 /* protect these with the ctl_lock */
  52 static LIST_HEAD(ctl_threads);
  53 static int regression_running = 0;
  54 static LIST_HEAD(lock_list);
  55 static int num_locks = 0;
  56
  57 /* cumulative stats for regression test */
  58 static atomic_t locks_requested = ATOMIC_INIT(0);
  59 static atomic_t converts_requested = ATOMIC_INIT(0);
  60 static atomic_t locks_granted = ATOMIC_INIT(0);
  61 static atomic_t locks_matched = ATOMIC_INIT(0);
  62
  63 /* making this a global avoids the problem of having pointers
  64  * to garbage after the test exits.
  65  */
  66 static struct lustre_handle regress_connh;
  67
  68 static int ldlm_do_decrement(void);
  69 static int ldlm_do_enqueue(struct ldlm_test_thread *thread);
  70 static int ldlm_do_convert(void);
  71
  72 /*
  73  * blocking ast for regression test.
  74  * Just cancels lock
  75  */
  76 static int ldlm_test_blocking_ast(struct ldlm_lock *lock,
  77                                   struct ldlm_lock_desc *new,
  78                                   void *data, __u32 data_len, int flag)
  79 {
  80         int rc;
  81         struct lustre_handle lockh;
  82         ENTRY;
  83
  84         switch (flag) {
  85         case LDLM_CB_BLOCKING:
  86                 LDLM_DEBUG(lock, "We're blocking. Cancelling lock");
  87                 ldlm_lock2handle(lock, &lockh);
  88                 rc = ldlm_cli_cancel(&lockh);
  89                 if (rc < 0) {
  90                         CERROR("ldlm_cli_cancel: %d\n", rc);
  91                         LBUG();
  92                 }
  93                 break;
  94         case LDLM_CB_CANCELING:
  95                 LDLM_DEBUG(lock, "this lock is being cancelled");
  96                 break;
  97         default:
  98                 LBUG();
  99         }
 100
 101         RETURN(0);
 102 }
 103
 104 /* blocking ast for basic tests. noop */
 105 static int ldlm_blocking_ast(struct ldlm_lock *lock,
 106                              struct ldlm_lock_desc *new,
 107                              void *data, __u32 data_len, int flag)
 108 {
 109         ENTRY;
 110         CERROR("ldlm_blocking_ast: lock=%p, new=%p, flag=%d\n", lock, new,
 111                flag);
 112         RETURN(0);
 113 }
 114
 115 /* Completion ast for regression test.
 116  * Does not sleep when blocked.
 117  */
 118 static int ldlm_test_completion_ast(struct ldlm_lock *lock, int flags)
 119 {
 120         struct ldlm_test_lock *lock_info;
 121         ENTRY;
 122
 123         if (flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
 124                      LDLM_FL_BLOCK_CONV)) {
 125                 LDLM_DEBUG(lock, "client-side enqueue returned a blocked lock");
 126                 RETURN(0);
 127         }
 128
 129         if (lock->l_granted_mode != lock->l_req_mode)
 130                 CERROR("completion ast called with non-granted lock\n");
 131
 132         /* add to list of granted locks */
 133
 134         if (flags & LDLM_FL_WAIT_NOREPROC) {
 135                 atomic_inc(&locks_matched);
 136                 LDLM_DEBUG(lock, "lock matched");
 137         } else {
 138                 atomic_inc(&locks_granted);
 139                 LDLM_DEBUG(lock, "lock granted");
 140         }
 141
 142         OBD_ALLOC(lock_info, sizeof(*lock_info));
 143         if (lock_info == NULL) {
 144                 LBUG();
 145                 RETURN(-ENOMEM);
 146         }
 147
 148         ldlm_lock2handle(lock, &lock_info->l_lockh);
 149
 150         spin_lock(&ctl_lock);
 151         list_add_tail(&lock_info->l_link, &lock_list);
 152         num_locks++;
 153         spin_unlock(&ctl_lock);
 154
 155         RETURN(0);
 156 }
 157
 158 int ldlm_test_basics(struct obd_device *obddev)
 159 {
 160         struct ldlm_namespace *ns;
 161         struct ldlm_resource *res;
 162         __u64 res_id[RES_NAME_SIZE] = {1, 2, 3};
 163         ldlm_error_t err;
 164         struct ldlm_lock *lock1, *lock;
 165         int flags;
 166         ENTRY;
 167
 168         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 169         if (ns == NULL)
 170                 LBUG();
 171
 172         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_CR, NULL, 0);
 173         if (lock1 == NULL)
 174                 LBUG();
 175         err = ldlm_lock_enqueue(ns, lock1, NULL, 0, &flags,
 176                                 ldlm_completion_ast, ldlm_blocking_ast);
 177         if (err != ELDLM_OK)
 178                 LBUG();
 179
 180         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_EX, NULL, 0);
 181         if (lock == NULL)
 182                 LBUG();
 183         err = ldlm_lock_enqueue(ns, lock, NULL, 0, &flags,
 184                                 ldlm_completion_ast, ldlm_blocking_ast);
 185         if (err != ELDLM_OK)
 186                 LBUG();
 187         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 188                 LBUG();
 189
 190         res = ldlm_resource_get(ns, NULL, res_id, LDLM_PLAIN, 1);
 191         if (res == NULL)
 192                 LBUG();
 193         ldlm_resource_dump(res);
 194
 195         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 196         if (res != NULL)
 197                 ldlm_reprocess_all(res);
 198
 199         ldlm_resource_dump(res);
 200         ldlm_namespace_free(ns);
 201
 202         RETURN(0);
 203 }
 204
 205 int ldlm_test_extents(struct obd_device *obddev)
 206 {
 207         struct ldlm_namespace *ns;
 208         struct ldlm_resource *res;
 209         struct ldlm_lock *lock, *lock1, *lock2;
 210         __u64 res_id[RES_NAME_SIZE] = {0, 0, 0};
 211         struct ldlm_extent ext1 = {4, 6}, ext2 = {6, 9}, ext3 = {10, 11};
 212         ldlm_error_t err;
 213         int flags;
 214         ENTRY;
 215
 216         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 217         if (ns == NULL)
 218                 LBUG();
 219
 220         flags = 0;
 221         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR, NULL,
 222                                  0);
 223         if (lock1 == NULL)
 224                 LBUG();
 225         err = ldlm_lock_enqueue(ns, lock1, &ext1, sizeof(ext1), &flags, NULL,
 226                                 NULL);
 227         if (err != ELDLM_OK)
 228                 LBUG();
 229         if (!(flags & LDLM_FL_LOCK_CHANGED))
 230                 LBUG();
 231
 232         flags = 0;
 233         lock2 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR,
 234                                 NULL, 0);
 235         err = ldlm_lock_enqueue(ns, lock2, &ext2, sizeof(ext2), &flags, NULL,
 236                                 NULL);
 237         if (err != ELDLM_OK)
 238                 LBUG();
 239         if (!(flags & LDLM_FL_LOCK_CHANGED))
 240                 LBUG();
 241
 242         flags = 0;
 243         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_EX, NULL, 0);
 244         if (lock == NULL)
 245                 LBUG();
 246         err = ldlm_lock_enqueue(ns, lock, &ext3, sizeof(ext3), &flags,
 247                                 NULL, NULL);
 248         if (err != ELDLM_OK)
 249                 LBUG();
 250         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 251                 LBUG();
 252         if (flags & LDLM_FL_LOCK_CHANGED)
 253                 LBUG();
 254
 255         /* Convert/cancel blocking locks */
 256         flags = 0;
 257         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 258         if (res != NULL)
 259                 ldlm_reprocess_all(res);
 260
 261         ldlm_lock_cancel(lock2);
 262         if (res != NULL)
 263                 ldlm_reprocess_all(res);
 264
 265         /* Dump the results */
 266         res = ldlm_resource_get(ns, NULL, res_id, LDLM_EXTENT, 0);
 267         if (res == NULL)
 268                 LBUG();
 269         ldlm_resource_dump(res);
 270         ldlm_namespace_free(ns);
 271
 272         RETURN(0);
 273 }
 274
 275 static int ldlm_test_network(struct obd_device *obddev,
 276                              struct lustre_handle *connh)
 277 {
 278
 279         __u64 res_id[RES_NAME_SIZE] = {1, 2, 3};
 280         struct ldlm_extent ext = {4, 6};
 281         struct lustre_handle lockh1;
 282         struct ldlm_lock *lock;
 283         int flags = 0;
 284         ldlm_error_t err;
 285         ENTRY;
 286
 287         err = ldlm_cli_enqueue(connh, NULL, obddev->obd_namespace, NULL, res_id,
 288                                LDLM_EXTENT, &ext, sizeof(ext), LCK_PR, &flags,
 289                                ldlm_completion_ast, NULL, NULL, 0, &lockh1);
 290
 291         CERROR("ldlm_cli_enqueue: %d\n", err);
 292
 293         flags = 0;
 294         err = ldlm_cli_convert(&lockh1, LCK_EX, &flags);
 295         CERROR("ldlm_cli_convert: %d\n", err);
 296
 297         lock = ldlm_handle2lock(&lockh1);
 298         ldlm_lock_dump(D_OTHER, lock);
 299         ldlm_lock_put(lock);
 300
 301         /* Need to decrement old mode. Don't bother incrementing new
 302          * mode since the test is done.
 303          */
 304         if (err == ELDLM_OK)
 305                 ldlm_lock_decref(&lockh1, LCK_PR);
 306
 307         RETURN(err);
 308 }
 309
 310 static int ldlm_do_decrement(void)
 311 {
 312         struct ldlm_test_lock *lock_info;
 313         struct ldlm_lock *lock;
 314         int rc = 0;
 315         ENTRY;
 316
 317         spin_lock(&ctl_lock);
 318         if(list_empty(&lock_list)) {
 319                 CERROR("lock_list is empty\n");
 320                 spin_unlock(&ctl_lock);
 321                 RETURN(0);
 322         }
 323
 324         /* delete from list */
 325         lock_info = list_entry(lock_list.next,
 326                         struct ldlm_test_lock, l_link);
 327         list_del(lock_list.next);
 328         num_locks--;
 329         spin_unlock(&ctl_lock);
 330
 331         /* decrement and free the info */
 332         lock = ldlm_handle2lock(&lock_info->l_lockh);
 333         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 334         ldlm_lock_put(lock);
 335
 336         OBD_FREE(lock_info, sizeof(*lock_info));
 337
 338         RETURN(rc);
 339 }
 340
 341 static int ldlm_do_enqueue(struct ldlm_test_thread *thread)
 342 {
 343         struct lustre_handle lockh;
 344         __u64 res_id[3] = {0};
 345         __u32 lock_mode;
 346         struct ldlm_extent ext;
 347         unsigned char random;
 348         int flags = 0, rc = 0;
 349         ENTRY;
 350
 351         /* Pick a random resource from 1 to num_resources */
 352         get_random_bytes(&random, sizeof(random));
 353         res_id[0] = random % num_resources;
 354
 355         /* Pick a random lock mode */
 356         get_random_bytes(&random, sizeof(random));
 357         lock_mode = random % LCK_NL + 1;
 358
 359         /* Pick a random extent */
 360         get_random_bytes(&random, sizeof(random));
 361         ext.start = random % num_extents;
 362         get_random_bytes(&random, sizeof(random));
 363         ext.end = random %
 364                 (num_extents - (int)ext.start) + ext.start;
 365
 366         LDLM_DEBUG_NOLOCK("about to enqueue with resource "LPX64", mode %d,"
 367                           " extent "LPX64" -> "LPX64, res_id[0], lock_mode,
 368                           ext.start, ext.end);
 369
 370         rc = ldlm_match_or_enqueue(&regress_connh, NULL,
 371                                    thread->obddev->obd_namespace,
 372                                    NULL, res_id, LDLM_EXTENT, &ext,
 373                                    sizeof(ext), lock_mode, &flags,
 374                                    ldlm_test_completion_ast,
 375                                    ldlm_test_blocking_ast,
 376                                    NULL, 0, &lockh);
 377
 378         atomic_inc(&locks_requested);
 379
 380         if (rc < 0) {
 381                 CERROR("ldlm_cli_enqueue: %d\n", rc);
 382                 LBUG();
 383         }
 384
 385         RETURN(rc);
 386 }
 387
 388 static int ldlm_do_convert(void)
 389 {
 390         __u32 lock_mode;
 391         unsigned char random;
 392         int flags = 0, rc = 0;
 393         struct ldlm_test_lock *lock_info;
 394         struct ldlm_lock *lock;
 395         ENTRY;
 396
 397         /* delete from list */
 398         spin_lock(&ctl_lock);
 399         lock_info = list_entry(lock_list.next, struct ldlm_test_lock, l_link);
 400         list_del(lock_list.next);
 401         num_locks--;
 402         spin_unlock(&ctl_lock);
 403
 404         /* Pick a random lock mode */
 405         get_random_bytes(&random, sizeof(random));
 406         lock_mode = random % LCK_NL + 1;
 407
 408         /* do the conversion */
 409         rc = ldlm_cli_convert(&lock_info->l_lockh , lock_mode, &flags);
 410         atomic_inc(&converts_requested);
 411
 412         if (rc < 0) {
 413                 CERROR("ldlm_cli_convert: %d\n", rc);
 414                 LBUG();
 415         }
 416
 417         /*
 418          *  Adjust reference counts.
 419          *  FIXME: This is technically a bit... wrong,
 420          *  since we don't know when/if the convert succeeded
 421          */
 422         ldlm_lock_addref(&lock_info->l_lockh, lock_mode);
 423         lock = ldlm_handle2lock(&lock_info->l_lockh);
 424         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 425         ldlm_lock_put(lock);
 426
 427         OBD_FREE(lock_info, sizeof(*lock_info));
 428
 429         RETURN(rc);
 430 }
 431
 432
 433
 434 static int ldlm_test_main(void *data)
 435 {
 436         struct ldlm_test_thread *thread = data;
 437         unsigned long flags;
 438         ENTRY;
 439
 440         lock_kernel();
 441         daemonize();
 442 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0)
 443         sigfillset(&current->blocked);
 444         recalc_sigpending();
 445 #else
 446         spin_lock_irqsave(&current->sigmask_lock, flags);
 447         sigfillset(&current->blocked);
 448         recalc_sigpending(current);
 449         spin_unlock_irqrestore(&current->sigmask_lock, flags);
 450 #endif
 451
 452         sprintf(current->comm, "ldlm_test");
 453         unlock_kernel();
 454
 455         /* Record that the thread is running */
 456         thread->t_flags |= SVC_RUNNING;
 457         wake_up(&thread->t_ctl_waitq);
 458
 459         while (!(thread->t_flags & SVC_STOPPING)) {
 460                 unsigned char random;
 461                 unsigned char dec_chance, con_chance;
 462                 unsigned char chance_left = 100;
 463
 464                 spin_lock(&ctl_lock);
 465                 /* probability of decrementing increases linearly
 466                  * as more locks are held.
 467                  */
 468                 dec_chance = chance_left * num_locks / max_locks;
 469                 chance_left -= dec_chance;
 470
 471                 /* FIXME: conversions temporarily disabled
 472                  * until they are working correctly.
 473                  */
 474                 /* con_chance = chance_left * num_locks / max_locks; */
 475                 con_chance = 0;
 476                 chance_left -= con_chance;
 477                 spin_unlock(&ctl_lock);
 478
 479                 get_random_bytes(&random, sizeof(random));
 480
 481                 random = random % 100;
 482                 if (random < dec_chance)
 483                         ldlm_do_decrement();
 484                 else if (random < (dec_chance + con_chance))
 485                         ldlm_do_convert();
 486                 else
 487                         ldlm_do_enqueue(thread);
 488
 489                 LDLM_DEBUG_NOLOCK("locks requested: %d, "
 490                                   "conversions requested %d",
 491                                   atomic_read(&locks_requested),
 492                                   atomic_read(&converts_requested));
 493                 LDLM_DEBUG_NOLOCK("locks granted: %d, "
 494                                   "locks matched: %d",
 495                                   atomic_read(&locks_granted),
 496                                   atomic_read(&locks_matched));
 497
 498                 spin_lock(&ctl_lock);
 499                 LDLM_DEBUG_NOLOCK("lock references currently held: %d, ",
 500                                   num_locks);
 501                 spin_unlock(&ctl_lock);
 502
 503                 /*
 504                  * We don't sleep after a lock being blocked, so let's
 505                  * make sure other things can run.
 506                  */
 507                 schedule();
 508         }
 509
 510         thread->t_flags |= SVC_STOPPED;
 511         wake_up(&thread->t_ctl_waitq);
 512
 513         RETURN(0);
 514 }
 515
 516 static int ldlm_start_thread(struct obd_device *obddev,
 517                              struct lustre_handle *connh)
 518 {
 519         struct ldlm_test_thread *test;
 520         int rc;
 521         ENTRY;
 522
 523         OBD_ALLOC(test, sizeof(*test));
 524         if (test == NULL) {
 525                 LBUG();
 526                 RETURN(-ENOMEM);
 527         }
 528         init_waitqueue_head(&test->t_ctl_waitq);
 529
 530         test->obddev = obddev;
 531
 532         spin_lock(&ctl_lock);
 533         list_add(&test->t_link, &ctl_threads);
 534         spin_unlock(&ctl_lock);
 535
 536         rc = kernel_thread(ldlm_test_main, (void *)test,
 537                            CLONE_VM | CLONE_FS | CLONE_FILES);
 538         if (rc < 0) {
 539                 CERROR("cannot start thread\n");
 540                 RETURN(-EINVAL);
 541         }
 542         wait_event(test->t_ctl_waitq, test->t_flags & SVC_RUNNING);
 543
 544         RETURN(0);
 545 }
 546
 547 int ldlm_regression_start(struct obd_device *obddev,
 548                           struct lustre_handle *connh,
 549                           unsigned int threads, unsigned int max_locks_in,
 550                           unsigned int num_resources_in,
 551                           unsigned int num_extents_in)
 552 {
 553         int i, rc = 0;
 554         ENTRY;
 555
 556         spin_lock(&ctl_lock);
 557         if (regression_running) {
 558                 CERROR("You can't start the ldlm regression twice.\n");
 559                 spin_unlock(&ctl_lock);
 560                 RETURN(-EINVAL);
 561         }
 562         regression_running = 1;
 563         spin_unlock(&ctl_lock);
 564
 565         regress_connh = *connh;
 566         max_locks = max_locks_in;
 567         num_resources = num_resources_in;
 568         num_extents = num_extents_in;
 569
 570         LDLM_DEBUG_NOLOCK("regression test started: threads: %d, max_locks: "
 571                           "%d, num_res: %d, num_ext: %d\n",
 572                           threads, max_locks_in, num_resources_in,
 573                           num_extents_in);
 574
 575         for (i = 0; i < threads; i++) {
 576                 rc = ldlm_start_thread(obddev, connh);
 577                 if (rc < 0)
 578                         GOTO(cleanup, rc);
 579         }
 580
 581  cleanup:
 582         if (rc < 0)
 583                 ldlm_regression_stop();
 584         RETURN(rc);
 585 }
 586
 587 int ldlm_regression_stop(void)
 588 {
 589         ENTRY;
 590
 591         spin_lock(&ctl_lock);
 592         if (!regression_running) {
 593                 CERROR("The ldlm regression isn't started.\n");
 594                 spin_unlock(&ctl_lock);
 595                 RETURN(-EINVAL);
 596         }
 597
 598         while (!list_empty(&ctl_threads)) {
 599                 struct ldlm_test_thread *thread;
 600                 thread = list_entry(ctl_threads.next, struct ldlm_test_thread,
 601                                     t_link);
 602
 603                 thread->t_flags |= SVC_STOPPING;
 604
 605                 spin_unlock(&ctl_lock);
 606                 wake_up(&thread->t_ctl_waitq);
 607                 wait_event(thread->t_ctl_waitq, thread->t_flags & SVC_STOPPED);
 608                 spin_lock(&ctl_lock);
 609
 610                 list_del(&thread->t_link);
 611                 OBD_FREE(thread, sizeof(*thread));
 612         }
 613
 614         /* decrement all held locks */
 615         while (!list_empty(&lock_list)) {
 616                 struct ldlm_lock *lock;
 617                 struct ldlm_test_lock *lock_info =
 618                        list_entry(lock_list.next, struct ldlm_test_lock,
 619                                    l_link);
 620                 list_del(lock_list.next);
 621                 num_locks--;
 622
 623                 lock = ldlm_handle2lock(&lock_info->l_lockh);
 624                 ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 625                 ldlm_lock_put(lock);
 626
 627                 OBD_FREE(lock_info, sizeof(*lock_info));
 628         }
 629
 630         regression_running = 0;
 631         spin_unlock(&ctl_lock);
 632
 633         RETURN(0);
 634 }
 635
 636 int ldlm_test(struct obd_device *obddev, struct lustre_handle *connh)
 637 {
 638         int rc;
 639         rc = ldlm_test_basics(obddev);
 640         if (rc)
 641                 RETURN(rc);
 642
 643         rc = ldlm_test_extents(obddev);
 644         if (rc)
 645                 RETURN(rc);
 646
 647         rc = ldlm_test_network(obddev, connh);
 648         RETURN(rc);
 649 }