lustre/ldlm/ldlm_test.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (c) 2002 Cluster File Systems, Inc. <info@clusterfs.com>
   5  * Copyright (c) 2002 Lawrence Livermore National Laboratory
   6  *  Author: James Newsome <newsome2@llnl.gov>
   7  *
   8  *   This file is part of Lustre, http://www.lustre.org.
   9  *
  10  *   Lustre is free software; you can redistribute it and/or
  11  *   modify it under the terms of version 2 of the GNU General Public
  12  *   License as published by the Free Software Foundation.
  13  *
  14  *   Lustre is distributed in the hope that it will be useful,
  15  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  *   GNU General Public License for more details.
  18  *
  19  *   You should have received a copy of the GNU General Public License
  20  *   along with Lustre; if not, write to the Free Software
  21  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22  */
  23
  24 #define DEBUG_SUBSYSTEM S_LDLM
  25
  26 #include <asm/atomic.h>
  27 #include <linux/types.h>
  28 #include <linux/random.h>
  29
  30 #include <linux/lustre_dlm.h>
  31
  32 struct ldlm_test_thread {
  33         struct obd_device *obddev;
  34         struct ldlm_namespace *t_ns;
  35         struct list_head t_link;
  36         __u32 t_flags;
  37         wait_queue_head_t t_ctl_waitq;
  38 };
  39
  40 struct ldlm_test_lock {
  41         struct list_head l_link;
  42         struct lustre_handle l_lockh;
  43 };
  44
  45 static unsigned int max_locks;
  46 static unsigned int num_resources;
  47 static unsigned int num_extents;
  48
  49 static spinlock_t ctl_lock = SPIN_LOCK_UNLOCKED;
  50 /* protect these with the ctl_lock */
  51 static LIST_HEAD(ctl_threads);
  52 static int regression_running = 0;
  53 static LIST_HEAD(lock_list);
  54 static int num_locks = 0;
  55
  56 /* cumulative stats for regression test */
  57 static atomic_t locks_requested = ATOMIC_INIT(0);
  58 static atomic_t converts_requested = ATOMIC_INIT(0);
  59 static atomic_t locks_granted = ATOMIC_INIT(0);
  60 static atomic_t locks_matched = ATOMIC_INIT(0);
  61
  62 /* making this a global avoids the problem of having pointers
  63  * to garbage after the test exits.
  64  */
  65 static struct lustre_handle regress_connh;
  66
  67 static int ldlm_do_decrement(void);
  68 static int ldlm_do_enqueue(struct ldlm_test_thread *thread);
  69 static int ldlm_do_convert(void);
  70
  71 /*
  72  * blocking ast for regression test.
  73  * Just cancels lock
  74  */
  75 static int ldlm_test_blocking_ast(struct ldlm_lock *lock,
  76                                   struct ldlm_lock_desc *new,
  77                                   void *data, __u32 data_len)
  78 {
  79         int rc;
  80         struct lustre_handle lockh;
  81         ENTRY;
  82
  83         LDLM_DEBUG(lock, "We're blocking. Cancelling lock");
  84         ldlm_lock2handle(lock, &lockh);
  85         rc = ldlm_cli_cancel(&lockh);
  86         if (rc < 0) {
  87                 CERROR("ldlm_cli_cancel: %d\n", rc);
  88                 LBUG();
  89         }
  90
  91         RETURN(0);
  92 }
  93
  94 /* blocking ast for basic tests. noop */
  95 static int ldlm_blocking_ast(struct ldlm_lock *lock,
  96                              struct ldlm_lock_desc *new,
  97                              void *data, __u32 data_len)
  98 {
  99         ENTRY;
 100         CERROR("ldlm_blocking_ast: lock=%p, new=%p\n", lock, new);
 101         RETURN(0);
 102 }
 103
 104 /* Completion ast for regression test.
 105  * Does not sleep when blocked.
 106  */
 107 static int ldlm_test_completion_ast(struct ldlm_lock *lock, int flags)
 108 {
 109         struct ldlm_test_lock *lock_info;
 110         ENTRY;
 111
 112         if (flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
 113                       LDLM_FL_BLOCK_CONV)) {
 114
 115                 LDLM_DEBUG(lock, "client-side enqueue returned a blocked lock");
 116                 RETURN(0);
 117         }
 118
 119         if (lock->l_granted_mode != lock->l_req_mode)
 120                 CERROR("completion ast called with non-granted lock\n");
 121
 122         /* add to list of granted locks */
 123
 124         if (flags & LDLM_FL_WAIT_NOREPROC) {
 125                 atomic_inc(&locks_matched);
 126                 LDLM_DEBUG(lock, "lock matched");
 127         } else {
 128                 atomic_inc(&locks_granted);
 129                 LDLM_DEBUG(lock, "lock granted");
 130         }
 131
 132         OBD_ALLOC(lock_info, sizeof(*lock_info));
 133         if (lock_info == NULL) {
 134                 LBUG();
 135                 RETURN(-ENOMEM);
 136         }
 137
 138         ldlm_lock2handle(lock, &lock_info->l_lockh);
 139
 140         spin_lock(&ctl_lock);
 141         list_add_tail(&lock_info->l_link, &lock_list);
 142         num_locks++;
 143         spin_unlock(&ctl_lock);
 144
 145         RETURN(0);
 146 }
 147
 148 int ldlm_test_basics(struct obd_device *obddev)
 149 {
 150         struct ldlm_namespace *ns;
 151         struct ldlm_resource *res;
 152         __u64 res_id[RES_NAME_SIZE] = {1, 2, 3};
 153         ldlm_error_t err;
 154         struct ldlm_lock *lock1, *lock;
 155         int flags;
 156         ENTRY;
 157
 158         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 159         if (ns == NULL)
 160                 LBUG();
 161
 162         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_CR, NULL, 0);
 163         if (lock1 == NULL)
 164                 LBUG();
 165         err = ldlm_lock_enqueue(lock1, NULL, 0, &flags,
 166                                 ldlm_completion_ast, ldlm_blocking_ast);
 167         if (err != ELDLM_OK)
 168                 LBUG();
 169
 170         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_PLAIN, LCK_EX, NULL, 0);
 171         if (lock == NULL)
 172                 LBUG();
 173         err = ldlm_lock_enqueue(lock, NULL, 0, &flags,
 174                                 ldlm_completion_ast, ldlm_blocking_ast);
 175         if (err != ELDLM_OK)
 176                 LBUG();
 177         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 178                 LBUG();
 179
 180         res = ldlm_resource_get(ns, NULL, res_id, LDLM_PLAIN, 1);
 181         if (res == NULL)
 182                 LBUG();
 183         ldlm_resource_dump(res);
 184
 185         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 186         if (res != NULL)
 187                 ldlm_reprocess_all(res);
 188
 189         ldlm_resource_dump(res);
 190         ldlm_namespace_free(ns);
 191
 192         RETURN(0);
 193 }
 194
 195 int ldlm_test_extents(struct obd_device *obddev)
 196 {
 197         struct ldlm_namespace *ns;
 198         struct ldlm_resource *res;
 199         struct ldlm_lock *lock, *lock1, *lock2;
 200         __u64 res_id[RES_NAME_SIZE] = {0, 0, 0};
 201         struct ldlm_extent ext1 = {4, 6}, ext2 = {6, 9}, ext3 = {10, 11};
 202         ldlm_error_t err;
 203         int flags;
 204         ENTRY;
 205
 206         ns = ldlm_namespace_new("test_server", LDLM_NAMESPACE_SERVER);
 207         if (ns == NULL)
 208                 LBUG();
 209
 210         flags = 0;
 211         lock1 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR, NULL,
 212                                  0);
 213         if (lock1 == NULL)
 214                 LBUG();
 215         err = ldlm_lock_enqueue(lock1, &ext1, sizeof(ext1), &flags, NULL, NULL);
 216         if (err != ELDLM_OK)
 217                 LBUG();
 218         if (!(flags & LDLM_FL_LOCK_CHANGED))
 219                 LBUG();
 220
 221         flags = 0;
 222         lock2 = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_PR,
 223                                 NULL, 0);
 224         err = ldlm_lock_enqueue(lock2, &ext2, sizeof(ext2), &flags, NULL, NULL);
 225         if (err != ELDLM_OK)
 226                 LBUG();
 227         if (!(flags & LDLM_FL_LOCK_CHANGED))
 228                 LBUG();
 229
 230         flags = 0;
 231         lock = ldlm_lock_create(ns, NULL, res_id, LDLM_EXTENT, LCK_EX, NULL, 0);
 232         if (lock == NULL)
 233                 LBUG();
 234         err = ldlm_lock_enqueue(lock, &ext3, sizeof(ext3), &flags,
 235                                 NULL, NULL);
 236         if (err != ELDLM_OK)
 237                 LBUG();
 238         if (!(flags & LDLM_FL_BLOCK_GRANTED))
 239                 LBUG();
 240         if (flags & LDLM_FL_LOCK_CHANGED)
 241                 LBUG();
 242
 243         /* Convert/cancel blocking locks */
 244         flags = 0;
 245         res = ldlm_lock_convert(lock1, LCK_NL, &flags);
 246         if (res != NULL)
 247                 ldlm_reprocess_all(res);
 248
 249         ldlm_lock_cancel(lock2);
 250         if (res != NULL)
 251                 ldlm_reprocess_all(res);
 252
 253         /* Dump the results */
 254         res = ldlm_resource_get(ns, NULL, res_id, LDLM_EXTENT, 0);
 255         if (res == NULL)
 256                 LBUG();
 257         ldlm_resource_dump(res);
 258         ldlm_namespace_free(ns);
 259
 260         RETURN(0);
 261 }
 262
 263 static int ldlm_test_network(struct obd_device *obddev,
 264                              struct lustre_handle *connh)
 265 {
 266
 267         __u64 res_id[RES_NAME_SIZE] = {1, 2, 3};
 268         struct ldlm_extent ext = {4, 6};
 269         struct lustre_handle lockh1;
 270         struct ldlm_lock *lock;
 271         int flags = 0;
 272         ldlm_error_t err;
 273         ENTRY;
 274
 275         err = ldlm_cli_enqueue(connh, NULL, obddev->obd_namespace, NULL, res_id,
 276                                LDLM_EXTENT, &ext, sizeof(ext), LCK_PR, &flags,
 277                                ldlm_completion_ast, NULL, NULL, 0, &lockh1);
 278
 279         CERROR("ldlm_cli_enqueue: %d\n", err);
 280
 281         flags = 0;
 282         err = ldlm_cli_convert(&lockh1, LCK_EX, &flags);
 283         CERROR("ldlm_cli_convert: %d\n", err);
 284
 285         lock = ldlm_handle2lock(&lockh1);
 286         ldlm_lock_dump(lock);
 287         ldlm_lock_put(lock);
 288
 289         /* Need to decrement old mode. Don't bother incrementing new
 290          * mode since the test is done.
 291          */
 292         if (err == ELDLM_OK)
 293                 ldlm_lock_decref(&lockh1, LCK_PR);
 294
 295         RETURN(err);
 296 }
 297
 298 static int ldlm_do_decrement(void)
 299 {
 300         struct ldlm_test_lock *lock_info;
 301         struct ldlm_lock *lock;
 302         int rc = 0;
 303         ENTRY;
 304
 305         spin_lock(&ctl_lock);
 306         if(list_empty(&lock_list)) {
 307                 CERROR("lock_list is empty\n");
 308                 spin_unlock(&ctl_lock);
 309                 RETURN(0);
 310         }
 311
 312         /* delete from list */
 313         lock_info = list_entry(lock_list.next,
 314                         struct ldlm_test_lock, l_link);
 315         list_del(lock_list.next);
 316         num_locks--;
 317         spin_unlock(&ctl_lock);
 318
 319         /* decrement and free the info */
 320         lock = ldlm_handle2lock(&lock_info->l_lockh);
 321         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 322         ldlm_lock_put(lock);
 323
 324         OBD_FREE(lock_info, sizeof(*lock_info));
 325
 326         RETURN(rc);
 327 }
 328
 329 static int ldlm_do_enqueue(struct ldlm_test_thread *thread)
 330 {
 331         struct lustre_handle lockh;
 332         __u64 res_id[3] = {0};
 333         __u32 lock_mode;
 334         struct ldlm_extent ext;
 335         unsigned char random;
 336         int flags = 0, rc = 0;
 337         ENTRY;
 338
 339         /* Pick a random resource from 1 to num_resources */
 340         get_random_bytes(&random, sizeof(random));
 341         res_id[0] = random % num_resources;
 342
 343         /* Pick a random lock mode */
 344         get_random_bytes(&random, sizeof(random));
 345         lock_mode = random % LCK_NL + 1;
 346
 347         /* Pick a random extent */
 348         get_random_bytes(&random, sizeof(random));
 349         ext.start = random % num_extents;
 350         get_random_bytes(&random, sizeof(random));
 351         ext.end = random %
 352                 (num_extents - (int)ext.start) + ext.start;
 353
 354         LDLM_DEBUG_NOLOCK("about to enqueue with resource %d, mode %d,"
 355                           " extent %d -> %d",
 356                           (int)res_id[0],
 357                           lock_mode,
 358                           (int)ext.start,
 359                           (int)ext.end);
 360
 361         rc = ldlm_match_or_enqueue(&regress_connh,
 362                                    NULL,
 363                                    thread->obddev->obd_namespace,
 364                                    NULL, res_id, LDLM_EXTENT, &ext,
 365                                    sizeof(ext), lock_mode, &flags,
 366                                    ldlm_test_completion_ast,
 367                                    ldlm_test_blocking_ast,
 368                                    NULL, 0, &lockh);
 369
 370         atomic_inc(&locks_requested);
 371
 372         if (rc < 0) {
 373                 CERROR("ldlm_cli_enqueue: %d\n", rc);
 374                 LBUG();
 375         }
 376
 377         RETURN(rc);
 378 }
 379
 380 static int ldlm_do_convert(void)
 381 {
 382         __u32 lock_mode;
 383         unsigned char random;
 384         int flags = 0, rc = 0;
 385         struct ldlm_test_lock *lock_info;
 386         struct ldlm_lock *lock;
 387         ENTRY;
 388
 389         /* delete from list */
 390         spin_lock(&ctl_lock);
 391         lock_info = list_entry(lock_list.next, struct ldlm_test_lock, l_link);
 392         list_del(lock_list.next);
 393         num_locks--;
 394         spin_unlock(&ctl_lock);
 395
 396         /* Pick a random lock mode */
 397         get_random_bytes(&random, sizeof(random));
 398         lock_mode = random % LCK_NL + 1;
 399
 400         /* do the conversion */
 401         rc = ldlm_cli_convert(&lock_info->l_lockh , lock_mode, &flags);
 402         atomic_inc(&converts_requested);
 403
 404         if (rc < 0) {
 405                 CERROR("ldlm_cli_convert: %d\n", rc);
 406                 LBUG();
 407         }
 408
 409         /*
 410          *  Adjust reference counts.
 411          *  FIXME: This is technically a bit... wrong,
 412          *  since we don't know when/if the convert succeeded
 413          */
 414         ldlm_lock_addref(&lock_info->l_lockh, lock_mode);
 415         lock = ldlm_handle2lock(&lock_info->l_lockh);
 416         ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 417         ldlm_lock_put(lock);
 418
 419         OBD_FREE(lock_info, sizeof(*lock_info));
 420
 421         RETURN(rc);
 422 }
 423
 424
 425
 426 static int ldlm_test_main(void *data)
 427 {
 428         struct ldlm_test_thread *thread = data;
 429         ENTRY;
 430
 431         lock_kernel();
 432         daemonize();
 433         spin_lock_irq(&current->sigmask_lock);
 434         sigfillset(&current->blocked);
 435         recalc_sigpending(current);
 436         spin_unlock_irq(&current->sigmask_lock);
 437
 438         sprintf(current->comm, "ldlm_test");
 439         spin_unlock();
 440
 441         /* Record that the thread is running */
 442         thread->t_flags |= SVC_RUNNING;
 443         wake_up(&thread->t_ctl_waitq);
 444
 445         while (!(thread->t_flags & SVC_STOPPING)) {
 446                 unsigned char random;
 447                 unsigned char dec_chance, con_chance;
 448                 unsigned char chance_left = 100;
 449
 450                 spin_lock(&ctl_lock);
 451                 /* probability of decrementing increases linearly
 452                  * as more locks are held.
 453                  */
 454                 dec_chance = chance_left * num_locks / max_locks;
 455                 chance_left -= dec_chance;
 456
 457                 /* FIXME: conversions temporarily disabled
 458                  * until they are working correctly.
 459                  */
 460                 /* con_chance = chance_left * num_locks / max_locks; */
 461                 con_chance = 0;
 462                 chance_left -= con_chance;
 463                 spin_unlock(&ctl_lock);
 464
 465                 get_random_bytes(&random, sizeof(random));
 466
 467                 random = random % 100;
 468                 if (random < dec_chance)
 469                         ldlm_do_decrement();
 470                 else if (random < (dec_chance + con_chance))
 471                         ldlm_do_convert();
 472                 else
 473                         ldlm_do_enqueue(thread);
 474
 475                 LDLM_DEBUG_NOLOCK("locks requested: %d, "
 476                                   "conversions requested %d",
 477                                   atomic_read(&locks_requested),
 478                                   atomic_read(&converts_requested));
 479                 LDLM_DEBUG_NOLOCK("locks granted: %d, "
 480                                   "locks matched: %d",
 481                                   atomic_read(&locks_granted),
 482                                   atomic_read(&locks_matched));
 483
 484                 spin_lock(&ctl_lock);
 485                 LDLM_DEBUG_NOLOCK("lock references currently held: %d, ",
 486                                   num_locks);
 487                 spin_unlock(&ctl_lock);
 488
 489                 /*
 490                  * We don't sleep after a lock being blocked, so let's
 491                  * make sure other things can run.
 492                  */
 493                 schedule();
 494         }
 495
 496         thread->t_flags |= SVC_STOPPED;
 497         wake_up(&thread->t_ctl_waitq);
 498
 499         RETURN(0);
 500 }
 501
 502 static int ldlm_start_thread(struct obd_device *obddev,
 503                              struct lustre_handle *connh)
 504 {
 505         struct ldlm_test_thread *test;
 506         int rc;
 507         ENTRY;
 508
 509         OBD_ALLOC(test, sizeof(*test));
 510         if (test == NULL) {
 511                 LBUG();
 512                 RETURN(-ENOMEM);
 513         }
 514         init_waitqueue_head(&test->t_ctl_waitq);
 515
 516         test->obddev = obddev;
 517
 518         spin_lock(&ctl_lock);
 519         list_add(&test->t_link, &ctl_threads);
 520         spin_unlock(&ctl_lock);
 521
 522         rc = kernel_thread(ldlm_test_main, (void *)test,
 523                            CLONE_VM | CLONE_FS | CLONE_FILES);
 524         if (rc < 0) {
 525                 CERROR("cannot start thread\n");
 526                 RETURN(-EINVAL);
 527         }
 528         wait_event(test->t_ctl_waitq, test->t_flags & SVC_RUNNING);
 529
 530         RETURN(0);
 531 }
 532
 533 int ldlm_regression_start(struct obd_device *obddev,
 534                           struct lustre_handle *connh,
 535                           unsigned int threads, unsigned int max_locks_in,
 536                           unsigned int num_resources_in,
 537                           unsigned int num_extents_in)
 538 {
 539         int i, rc = 0;
 540         ENTRY;
 541
 542         spin_lock(&ctl_lock);
 543         if (regression_running) {
 544                 CERROR("You can't start the ldlm regression twice.\n");
 545                 spin_unlock(&ctl_lock);
 546                 RETURN(-EINVAL);
 547         }
 548         regression_running = 1;
 549         spin_unlock(&ctl_lock);
 550
 551         regress_connh = *connh;
 552         max_locks = max_locks_in;
 553         num_resources = num_resources_in;
 554         num_extents = num_extents_in;
 555
 556         LDLM_DEBUG_NOLOCK("regression test started: threads: %d, max_locks: "
 557                           "%d, num_res: %d, num_ext: %d\n",
 558                           threads, max_locks_in, num_resources_in,
 559                           num_extents_in);
 560
 561         for (i = 0; i < threads; i++) {
 562                 rc = ldlm_start_thread(obddev, connh);
 563                 if (rc < 0)
 564                         GOTO(cleanup, rc);
 565         }
 566
 567  cleanup:
 568         if (rc < 0)
 569                 ldlm_regression_stop();
 570         RETURN(rc);
 571 }
 572
 573 int ldlm_regression_stop(void)
 574 {
 575         ENTRY;
 576
 577         spin_lock(&ctl_lock);
 578         if (!regression_running) {
 579                 CERROR("The ldlm regression isn't started.\n");
 580                 spin_unlock(&ctl_lock);
 581                 RETURN(-EINVAL);
 582         }
 583
 584         while (!list_empty(&ctl_threads)) {
 585                 struct ldlm_test_thread *thread;
 586                 thread = list_entry(ctl_threads.next, struct ldlm_test_thread,
 587                                     t_link);
 588
 589                 thread->t_flags |= SVC_STOPPING;
 590
 591                 spin_unlock(&ctl_lock);
 592                 wake_up(&thread->t_ctl_waitq);
 593                 wait_event(thread->t_ctl_waitq, thread->t_flags & SVC_STOPPED);
 594                 spin_lock(&ctl_lock);
 595
 596                 list_del(&thread->t_link);
 597                 OBD_FREE(thread, sizeof(*thread));
 598         }
 599
 600         /* decrement all held locks */
 601         while (!list_empty(&lock_list)) {
 602                 struct ldlm_lock *lock;
 603                 struct ldlm_test_lock *lock_info =
 604                        list_entry(lock_list.next, struct ldlm_test_lock,
 605                                    l_link);
 606                 list_del(lock_list.next);
 607                 num_locks--;
 608
 609                 lock = ldlm_handle2lock(&lock_info->l_lockh);
 610                 ldlm_lock_decref(&lock_info->l_lockh, lock->l_granted_mode);
 611                 ldlm_lock_put(lock);
 612
 613                 OBD_FREE(lock_info, sizeof(*lock_info));
 614         }
 615
 616         regression_running = 0;
 617         spin_unlock(&ctl_lock);
 618
 619         RETURN(0);
 620 }
 621
 622 int ldlm_test(struct obd_device *obddev, struct lustre_handle *connh)
 623 {
 624         int rc;
 625         rc = ldlm_test_basics(obddev);
 626         if (rc)
 627                 RETURN(rc);
 628
 629         rc = ldlm_test_extents(obddev);
 630         if (rc)
 631                 RETURN(rc);
 632
 633         rc = ldlm_test_network(obddev, connh);
 634         RETURN(rc);
 635 }