Whamcloud - gitweb
* Included changes from *old* b1_4/b_cray
authoreeb <eeb>
Fri, 7 Jan 2005 12:41:05 +0000 (12:41 +0000)
committereeb <eeb>
Fri, 7 Jan 2005 12:41:05 +0000 (12:41 +0000)
    *  OBD_FL_DEBUG_CHECK for consistent obdecho end-to-end checking

    *  struct echo_obd cleanup

    *  obdecho support for "difficult" reply testing (test_setattr)

    *  replaced cowardly eap_from_cookie() usage with EAP_FROM_COOKIE()
       which asserts it is being called correctly

    *  bracketed echo_client_iocontrol() guts with unlock/lock_kernel()
       for increased concurrency during echo ioctl execution

    *  better thread control and stats in obd.c

    *  n-threads-over-m-objects test_brw support (required for
       obdfilter-survey)

* openibnal changes

    *  re-enabled openib on 2.5/2.6 builds
       configure --with-openib=no can be used to disable openib explicitly if
       required.

    *  fixed some 64 bit casts

    *  just use kernel's page_to_phys()

* fixed CERROR format problem in libcfs/module.c

lustre/include/linux/lustre_idl.h
lustre/include/linux/obd.h
lustre/obdecho/echo.c
lustre/obdecho/echo_client.c
lustre/utils/lctl.c
lustre/utils/obd.c
lustre/utils/obdctl.h

index 55921d4..6ce059b 100644 (file)
@@ -245,6 +245,7 @@ typedef uint32_t        obd_count;
 #define OBD_FL_NORPC        (0x00000008) // if set in o_flags set in OSC not OST
 #define OBD_FL_IDONLY       (0x00000010) // if set in o_flags only adjust obj id
 #define OBD_FL_RECREATE_OBJS (0x00000020) // recreate missing obj
+#define OBD_FL_DEBUG_CHECK  (0x00000040) /* echo client/server debug check */
 
 #define OBD_INLINESZ    64
 
index b9fb45b..04ba628 100644 (file)
@@ -320,16 +320,11 @@ struct mds_obd {
 };
 
 struct echo_obd {
-        struct obdo oa;
-        spinlock_t eo_lock;
-        __u64 eo_lastino;
-        atomic_t eo_getattr;
-        atomic_t eo_setattr;
-        atomic_t eo_create;
-        atomic_t eo_destroy;
-        atomic_t eo_prep;
-        atomic_t eo_read;
-        atomic_t eo_write;
+        struct obdo          eo_oa;
+        spinlock_t           eo_lock;
+        __u64                eo_lastino;
+        struct lustre_handle eo_nl_lock;
+        atomic_t             eo_prep;
 };
 
 /*
index e9be311..3f12f12 100644 (file)
@@ -76,6 +76,21 @@ static int echo_disconnect(struct obd_export *exp, int flags)
         exp->exp_flags = flags;
         spin_unlock_irqrestore(&exp->exp_lock, irqflags);
 
+        /* complete all outstanding replies */
+        spin_lock_irqsave(&exp->exp_lock, irqflags);
+        while (!list_empty(&exp->exp_outstanding_replies)) {
+                struct ptlrpc_reply_state *rs =
+                        list_entry(exp->exp_outstanding_replies.next,
+                                   struct ptlrpc_reply_state, rs_exp_list);
+                struct ptlrpc_service *svc = rs->rs_srv_ni->sni_service;
+
+                spin_lock(&svc->srv_lock);
+                list_del_init(&rs->rs_exp_list);
+                ptlrpc_schedule_difficult_reply(rs);
+                spin_unlock(&svc->srv_lock);
+        }
+        spin_unlock_irqrestore(&exp->exp_lock, irqflags);
+
         return class_disconnect(exp, flags);
 }
 
@@ -122,7 +137,6 @@ int echo_create(struct obd_export *exp, struct obdo *oa,
 
         oa->o_id = echo_next_id(obd);
         oa->o_valid = OBD_MD_FLID;
-        atomic_inc(&obd->u.echo.eo_create);
 
         return 0;
 }
@@ -148,8 +162,6 @@ int echo_destroy(struct obd_export *exp, struct obdo *oa,
                 RETURN(-EINVAL);
         }
 
-        atomic_inc(&obd->u.echo.eo_destroy);
-
         return 0;
 }
 
@@ -170,7 +182,7 @@ static int echo_getattr(struct obd_export *exp, struct obdo *oa,
                 RETURN(-EINVAL);
         }
 
-        obdo_cpy_md(oa, &obd->u.echo.oa, oa->o_valid);
+        obdo_cpy_md(oa, &obd->u.echo.eo_oa, oa->o_valid);
         oa->o_id = id;
 
         return 0;
@@ -192,9 +204,14 @@ static int echo_setattr(struct obd_export *exp, struct obdo *oa,
                 RETURN(-EINVAL);
         }
 
-        memcpy(&obd->u.echo.oa, oa, sizeof(*oa));
+        memcpy(&obd->u.echo.eo_oa, oa, sizeof(*oa));
 
-        atomic_inc(&obd->u.echo.eo_setattr);
+        if (oa->o_id & 4) {
+                /* Save lock to force ACKed reply */
+                ldlm_lock_addref (&obd->u.echo.eo_nl_lock, LCK_NL);
+                oti->oti_ack_locks[0].mode = LCK_NL;
+                oti->oti_ack_locks[0].lock = obd->u.echo.eo_nl_lock;
+        }
 
         return 0;
 }
@@ -288,6 +305,9 @@ int echo_preprw(int cmd, struct obd_export *export, struct obdo *oa,
         for (i = 0; i < objcount; i++, obj++) {
                 int gfp_mask = (obj->ioo_id & 1) ? GFP_HIGHUSER : GFP_KERNEL;
                 int ispersistent = obj->ioo_id == ECHO_PERSISTENT_OBJID;
+                int debug_setup = (!ispersistent &&
+                                   (oa->o_valid & OBD_MD_FLFLAGS) != 0 &&
+                                   (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0);
                 int j;
 
                 for (j = 0 ; j < obj->ioo_bufcnt ; j++, nb++, r++) {
@@ -322,7 +342,7 @@ int echo_preprw(int cmd, struct obd_export *export, struct obdo *oa,
                         if (cmd & OBD_BRW_READ)
                                 r->rc = r->len;
 
-                        if (!ispersistent)
+                        if (debug_setup)
                                 echo_page_debug_setup(r->page, cmd, obj->ioo_id,
                                                       r->offset, r->len);
                 }
@@ -391,7 +411,9 @@ int echo_commitrw(int cmd, struct obd_export *export, struct obdo *oa,
 
         for (i = 0; i < objcount; i++, obj++) {
                 int verify = (rc == 0 &&
-                              obj->ioo_id != ECHO_PERSISTENT_OBJID);
+                              obj->ioo_id != ECHO_PERSISTENT_OBJID &&
+                              (oa->o_valid & OBD_MD_FLFLAGS) != 0 &&
+                              (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0);
                 int j;
 
                 for (j = 0 ; j < obj->ioo_bufcnt ; j++, r++) {
@@ -443,6 +465,9 @@ commitrw_cleanup:
 static int echo_setup(struct obd_device *obd, obd_count len, void *buf)
 {
         struct lprocfs_static_vars lvars;
+        int                        rc;
+        int                        lock_flags = 0;
+        struct ldlm_res_id         res_id = {.name = {1}};
         ENTRY;
 
         spin_lock_init(&obd->u.echo.eo_lock);
@@ -455,6 +480,12 @@ static int echo_setup(struct obd_device *obd, obd_count len, void *buf)
                 RETURN(-ENOMEM);
         }
 
+        rc = ldlm_cli_enqueue(NULL, NULL, obd->obd_namespace, res_id,
+                              LDLM_PLAIN, NULL, LCK_NL, &lock_flags,
+                              NULL, ldlm_completion_ast, NULL, NULL,
+                              NULL, 0, NULL, &obd->u.echo.eo_nl_lock);
+        LASSERT (rc == ELDLM_OK);
+
         lprocfs_init_vars(echo, &lvars);
         if (lprocfs_obd_setup(obd, lvars.obd_vars) == 0 &&
             lprocfs_alloc_obd_stats(obd, LPROC_ECHO_LAST) == 0) {
@@ -479,6 +510,13 @@ static int echo_cleanup(struct obd_device *obd, int flags)
         lprocfs_free_obd_stats(obd);
         lprocfs_obd_cleanup(obd);
 
+        ldlm_lock_decref (&obd->u.echo.eo_nl_lock, LCK_NL);
+
+        /* XXX Bug 3413; wait for a bit to ensure the BL callback has
+         * happened before calling ldlm_namespace_free() */
+        set_current_state (TASK_UNINTERRUPTIBLE);
+        schedule_timeout (HZ);
+        
         ldlm_namespace_free(obd->obd_namespace, flags & OBD_OPT_FORCE);
 
         leaked = atomic_read(&obd->u.echo.eo_prep);
index 67935cb..58d2358 100644 (file)
@@ -29,6 +29,7 @@
 #include <linux/iobuf.h>
 #endif
 #include <asm/div64.h>
+#include <linux/smp_lock.h>
 #else
 #include <liblustre.h>
 #endif
@@ -499,13 +500,13 @@ static int echo_client_kbrw(struct obd_device *obd, int rw, struct obdo *oa,
         obd_off                 off;
         int                     i;
         int                     rc;
-        int                     verify = 0;
+        int                     verify;
         int                     gfp_mask;
 
-        /* oa_id == ECHO_PERSISTENT_OBJID => speed test (no verification).
-         * oa & 1                         => use HIGHMEM */
+        verify = ((oa->o_id) != ECHO_PERSISTENT_OBJID &&
+                  (oa->o_valid & OBD_MD_FLFLAGS) != 0 &&
+                  (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0);
 
-        verify = (oa->o_id) != ECHO_PERSISTENT_OBJID;
         gfp_mask = ((oa->o_id & 2) == 0) ? GFP_KERNEL : GFP_HIGHUSER;
 
         LASSERT(rw == OBD_BRW_WRITE || rw == OBD_BRW_READ);
@@ -658,6 +659,10 @@ struct echo_async_page {
         struct list_head        eap_item;
 };
 
+#define EAP_FROM_COOKIE(c)                                                      \
+        (LASSERT(((struct echo_async_page *)(c))->eap_magic == EAP_MAGIC),      \
+         (struct echo_async_page *)(c))
+
 struct echo_async_state {
         spinlock_t              eas_lock;
         obd_off                 eas_next_offset;
@@ -681,14 +686,6 @@ static int eas_should_wake(struct echo_async_state *eas)
         return rc;
 };
 
-struct echo_async_page *eap_from_cookie(void *cookie)
-{
-        struct echo_async_page *eap = cookie;
-        if (eap->eap_magic != EAP_MAGIC)
-                return ERR_PTR(-EINVAL);
-        return eap;
-};
-
 static int ec_ap_make_ready(void *data, int cmd)
 {
         /* our pages are issued ready */
@@ -703,26 +700,23 @@ static int ec_ap_refresh_count(void *data, int cmd)
 }
 static void ec_ap_fill_obdo(void *data, int cmd, struct obdo *oa)
 {
-        struct echo_async_page *eap;
-        eap = eap_from_cookie(data);
-        if (IS_ERR(eap))
-                return;
+        struct echo_async_page *eap = EAP_FROM_COOKIE(data);
 
         memcpy(oa, &eap->eap_eas->eas_oa, sizeof(*oa));
 }
 
 static void ec_ap_completion(void *data, int cmd, struct obdo *oa, int rc)
 {
-        struct echo_async_page *eap = eap_from_cookie(data);
+        struct echo_async_page *eap = EAP_FROM_COOKIE(data);
         struct echo_async_state *eas;
         unsigned long flags;
 
-        if (IS_ERR(eap))
-                return;
         eas = eap->eap_eas;
 
         if (cmd == OBD_BRW_READ &&
-            eas->eas_oa.o_id != ECHO_PERSISTENT_OBJID)
+            eas->eas_oa.o_id != ECHO_PERSISTENT_OBJID &&
+            (eas->eas_oa.o_valid & OBD_MD_FLFLAGS) != 0 &&
+            (eas->eas_oa.o_flags & OBD_FL_DEBUG_CHECK) != 0)
                 echo_client_page_debug_check(eas->eas_lsm, eap->eap_page, 
                                              eas->eas_oa.o_id, eap->eap_off,
                                              PAGE_SIZE);
@@ -758,11 +752,12 @@ static int echo_client_async_page(struct obd_export *exp, int rw,
 #if 0
         int                     verify;
         int                     gfp_mask;
-        /* oa_id  == 0    => speed test (no verification) else...
-         * oa & 1         => use HIGHMEM
-         */
-        verify = (oa->o_id != 0);
-        gfp_mask = ((oa->o_id & 1) == 0) ? GFP_KERNEL : GFP_HIGHUSER;
+
+        verify = ((oa->o_id) != ECHO_PERSISTENT_OBJID &&
+                  (oa->o_valid & OBD_MD_FLFLAGS) != 0 &&
+                  (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0);
+
+        gfp_mask = ((oa->o_id & 2) == 0) ? GFP_KERNEL : GFP_HIGHUSER;
 #endif
 
         LASSERT(rw == OBD_BRW_WRITE || rw == OBD_BRW_READ);
@@ -847,7 +842,9 @@ static int echo_client_async_page(struct obd_export *exp, int rw,
                         break;
                 }
 
-                if (oa->o_id != ECHO_PERSISTENT_OBJID)
+                if (oa->o_id != ECHO_PERSISTENT_OBJID &&
+                    (oa->o_valid & OBD_MD_FLFLAGS) != 0 &&
+                    (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0)
                         echo_client_page_debug_setup(lsm, eap->eap_page, rw, 
                                                      oa->o_id, 
                                                      eap->eap_off, PAGE_SIZE);
@@ -951,7 +948,9 @@ static int echo_client_prep_commit(struct obd_export *exp, int rw,
                         if (page == NULL && lnb[i].rc == 0)
                                 continue;
 
-                        if (oa->o_id == ECHO_PERSISTENT_OBJID)
+                        if (oa->o_id == ECHO_PERSISTENT_OBJID ||
+                            (oa->o_valid & OBD_MD_FLFLAGS) == 0 ||
+                            (oa->o_flags & OBD_FL_DEBUG_CHECK) == 0)
                                 continue;
 
                         if (rw == OBD_BRW_WRITE)
@@ -1194,6 +1193,8 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp,
         int                     i;
         ENTRY;
 
+        unlock_kernel();
+
         memset(&dummy_oti, 0, sizeof(dummy_oti));
 
         obd = exp->exp_obd;
@@ -1312,6 +1313,8 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp,
                 ldlm_lock_decref(&ack_lock->lock, ack_lock->mode);
         }
 
+        lock_kernel();
+
         return rc;
 }
 
index 197b95e..bb41dd5 100644 (file)
@@ -193,6 +193,9 @@ command_t cmdlist[] = {
         {"test_getattr", jt_obd_test_getattr, 0,
          "do <num> getattrs (on OST object <objid> (objid+1 on each thread))\n"
          "usage: test_getattr <num> [verbose [[t]objid]]"},
+        {"test_setattr", jt_obd_test_setattr, 0,
+         "do <num> setattrs (on OST object <objid> (objid+1 on each thread))\n"
+         "usage: test_setattr <num> [verbose [[t]objid]]"},
         {"test_brw", jt_obd_test_brw, 0,
          "do <num> bulk read/writes (<npages> per I/O, on OST object <objid>)\n"
          "usage: test_brw [t]<num> [write [verbose [npages [[t]objid]]]]"},
index 95b9917..e9e1c04 100644 (file)
 #include <portals/ptlctl.h>
 #include "parser.h"
 #include <stdio.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <pthread.h>
 
-#define SHMEM_STATS 1
 #define MAX_STRING_SIZE 128
 #define DEVICES_LIST "/proc/fs/lustre/devices"
 
-#if SHMEM_STATS
-# include <sys/ipc.h>
-# include <sys/shm.h>
-
-# define MAX_SHMEM_COUNT 1024
-static long long *shared_counters;
-static long long counter_snapshot[2][MAX_SHMEM_COUNT];
+#define MAX_THREADS 1024
+struct shared_data {
+        __u64 counters[MAX_THREADS];
+        __u64 offsets[MAX_THREADS];
+        int   running;
+        int   barrier;
+        pthread_mutex_t mutex;
+        pthread_cond_t  cond;
+};
+static struct shared_data *shared_data;
+static __u64 counter_snapshot[2][MAX_THREADS];
+static int prev_valid;
 struct timeval prev_time;
-#endif
 
 static int jt_recording;
 static char rawbuf[8192];
@@ -79,6 +85,7 @@ static char *buf = rawbuf;
 static int max = sizeof(rawbuf);
 
 static int thread;
+static int nthreads;
 
 static uint32_t cur_device = MAX_OBD_DEVICES;
 
@@ -228,7 +235,7 @@ int parse_devname(char *func, char *name)
                 /* Assume it's a number.  This means that bogus strings become
                  * 0.  I might care about that some day. */
                 ret = strtoul(name, NULL, 0);
-                printf("Selected device %d\n", ret);
+                // printf("Selected device %d\n", ret);
         }
 
         return ret;
@@ -426,25 +433,24 @@ int do_disconnect(char *func, int verbose)
         return 0;
 }
 
-#if SHMEM_STATS
 static void shmem_setup(void)
 {
         /* Create new segment */
-        int shmid = shmget(IPC_PRIVATE, sizeof(counter_snapshot[0]), 0600);
+        int shmid = shmget(IPC_PRIVATE, sizeof(*shared_data), 0600);
 
         if (shmid == -1) {
-                fprintf(stderr, "Can't create shared memory counters: %s\n",
+                fprintf(stderr, "Can't create shared data: %s\n",
                         strerror(errno));
                 return;
         }
 
         /* Attatch to new segment */
-        shared_counters = (long long *)shmat(shmid, NULL, 0);
+        shared_data = (struct shared_data *)shmat(shmid, NULL, 0);
 
-        if (shared_counters == (long long *)(-1)) {
-                fprintf(stderr, "Can't attach shared memory counters: %s\n",
+        if (shared_data == (struct shared_data *)(-1)) {
+                fprintf(stderr, "Can't attach shared data: %s\n",
                         strerror(errno));
-                shared_counters = NULL;
+                shared_data = NULL;
                 return;
         }
 
@@ -452,47 +458,60 @@ static void shmem_setup(void)
          * Forks will inherit attached segments, so we should be OK.
          */
         if (shmctl(shmid, IPC_RMID, NULL) == -1) {
-                fprintf(stderr, "Can't destroy shared memory counters: %s\n",
+                fprintf(stderr, "Can't destroy shared data: %s\n",
                         strerror(errno));
         }
 }
 
-static inline void shmem_reset(void)
+static inline void shmem_reset(int total_threads)
 {
-        if (shared_counters == NULL)
+        if (shared_data == NULL)
                 return;
 
-        memset(shared_counters, 0, sizeof(counter_snapshot[0]));
+        memset(shared_data, 0, sizeof(*shared_data));
+        pthread_mutex_init(&shared_data->mutex, NULL);
+        pthread_cond_init(&shared_data->cond, NULL);
         memset(counter_snapshot, 0, sizeof(counter_snapshot));
-        gettimeofday(&prev_time, NULL);
+        prev_valid = 0;
+        shared_data->barrier = total_threads;
 }
 
 static inline void shmem_bump(void)
 {
-        if (shared_counters == NULL || thread <= 0 || thread > MAX_SHMEM_COUNT)
+        static int bumped_running;
+
+        if (shared_data == NULL || thread <= 0 || thread > MAX_THREADS)
                 return;
 
-        shared_counters[thread - 1]++;
+        pthread_mutex_lock(&shared_data->mutex);
+        shared_data->counters[thread - 1]++;
+        if (!bumped_running)
+                shared_data->running++;
+        pthread_mutex_unlock(&shared_data->mutex);
+        bumped_running = 1;
 }
 
-static void shmem_snap(int n)
+static void shmem_snap(int total_threads, int live_threads)
 {
         struct timeval this_time;
         int non_zero = 0;
-        long long total = 0;
+        __u64 total = 0;
         double secs;
+        int running;
         int i;
 
-        if (shared_counters == NULL || n > MAX_SHMEM_COUNT)
+        if (shared_data == NULL || total_threads > MAX_THREADS)
                 return;
 
-        memcpy(counter_snapshot[1], counter_snapshot[0],
-               n * sizeof(counter_snapshot[0][0]));
-        memcpy(counter_snapshot[0], shared_counters,
-               n * sizeof(counter_snapshot[0][0]));
+        pthread_mutex_lock(&shared_data->mutex);
+        memcpy(counter_snapshot[0], shared_data->counters,
+               total_threads * sizeof(counter_snapshot[0][0]));
+        running = shared_data->running;
+        pthread_mutex_unlock(&shared_data->mutex);
+
         gettimeofday(&this_time, NULL);
 
-        for (i = 0; i < n; i++) {
+        for (i = 0; i < total_threads; i++) {
                 long long this_count =
                         counter_snapshot[0][i] - counter_snapshot[1][i];
 
@@ -503,24 +522,21 @@ static void shmem_snap(int n)
         }
 
         secs = (this_time.tv_sec + this_time.tv_usec / 1000000.0) -
-                (prev_time.tv_sec + prev_time.tv_usec / 1000000.0);
+               (prev_time.tv_sec + prev_time.tv_usec / 1000000.0);
 
-        printf("%d/%d Total: %f/second\n", non_zero, n, total / secs);
+        if (prev_valid && 
+            live_threads == total_threads &&
+            secs > 0.0)                    /* someone screwed with the time? */
+                printf("%d/%d Total: %f/second\n", non_zero, total_threads, total / secs);
 
+        memcpy(counter_snapshot[1], counter_snapshot[0],
+               total_threads * sizeof(counter_snapshot[0][0]));
         prev_time = this_time;
+        if (!prev_valid && 
+            running == total_threads)
+                prev_valid = 1;
 }
 
-#define SHMEM_SETUP()   shmem_setup()
-#define SHMEM_RESET()   shmem_reset()
-#define SHMEM_BUMP()    shmem_bump()
-#define SHMEM_SNAP(n)   shmem_snap(n)
-#else
-#define SHMEM_SETUP()
-#define SHMEM_RESET()
-#define SHMEM_BUMP()
-#define SHMEM_SNAP(n)
-#endif
-
 extern command_t cmdlist[];
 
 static int do_device(char *func, char *devname)
@@ -583,8 +599,18 @@ int jt_opt_device(int argc, char **argv)
         return rc;
 }
 
+static void parent_sighandler (int sig)
+{
+        return;
+}
+
 int jt_opt_threads(int argc, char **argv)
 {
+        sigset_t         saveset;
+        sigset_t         sigset;
+        struct sigaction sigact;
+        struct sigaction saveact1;
+        struct sigaction saveact2;
         __u64 threads, next_thread;
         int verbose;
         int rc = 0;
@@ -595,8 +621,8 @@ int jt_opt_threads(int argc, char **argv)
                 return CMD_HELP;
 
         threads = strtoull(argv[1], &end, 0);
-        if (*end) {
-                fprintf(stderr, "error: %s: invalid page count '%s'\n",
+        if (*end || threads > MAX_THREADS) {
+                fprintf(stderr, "error: %s: invalid thread count '%s'\n",
                         jt_cmdname(argv[0]), argv[1]);
                 return CMD_HELP;
         }
@@ -609,7 +635,14 @@ int jt_opt_threads(int argc, char **argv)
                 printf("%s: starting "LPD64" threads on device %s running %s\n",
                        argv[0], threads, argv[3], argv[4]);
 
-        SHMEM_RESET();
+        shmem_reset(threads);
+
+        sigemptyset(&sigset);
+        sigaddset(&sigset, SIGALRM);
+        sigaddset(&sigset, SIGCHLD);
+        sigprocmask(SIG_BLOCK, &sigset, &saveset);
+
+        nthreads = threads;
 
         for (i = 1, next_thread = verbose; i <= threads; i++) {
                 rc = fork();
@@ -618,6 +651,8 @@ int jt_opt_threads(int argc, char **argv)
                                 strerror(rc = errno));
                         break;
                 } else if (rc == 0) {
+                        sigprocmask(SIG_SETMASK, &saveset, NULL);
+
                         thread = i;
                         argv[2] = "--device";
                         return jt_opt_device(argc - 2, argv + 2);
@@ -630,45 +665,64 @@ int jt_opt_threads(int argc, char **argv)
         if (!thread) {          /* parent process */
                 int live_threads = threads;
 
+                sigemptyset(&sigset);
+                sigemptyset(&sigact.sa_mask);
+                sigact.sa_handler = parent_sighandler;
+                sigact.sa_flags = 0;
+                
+                sigaction(SIGALRM, &sigact, &saveact1);
+                sigaction(SIGCHLD, &sigact, &saveact2);
+
                 while (live_threads > 0) {
                         int status;
                         pid_t ret;
 
-                        ret = waitpid(0, &status, verbose < 0 ? WNOHANG : 0);
-                        if (ret == 0) {
-                                if (verbose >= 0)
-                                        abort();
-
-                                sleep(-verbose);
-                                SHMEM_SNAP(threads);
-                                continue;
+                        if (verbose < 0)        /* periodic stats */
+                                alarm(-verbose);
+
+                        sigsuspend(&sigset);
+                        alarm(0);
+
+                        while (live_threads > 0) {
+                                ret = waitpid(0, &status, WNOHANG);
+                                if (ret == 0)
+                                        break;
+                                
+                                if (ret < 0) {
+                                        fprintf(stderr, "error: %s: wait - %s\n",
+                                                argv[0], strerror(errno));
+                                        if (!rc)
+                                                rc = errno;
+                                        continue;
+                                } else {
+                                        /*
+                                         * This is a hack.  We _should_ be able
+                                         * to use WIFEXITED(status) to see if
+                                         * there was an error, but it appears
+                                         * to be broken and it always returns 1
+                                         * (OK).  See wait(2).
+                                         */
+                                        int err = WEXITSTATUS(status);
+                                        if (err || WIFSIGNALED(status))
+                                                fprintf(stderr,
+                                                        "%s: PID %d had rc=%d\n",
+                                                        argv[0], ret, err);
+                                        if (!rc)
+                                                rc = err;
+
+                                        live_threads--;
+                                }
                         }
 
-                        if (ret < 0) {
-                                fprintf(stderr, "error: %s: wait - %s\n",
-                                        argv[0], strerror(errno));
-                                if (!rc)
-                                        rc = errno;
-                        } else {
-                                /*
-                                 * This is a hack.  We _should_ be able to use
-                                 * WIFEXITED(status) to see if there was an
-                                 * error, but it appears to be broken and it
-                                 * always returns 1 (OK).  See wait(2).
-                                 */
-                                int err = WEXITSTATUS(status);
-                                if (err || WIFSIGNALED(status))
-                                        fprintf(stderr,
-                                                "%s: PID %d had rc=%d\n",
-                                                argv[0], ret, err);
-                                if (!rc)
-                                        rc = err;
-
-                                live_threads--;
-                        }
+                        /* Show stats while all threads running */
+                        if (verbose < 0)
+                                shmem_snap(threads, live_threads);
                 }
+                sigaction(SIGCHLD, &saveact2, NULL);
+                sigaction(SIGALRM, &saveact1, NULL);
         }
 
+        sigprocmask(SIG_SETMASK, &saveset, NULL);
         return rc;
 }
 
@@ -1001,7 +1055,7 @@ int jt_obd_create(int argc, char **argv)
                 IOC_PACK(argv[0], data);
                 rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_CREATE, buf);
                 IOC_UNPACK(argv[0], data);
-                SHMEM_BUMP();
+                shmem_bump();
                 if (rc < 0) {
                         fprintf(stderr, "error: %s: #%d - %s\n",
                                 jt_cmdname(argv[0]), i, strerror(rc = errno));
@@ -1054,6 +1108,90 @@ int jt_obd_setattr(int argc, char **argv)
         return rc;
 }
 
+int jt_obd_test_setattr(int argc, char **argv)
+{
+        struct obd_ioctl_data data;
+        struct timeval start, next_time;
+        __u64 i, count, next_count;
+        int verbose = 1;
+        obd_id objid = 3;
+        char *end;
+        int rc = 0;
+
+        if (argc < 2 || argc > 4)
+                return CMD_HELP;
+
+        IOC_INIT(data);
+        count = strtoull(argv[1], &end, 0);
+        if (*end) {
+                fprintf(stderr, "error: %s: invalid iteration count '%s'\n",
+                        jt_cmdname(argv[0]), argv[1]);
+                return CMD_HELP;
+        }
+
+        if (argc >= 3) {
+                verbose = get_verbose(argv[0], argv[2]);
+                if (verbose == BAD_VERBOSE)
+                        return CMD_HELP;
+        }
+
+        if (argc >= 4) {
+                if (argv[3][0] == 't') {
+                        objid = strtoull(argv[3] + 1, &end, 0);
+                        if (thread)
+                                objid += thread - 1;
+                } else
+                        objid = strtoull(argv[3], &end, 0);
+                if (*end) {
+                        fprintf(stderr, "error: %s: invalid objid '%s'\n",
+                                jt_cmdname(argv[0]), argv[3]);
+                        return CMD_HELP;
+                }
+        }
+
+        gettimeofday(&start, NULL);
+        next_time.tv_sec = start.tv_sec - verbose;
+        next_time.tv_usec = start.tv_usec;
+        if (verbose != 0)
+                printf("%s: setting "LPD64" attrs (objid "LPX64"): %s",
+                       jt_cmdname(argv[0]), count, objid, ctime(&start.tv_sec));
+
+        for (i = 1, next_count = verbose; i <= count; i++) {
+                data.ioc_obdo1.o_id = objid;
+                data.ioc_obdo1.o_mode = S_IFREG;
+                data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE;
+                IOC_PACK(argv[0], data);
+                rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_SETATTR, &data);
+                shmem_bump();
+                if (rc < 0) {
+                        fprintf(stderr, "error: %s: #"LPD64" - %d:%s\n",
+                                jt_cmdname(argv[0]), i, errno, strerror(rc = errno));
+                        break;
+                } else {
+                        if (be_verbose
+                            (verbose, &next_time, i, &next_count, count))
+                                printf("%s: set attr #"LPD64"\n",
+                                       jt_cmdname(argv[0]), i);
+                }
+        }
+
+        if (!rc) {
+                struct timeval end;
+                double diff;
+
+                gettimeofday(&end, NULL);
+
+                diff = difftime(&end, &start);
+
+                --i;
+                if (verbose != 0)
+                        printf("%s: "LPD64" attrs in %.3fs (%.3f attr/s): %s",
+                               jt_cmdname(argv[0]), i, diff, i / diff,
+                               ctime(&end.tv_sec));
+        }
+        return rc;
+}
+
 int jt_obd_destroy(int argc, char **argv)
 {
         struct obd_ioctl_data data;
@@ -1102,7 +1240,7 @@ int jt_obd_destroy(int argc, char **argv)
                 IOC_PACK(argv[0], data);
                 rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_DESTROY, buf);
                 IOC_UNPACK(argv[0], data);
-                SHMEM_BUMP();
+                shmem_bump();
                 if (rc < 0) {
                         fprintf(stderr, "error: %s: objid "LPX64": %s\n",
                                 jt_cmdname(argv[0]), id, strerror(rc = errno));
@@ -1161,7 +1299,7 @@ int jt_obd_test_getattr(int argc, char **argv)
         char *end;
         int rc = 0;
 
-        if (argc < 2 && argc > 4)
+        if (argc < 2 || argc > 4)
                 return CMD_HELP;
 
         IOC_INIT(data);
@@ -1205,7 +1343,7 @@ int jt_obd_test_getattr(int argc, char **argv)
                 data.ioc_obdo1.o_valid = 0xffffffff;
                 IOC_PACK(argv[0], data);
                 rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_GETATTR, &data);
-                SHMEM_BUMP();
+                shmem_bump();
                 if (rc < 0) {
                         fprintf(stderr, "error: %s: #"LPD64" - %d:%s\n",
                                 jt_cmdname(argv[0]), i, errno, strerror(rc = errno));
@@ -1239,9 +1377,14 @@ int jt_obd_test_brw(int argc, char **argv)
 {
         struct obd_ioctl_data data;
         struct timeval start, next_time;
-        __u64 count, next_count, len, thr_offset = 0, objid = 3;
+        __u64 count, next_count, len, stride, thr_offset = 0, objid = 3;
         int write = 0, verbose = 1, cmd, i, rc = 0, pages = 1;
+        long n;
         int repeat_offset = 0;
+        unsigned long long ull;
+        int  nthr_per_obj = 0;
+        int  verify = 1;
+        int  obj_idx = 0;
         char *end;
 
         if (argc < 2 || argc > 7) {
@@ -1250,14 +1393,7 @@ int jt_obd_test_brw(int argc, char **argv)
                 return CMD_HELP;
         }
 
-        /* make each thread write to a different offset */
-        if (argv[1][0] == 't') {
-                count = strtoull(argv[1] + 1, &end, 0);
-                if (thread)
-                        thr_offset = thread - 1;
-        } else
-                count = strtoull(argv[1], &end, 0);
-
+        count = strtoull(argv[1], &end, 0);
         if (*end) {
                 fprintf(stderr, "error: %s: bad iteration count '%s'\n",
                         jt_cmdname(argv[0]), argv[1]);
@@ -1269,9 +1405,22 @@ int jt_obd_test_brw(int argc, char **argv)
                         write = 1;
                 /* else it's a read */
 
-                if (argv[2][0] != 0 &&
-                    argv[2][1] == 'r')
-                        repeat_offset = 1;
+                if (argv[2][0] != 0)
+                        for (i = 1; argv[2][i] != 0; i++)
+                                switch (argv[2][i]) {
+                                case 'r':
+                                        repeat_offset = 1;
+                                        break;
+                                        
+                                case 'x':
+                                        verify = 0;
+                                        break;
+                                        
+                                default:
+                                        fprintf (stderr, "Can't parse cmd '%s'\n",
+                                                 argv[2]);
+                                        return CMD_HELP;
+                                }
         }
 
         if (argc >= 4) {
@@ -1288,13 +1437,23 @@ int jt_obd_test_brw(int argc, char **argv)
                         return CMD_HELP;
                 }
         }
+
         if (argc >= 6) {
-                if (argv[5][0] == 't') {
+                if (thread &&
+                    (n = strtol(argv[5], &end, 0)) > 0 &&
+                    *end == 't' &&
+                    (ull = strtoull(end + 1, &end, 0)) > 0 &&
+                    *end == 0) {
+                        nthr_per_obj = n;
+                        objid = ull;
+                } else if (thread &&
+                           argv[5][0] == 't') {
+                        nthr_per_obj = 1;
                         objid = strtoull(argv[5] + 1, &end, 0);
-                        if (thread)
-                                objid += thread - 1;
-                } else
+                } else {
+                        nthr_per_obj = 0;
                         objid = strtoull(argv[5], &end, 0);
+                }
                 if (*end) {
                         fprintf(stderr, "error: %s: bad objid '%s'\n",
                                 jt_cmdname(argv[0]), argv[5]);
@@ -1338,12 +1497,39 @@ int jt_obd_test_brw(int argc, char **argv)
         }
 
         len = pages * PAGE_SIZE;
+        stride = len;
+
+        if (thread) {
+                pthread_mutex_lock (&shared_data->mutex);
+                if (nthr_per_obj != 0) {
+                        /* threads interleave */
+                        obj_idx = (thread - 1)/nthr_per_obj;
+                        objid += obj_idx;
+                        stride *= nthr_per_obj;
+                        thr_offset = ((thread - 1) % nthr_per_obj) * len;
+                        if (thr_offset == 0)
+                                shared_data->offsets[obj_idx] = stride;
+                } else {
+                        /* threads disjoint */
+                        thr_offset = (thread - 1) * len;
+                }
+
+                shared_data->barrier--;
+                if (shared_data->barrier == 0)
+                        pthread_cond_broadcast(&shared_data->cond);
+                else
+                        pthread_cond_wait(&shared_data->cond,
+                                          &shared_data->mutex);
+
+                pthread_mutex_unlock (&shared_data->mutex);
+        }
 
         data.ioc_obdo1.o_id = objid;
         data.ioc_obdo1.o_mode = S_IFREG;
-        data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE;
+        data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE | OBD_MD_FLFLAGS;
+        data.ioc_obdo1.o_flags = (verify ? OBD_FL_DEBUG_CHECK : 0);
         data.ioc_count = len;
-        data.ioc_offset = thr_offset * len * count;
+        data.ioc_offset = (repeat_offset ? 0 : thr_offset);
 
         gettimeofday(&start, NULL);
         next_time.tv_sec = start.tv_sec - verbose;
@@ -1359,18 +1545,28 @@ int jt_obd_test_brw(int argc, char **argv)
                 data.ioc_obdo1.o_valid &= ~(OBD_MD_FLBLOCKS|OBD_MD_FLGRANT);
                 IOC_PACK(argv[0], data);
                 rc = l2_ioctl(OBD_DEV_ID, cmd, buf);
-                SHMEM_BUMP();
+                shmem_bump();
                 if (rc) {
                         fprintf(stderr, "error: %s: #%d - %s on %s\n",
                                 jt_cmdname(argv[0]), i, strerror(rc = errno),
                                 write ? "write" : "read");
                         break;
                 } else if (be_verbose(verbose, &next_time,i, &next_count,count))
-                        printf("%s: %s number %dx%d\n", jt_cmdname(argv[0]),
-                               write ? "write" : "read", i, pages);
-
-                if (!repeat_offset)
-                        data.ioc_offset += len;
+                        printf("%s: %s number %d @ "LPD64":"LPU64" for %d\n",
+                               jt_cmdname(argv[0]), write ? "write" : "read", i,
+                               data.ioc_obdo1.o_id, data.ioc_offset,
+                               (int)(pages * PAGE_SIZE));
+
+                if (!repeat_offset) {
+                        if (stride == len) {
+                                data.ioc_offset += stride;
+                        } else if (i < count) {
+                                pthread_mutex_lock (&shared_data->mutex);
+                                data.ioc_offset = shared_data->offsets[obj_idx];
+                                shared_data->offsets[obj_idx] += len;
+                                pthread_mutex_unlock (&shared_data->mutex);
+                        }
+                }
         }
 
         if (!rc) {
@@ -1988,7 +2184,7 @@ int jt_llog_remove(int argc, char **argv)
                 if (argc == 3)
                         fprintf(stdout, "log %s are removed.\n", argv[2]);
                 else
-                        fprintf(stdout, "the log in catlog %s are removed. \n", argv[1]);
+                        fprintf(stdout, "the log in catalog %s are removed. \n", argv[1]);
         } else
                 fprintf(stderr, "OBD_IOC_LLOG_REMOVE failed: %s\n",
                         strerror(errno));
@@ -2007,7 +2203,7 @@ static void signal_server(int sig)
 
 int obd_initialize(int argc, char **argv)
 {
-        SHMEM_SETUP();
+        shmem_setup();
         register_ioc_dev(OBD_DEV_ID, OBD_DEV_PATH);
 
         return 0;
index b68d02d..bec2e12 100644 (file)
@@ -36,6 +36,7 @@ int jt_obd_abort_recovery(int argc, char **argv);
 int jt_obd_list(int argc, char **argv);
 int jt_obd_create(int argc, char **argv);
 int jt_obd_setattr(int argc, char **argv);
+int jt_obd_test_setattr(int argc, char **argv);
 int jt_obd_destroy(int argc, char **argv);
 int jt_obd_getattr(int argc, char **argv);
 int jt_obd_test_getattr(int argc, char **argv);