From 275e54a6cab5480929d7e4d486cfa916132428a8 Mon Sep 17 00:00:00 2001 From: eeb Date: Fri, 7 Jan 2005 12:41:05 +0000 Subject: [PATCH] * Included changes from *old* b1_4/b_cray * OBD_FL_DEBUG_CHECK for consistent obdecho end-to-end checking * struct echo_obd cleanup * obdecho support for "difficult" reply testing (test_setattr) * replaced cowardly eap_from_cookie() usage with EAP_FROM_COOKIE() which asserts it is being called correctly * bracketed echo_client_iocontrol() guts with unlock/lock_kernel() for increased concurrency during echo ioctl execution * better thread control and stats in obd.c * n-threads-over-m-objects test_brw support (required for obdfilter-survey) * openibnal changes * re-enabled openib on 2.5/2.6 builds configure --with-openib=no can be used to disable openib explicitly if required. * fixed some 64 bit casts * just use kernel's page_to_phys() * fixed CERROR format problem in libcfs/module.c --- lustre/include/linux/lustre_idl.h | 1 + lustre/include/linux/obd.h | 15 +- lustre/obdecho/echo.c | 54 ++++- lustre/obdecho/echo_client.c | 57 +++--- lustre/utils/lctl.c | 3 + lustre/utils/obd.c | 412 ++++++++++++++++++++++++++++---------- lustre/utils/obdctl.h | 1 + 7 files changed, 390 insertions(+), 153 deletions(-) diff --git a/lustre/include/linux/lustre_idl.h b/lustre/include/linux/lustre_idl.h index 55921d4..6ce059b 100644 --- a/lustre/include/linux/lustre_idl.h +++ b/lustre/include/linux/lustre_idl.h @@ -245,6 +245,7 @@ typedef uint32_t obd_count; #define OBD_FL_NORPC (0x00000008) // if set in o_flags set in OSC not OST #define OBD_FL_IDONLY (0x00000010) // if set in o_flags only adjust obj id #define OBD_FL_RECREATE_OBJS (0x00000020) // recreate missing obj +#define OBD_FL_DEBUG_CHECK (0x00000040) /* echo client/server debug check */ #define OBD_INLINESZ 64 diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index b9fb45b..04ba628 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -320,16 +320,11 @@ struct mds_obd { }; struct echo_obd { - struct obdo oa; - spinlock_t eo_lock; - __u64 eo_lastino; - atomic_t eo_getattr; - atomic_t eo_setattr; - atomic_t eo_create; - atomic_t eo_destroy; - atomic_t eo_prep; - atomic_t eo_read; - atomic_t eo_write; + struct obdo eo_oa; + spinlock_t eo_lock; + __u64 eo_lastino; + struct lustre_handle eo_nl_lock; + atomic_t eo_prep; }; /* diff --git a/lustre/obdecho/echo.c b/lustre/obdecho/echo.c index e9be311..3f12f12 100644 --- a/lustre/obdecho/echo.c +++ b/lustre/obdecho/echo.c @@ -76,6 +76,21 @@ static int echo_disconnect(struct obd_export *exp, int flags) exp->exp_flags = flags; spin_unlock_irqrestore(&exp->exp_lock, irqflags); + /* complete all outstanding replies */ + spin_lock_irqsave(&exp->exp_lock, irqflags); + while (!list_empty(&exp->exp_outstanding_replies)) { + struct ptlrpc_reply_state *rs = + list_entry(exp->exp_outstanding_replies.next, + struct ptlrpc_reply_state, rs_exp_list); + struct ptlrpc_service *svc = rs->rs_srv_ni->sni_service; + + spin_lock(&svc->srv_lock); + list_del_init(&rs->rs_exp_list); + ptlrpc_schedule_difficult_reply(rs); + spin_unlock(&svc->srv_lock); + } + spin_unlock_irqrestore(&exp->exp_lock, irqflags); + return class_disconnect(exp, flags); } @@ -122,7 +137,6 @@ int echo_create(struct obd_export *exp, struct obdo *oa, oa->o_id = echo_next_id(obd); oa->o_valid = OBD_MD_FLID; - atomic_inc(&obd->u.echo.eo_create); return 0; } @@ -148,8 +162,6 @@ int echo_destroy(struct obd_export *exp, struct obdo *oa, RETURN(-EINVAL); } - atomic_inc(&obd->u.echo.eo_destroy); - return 0; } @@ -170,7 +182,7 @@ static int echo_getattr(struct obd_export *exp, struct obdo *oa, RETURN(-EINVAL); } - obdo_cpy_md(oa, &obd->u.echo.oa, oa->o_valid); + obdo_cpy_md(oa, &obd->u.echo.eo_oa, oa->o_valid); oa->o_id = id; return 0; @@ -192,9 +204,14 @@ static int echo_setattr(struct obd_export *exp, struct obdo *oa, RETURN(-EINVAL); } - memcpy(&obd->u.echo.oa, oa, sizeof(*oa)); + memcpy(&obd->u.echo.eo_oa, oa, sizeof(*oa)); - atomic_inc(&obd->u.echo.eo_setattr); + if (oa->o_id & 4) { + /* Save lock to force ACKed reply */ + ldlm_lock_addref (&obd->u.echo.eo_nl_lock, LCK_NL); + oti->oti_ack_locks[0].mode = LCK_NL; + oti->oti_ack_locks[0].lock = obd->u.echo.eo_nl_lock; + } return 0; } @@ -288,6 +305,9 @@ int echo_preprw(int cmd, struct obd_export *export, struct obdo *oa, for (i = 0; i < objcount; i++, obj++) { int gfp_mask = (obj->ioo_id & 1) ? GFP_HIGHUSER : GFP_KERNEL; int ispersistent = obj->ioo_id == ECHO_PERSISTENT_OBJID; + int debug_setup = (!ispersistent && + (oa->o_valid & OBD_MD_FLFLAGS) != 0 && + (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0); int j; for (j = 0 ; j < obj->ioo_bufcnt ; j++, nb++, r++) { @@ -322,7 +342,7 @@ int echo_preprw(int cmd, struct obd_export *export, struct obdo *oa, if (cmd & OBD_BRW_READ) r->rc = r->len; - if (!ispersistent) + if (debug_setup) echo_page_debug_setup(r->page, cmd, obj->ioo_id, r->offset, r->len); } @@ -391,7 +411,9 @@ int echo_commitrw(int cmd, struct obd_export *export, struct obdo *oa, for (i = 0; i < objcount; i++, obj++) { int verify = (rc == 0 && - obj->ioo_id != ECHO_PERSISTENT_OBJID); + obj->ioo_id != ECHO_PERSISTENT_OBJID && + (oa->o_valid & OBD_MD_FLFLAGS) != 0 && + (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0); int j; for (j = 0 ; j < obj->ioo_bufcnt ; j++, r++) { @@ -443,6 +465,9 @@ commitrw_cleanup: static int echo_setup(struct obd_device *obd, obd_count len, void *buf) { struct lprocfs_static_vars lvars; + int rc; + int lock_flags = 0; + struct ldlm_res_id res_id = {.name = {1}}; ENTRY; spin_lock_init(&obd->u.echo.eo_lock); @@ -455,6 +480,12 @@ static int echo_setup(struct obd_device *obd, obd_count len, void *buf) RETURN(-ENOMEM); } + rc = ldlm_cli_enqueue(NULL, NULL, obd->obd_namespace, res_id, + LDLM_PLAIN, NULL, LCK_NL, &lock_flags, + NULL, ldlm_completion_ast, NULL, NULL, + NULL, 0, NULL, &obd->u.echo.eo_nl_lock); + LASSERT (rc == ELDLM_OK); + lprocfs_init_vars(echo, &lvars); if (lprocfs_obd_setup(obd, lvars.obd_vars) == 0 && lprocfs_alloc_obd_stats(obd, LPROC_ECHO_LAST) == 0) { @@ -479,6 +510,13 @@ static int echo_cleanup(struct obd_device *obd, int flags) lprocfs_free_obd_stats(obd); lprocfs_obd_cleanup(obd); + ldlm_lock_decref (&obd->u.echo.eo_nl_lock, LCK_NL); + + /* XXX Bug 3413; wait for a bit to ensure the BL callback has + * happened before calling ldlm_namespace_free() */ + set_current_state (TASK_UNINTERRUPTIBLE); + schedule_timeout (HZ); + ldlm_namespace_free(obd->obd_namespace, flags & OBD_OPT_FORCE); leaked = atomic_read(&obd->u.echo.eo_prep); diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 67935cb..58d2358 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -29,6 +29,7 @@ #include #endif #include +#include #else #include #endif @@ -499,13 +500,13 @@ static int echo_client_kbrw(struct obd_device *obd, int rw, struct obdo *oa, obd_off off; int i; int rc; - int verify = 0; + int verify; int gfp_mask; - /* oa_id == ECHO_PERSISTENT_OBJID => speed test (no verification). - * oa & 1 => use HIGHMEM */ + verify = ((oa->o_id) != ECHO_PERSISTENT_OBJID && + (oa->o_valid & OBD_MD_FLFLAGS) != 0 && + (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0); - verify = (oa->o_id) != ECHO_PERSISTENT_OBJID; gfp_mask = ((oa->o_id & 2) == 0) ? GFP_KERNEL : GFP_HIGHUSER; LASSERT(rw == OBD_BRW_WRITE || rw == OBD_BRW_READ); @@ -658,6 +659,10 @@ struct echo_async_page { struct list_head eap_item; }; +#define EAP_FROM_COOKIE(c) \ + (LASSERT(((struct echo_async_page *)(c))->eap_magic == EAP_MAGIC), \ + (struct echo_async_page *)(c)) + struct echo_async_state { spinlock_t eas_lock; obd_off eas_next_offset; @@ -681,14 +686,6 @@ static int eas_should_wake(struct echo_async_state *eas) return rc; }; -struct echo_async_page *eap_from_cookie(void *cookie) -{ - struct echo_async_page *eap = cookie; - if (eap->eap_magic != EAP_MAGIC) - return ERR_PTR(-EINVAL); - return eap; -}; - static int ec_ap_make_ready(void *data, int cmd) { /* our pages are issued ready */ @@ -703,26 +700,23 @@ static int ec_ap_refresh_count(void *data, int cmd) } static void ec_ap_fill_obdo(void *data, int cmd, struct obdo *oa) { - struct echo_async_page *eap; - eap = eap_from_cookie(data); - if (IS_ERR(eap)) - return; + struct echo_async_page *eap = EAP_FROM_COOKIE(data); memcpy(oa, &eap->eap_eas->eas_oa, sizeof(*oa)); } static void ec_ap_completion(void *data, int cmd, struct obdo *oa, int rc) { - struct echo_async_page *eap = eap_from_cookie(data); + struct echo_async_page *eap = EAP_FROM_COOKIE(data); struct echo_async_state *eas; unsigned long flags; - if (IS_ERR(eap)) - return; eas = eap->eap_eas; if (cmd == OBD_BRW_READ && - eas->eas_oa.o_id != ECHO_PERSISTENT_OBJID) + eas->eas_oa.o_id != ECHO_PERSISTENT_OBJID && + (eas->eas_oa.o_valid & OBD_MD_FLFLAGS) != 0 && + (eas->eas_oa.o_flags & OBD_FL_DEBUG_CHECK) != 0) echo_client_page_debug_check(eas->eas_lsm, eap->eap_page, eas->eas_oa.o_id, eap->eap_off, PAGE_SIZE); @@ -758,11 +752,12 @@ static int echo_client_async_page(struct obd_export *exp, int rw, #if 0 int verify; int gfp_mask; - /* oa_id == 0 => speed test (no verification) else... - * oa & 1 => use HIGHMEM - */ - verify = (oa->o_id != 0); - gfp_mask = ((oa->o_id & 1) == 0) ? GFP_KERNEL : GFP_HIGHUSER; + + verify = ((oa->o_id) != ECHO_PERSISTENT_OBJID && + (oa->o_valid & OBD_MD_FLFLAGS) != 0 && + (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0); + + gfp_mask = ((oa->o_id & 2) == 0) ? GFP_KERNEL : GFP_HIGHUSER; #endif LASSERT(rw == OBD_BRW_WRITE || rw == OBD_BRW_READ); @@ -847,7 +842,9 @@ static int echo_client_async_page(struct obd_export *exp, int rw, break; } - if (oa->o_id != ECHO_PERSISTENT_OBJID) + if (oa->o_id != ECHO_PERSISTENT_OBJID && + (oa->o_valid & OBD_MD_FLFLAGS) != 0 && + (oa->o_flags & OBD_FL_DEBUG_CHECK) != 0) echo_client_page_debug_setup(lsm, eap->eap_page, rw, oa->o_id, eap->eap_off, PAGE_SIZE); @@ -951,7 +948,9 @@ static int echo_client_prep_commit(struct obd_export *exp, int rw, if (page == NULL && lnb[i].rc == 0) continue; - if (oa->o_id == ECHO_PERSISTENT_OBJID) + if (oa->o_id == ECHO_PERSISTENT_OBJID || + (oa->o_valid & OBD_MD_FLFLAGS) == 0 || + (oa->o_flags & OBD_FL_DEBUG_CHECK) == 0) continue; if (rw == OBD_BRW_WRITE) @@ -1194,6 +1193,8 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int i; ENTRY; + unlock_kernel(); + memset(&dummy_oti, 0, sizeof(dummy_oti)); obd = exp->exp_obd; @@ -1312,6 +1313,8 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, ldlm_lock_decref(&ack_lock->lock, ack_lock->mode); } + lock_kernel(); + return rc; } diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 197b95e..bb41dd5 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -193,6 +193,9 @@ command_t cmdlist[] = { {"test_getattr", jt_obd_test_getattr, 0, "do getattrs (on OST object (objid+1 on each thread))\n" "usage: test_getattr [verbose [[t]objid]]"}, + {"test_setattr", jt_obd_test_setattr, 0, + "do setattrs (on OST object (objid+1 on each thread))\n" + "usage: test_setattr [verbose [[t]objid]]"}, {"test_brw", jt_obd_test_brw, 0, "do bulk read/writes ( per I/O, on OST object )\n" "usage: test_brw [t] [write [verbose [npages [[t]objid]]]]"}, diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index 95b9917..e9e1c04 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -58,20 +58,26 @@ #include #include "parser.h" #include +#include +#include +#include -#define SHMEM_STATS 1 #define MAX_STRING_SIZE 128 #define DEVICES_LIST "/proc/fs/lustre/devices" -#if SHMEM_STATS -# include -# include - -# define MAX_SHMEM_COUNT 1024 -static long long *shared_counters; -static long long counter_snapshot[2][MAX_SHMEM_COUNT]; +#define MAX_THREADS 1024 +struct shared_data { + __u64 counters[MAX_THREADS]; + __u64 offsets[MAX_THREADS]; + int running; + int barrier; + pthread_mutex_t mutex; + pthread_cond_t cond; +}; +static struct shared_data *shared_data; +static __u64 counter_snapshot[2][MAX_THREADS]; +static int prev_valid; struct timeval prev_time; -#endif static int jt_recording; static char rawbuf[8192]; @@ -79,6 +85,7 @@ static char *buf = rawbuf; static int max = sizeof(rawbuf); static int thread; +static int nthreads; static uint32_t cur_device = MAX_OBD_DEVICES; @@ -228,7 +235,7 @@ int parse_devname(char *func, char *name) /* Assume it's a number. This means that bogus strings become * 0. I might care about that some day. */ ret = strtoul(name, NULL, 0); - printf("Selected device %d\n", ret); + // printf("Selected device %d\n", ret); } return ret; @@ -426,25 +433,24 @@ int do_disconnect(char *func, int verbose) return 0; } -#if SHMEM_STATS static void shmem_setup(void) { /* Create new segment */ - int shmid = shmget(IPC_PRIVATE, sizeof(counter_snapshot[0]), 0600); + int shmid = shmget(IPC_PRIVATE, sizeof(*shared_data), 0600); if (shmid == -1) { - fprintf(stderr, "Can't create shared memory counters: %s\n", + fprintf(stderr, "Can't create shared data: %s\n", strerror(errno)); return; } /* Attatch to new segment */ - shared_counters = (long long *)shmat(shmid, NULL, 0); + shared_data = (struct shared_data *)shmat(shmid, NULL, 0); - if (shared_counters == (long long *)(-1)) { - fprintf(stderr, "Can't attach shared memory counters: %s\n", + if (shared_data == (struct shared_data *)(-1)) { + fprintf(stderr, "Can't attach shared data: %s\n", strerror(errno)); - shared_counters = NULL; + shared_data = NULL; return; } @@ -452,47 +458,60 @@ static void shmem_setup(void) * Forks will inherit attached segments, so we should be OK. */ if (shmctl(shmid, IPC_RMID, NULL) == -1) { - fprintf(stderr, "Can't destroy shared memory counters: %s\n", + fprintf(stderr, "Can't destroy shared data: %s\n", strerror(errno)); } } -static inline void shmem_reset(void) +static inline void shmem_reset(int total_threads) { - if (shared_counters == NULL) + if (shared_data == NULL) return; - memset(shared_counters, 0, sizeof(counter_snapshot[0])); + memset(shared_data, 0, sizeof(*shared_data)); + pthread_mutex_init(&shared_data->mutex, NULL); + pthread_cond_init(&shared_data->cond, NULL); memset(counter_snapshot, 0, sizeof(counter_snapshot)); - gettimeofday(&prev_time, NULL); + prev_valid = 0; + shared_data->barrier = total_threads; } static inline void shmem_bump(void) { - if (shared_counters == NULL || thread <= 0 || thread > MAX_SHMEM_COUNT) + static int bumped_running; + + if (shared_data == NULL || thread <= 0 || thread > MAX_THREADS) return; - shared_counters[thread - 1]++; + pthread_mutex_lock(&shared_data->mutex); + shared_data->counters[thread - 1]++; + if (!bumped_running) + shared_data->running++; + pthread_mutex_unlock(&shared_data->mutex); + bumped_running = 1; } -static void shmem_snap(int n) +static void shmem_snap(int total_threads, int live_threads) { struct timeval this_time; int non_zero = 0; - long long total = 0; + __u64 total = 0; double secs; + int running; int i; - if (shared_counters == NULL || n > MAX_SHMEM_COUNT) + if (shared_data == NULL || total_threads > MAX_THREADS) return; - memcpy(counter_snapshot[1], counter_snapshot[0], - n * sizeof(counter_snapshot[0][0])); - memcpy(counter_snapshot[0], shared_counters, - n * sizeof(counter_snapshot[0][0])); + pthread_mutex_lock(&shared_data->mutex); + memcpy(counter_snapshot[0], shared_data->counters, + total_threads * sizeof(counter_snapshot[0][0])); + running = shared_data->running; + pthread_mutex_unlock(&shared_data->mutex); + gettimeofday(&this_time, NULL); - for (i = 0; i < n; i++) { + for (i = 0; i < total_threads; i++) { long long this_count = counter_snapshot[0][i] - counter_snapshot[1][i]; @@ -503,24 +522,21 @@ static void shmem_snap(int n) } secs = (this_time.tv_sec + this_time.tv_usec / 1000000.0) - - (prev_time.tv_sec + prev_time.tv_usec / 1000000.0); + (prev_time.tv_sec + prev_time.tv_usec / 1000000.0); - printf("%d/%d Total: %f/second\n", non_zero, n, total / secs); + if (prev_valid && + live_threads == total_threads && + secs > 0.0) /* someone screwed with the time? */ + printf("%d/%d Total: %f/second\n", non_zero, total_threads, total / secs); + memcpy(counter_snapshot[1], counter_snapshot[0], + total_threads * sizeof(counter_snapshot[0][0])); prev_time = this_time; + if (!prev_valid && + running == total_threads) + prev_valid = 1; } -#define SHMEM_SETUP() shmem_setup() -#define SHMEM_RESET() shmem_reset() -#define SHMEM_BUMP() shmem_bump() -#define SHMEM_SNAP(n) shmem_snap(n) -#else -#define SHMEM_SETUP() -#define SHMEM_RESET() -#define SHMEM_BUMP() -#define SHMEM_SNAP(n) -#endif - extern command_t cmdlist[]; static int do_device(char *func, char *devname) @@ -583,8 +599,18 @@ int jt_opt_device(int argc, char **argv) return rc; } +static void parent_sighandler (int sig) +{ + return; +} + int jt_opt_threads(int argc, char **argv) { + sigset_t saveset; + sigset_t sigset; + struct sigaction sigact; + struct sigaction saveact1; + struct sigaction saveact2; __u64 threads, next_thread; int verbose; int rc = 0; @@ -595,8 +621,8 @@ int jt_opt_threads(int argc, char **argv) return CMD_HELP; threads = strtoull(argv[1], &end, 0); - if (*end) { - fprintf(stderr, "error: %s: invalid page count '%s'\n", + if (*end || threads > MAX_THREADS) { + fprintf(stderr, "error: %s: invalid thread count '%s'\n", jt_cmdname(argv[0]), argv[1]); return CMD_HELP; } @@ -609,7 +635,14 @@ int jt_opt_threads(int argc, char **argv) printf("%s: starting "LPD64" threads on device %s running %s\n", argv[0], threads, argv[3], argv[4]); - SHMEM_RESET(); + shmem_reset(threads); + + sigemptyset(&sigset); + sigaddset(&sigset, SIGALRM); + sigaddset(&sigset, SIGCHLD); + sigprocmask(SIG_BLOCK, &sigset, &saveset); + + nthreads = threads; for (i = 1, next_thread = verbose; i <= threads; i++) { rc = fork(); @@ -618,6 +651,8 @@ int jt_opt_threads(int argc, char **argv) strerror(rc = errno)); break; } else if (rc == 0) { + sigprocmask(SIG_SETMASK, &saveset, NULL); + thread = i; argv[2] = "--device"; return jt_opt_device(argc - 2, argv + 2); @@ -630,45 +665,64 @@ int jt_opt_threads(int argc, char **argv) if (!thread) { /* parent process */ int live_threads = threads; + sigemptyset(&sigset); + sigemptyset(&sigact.sa_mask); + sigact.sa_handler = parent_sighandler; + sigact.sa_flags = 0; + + sigaction(SIGALRM, &sigact, &saveact1); + sigaction(SIGCHLD, &sigact, &saveact2); + while (live_threads > 0) { int status; pid_t ret; - ret = waitpid(0, &status, verbose < 0 ? WNOHANG : 0); - if (ret == 0) { - if (verbose >= 0) - abort(); - - sleep(-verbose); - SHMEM_SNAP(threads); - continue; + if (verbose < 0) /* periodic stats */ + alarm(-verbose); + + sigsuspend(&sigset); + alarm(0); + + while (live_threads > 0) { + ret = waitpid(0, &status, WNOHANG); + if (ret == 0) + break; + + if (ret < 0) { + fprintf(stderr, "error: %s: wait - %s\n", + argv[0], strerror(errno)); + if (!rc) + rc = errno; + continue; + } else { + /* + * This is a hack. We _should_ be able + * to use WIFEXITED(status) to see if + * there was an error, but it appears + * to be broken and it always returns 1 + * (OK). See wait(2). + */ + int err = WEXITSTATUS(status); + if (err || WIFSIGNALED(status)) + fprintf(stderr, + "%s: PID %d had rc=%d\n", + argv[0], ret, err); + if (!rc) + rc = err; + + live_threads--; + } } - if (ret < 0) { - fprintf(stderr, "error: %s: wait - %s\n", - argv[0], strerror(errno)); - if (!rc) - rc = errno; - } else { - /* - * This is a hack. We _should_ be able to use - * WIFEXITED(status) to see if there was an - * error, but it appears to be broken and it - * always returns 1 (OK). See wait(2). - */ - int err = WEXITSTATUS(status); - if (err || WIFSIGNALED(status)) - fprintf(stderr, - "%s: PID %d had rc=%d\n", - argv[0], ret, err); - if (!rc) - rc = err; - - live_threads--; - } + /* Show stats while all threads running */ + if (verbose < 0) + shmem_snap(threads, live_threads); } + sigaction(SIGCHLD, &saveact2, NULL); + sigaction(SIGALRM, &saveact1, NULL); } + sigprocmask(SIG_SETMASK, &saveset, NULL); return rc; } @@ -1001,7 +1055,7 @@ int jt_obd_create(int argc, char **argv) IOC_PACK(argv[0], data); rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_CREATE, buf); IOC_UNPACK(argv[0], data); - SHMEM_BUMP(); + shmem_bump(); if (rc < 0) { fprintf(stderr, "error: %s: #%d - %s\n", jt_cmdname(argv[0]), i, strerror(rc = errno)); @@ -1054,6 +1108,90 @@ int jt_obd_setattr(int argc, char **argv) return rc; } +int jt_obd_test_setattr(int argc, char **argv) +{ + struct obd_ioctl_data data; + struct timeval start, next_time; + __u64 i, count, next_count; + int verbose = 1; + obd_id objid = 3; + char *end; + int rc = 0; + + if (argc < 2 || argc > 4) + return CMD_HELP; + + IOC_INIT(data); + count = strtoull(argv[1], &end, 0); + if (*end) { + fprintf(stderr, "error: %s: invalid iteration count '%s'\n", + jt_cmdname(argv[0]), argv[1]); + return CMD_HELP; + } + + if (argc >= 3) { + verbose = get_verbose(argv[0], argv[2]); + if (verbose == BAD_VERBOSE) + return CMD_HELP; + } + + if (argc >= 4) { + if (argv[3][0] == 't') { + objid = strtoull(argv[3] + 1, &end, 0); + if (thread) + objid += thread - 1; + } else + objid = strtoull(argv[3], &end, 0); + if (*end) { + fprintf(stderr, "error: %s: invalid objid '%s'\n", + jt_cmdname(argv[0]), argv[3]); + return CMD_HELP; + } + } + + gettimeofday(&start, NULL); + next_time.tv_sec = start.tv_sec - verbose; + next_time.tv_usec = start.tv_usec; + if (verbose != 0) + printf("%s: setting "LPD64" attrs (objid "LPX64"): %s", + jt_cmdname(argv[0]), count, objid, ctime(&start.tv_sec)); + + for (i = 1, next_count = verbose; i <= count; i++) { + data.ioc_obdo1.o_id = objid; + data.ioc_obdo1.o_mode = S_IFREG; + data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE; + IOC_PACK(argv[0], data); + rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_SETATTR, &data); + shmem_bump(); + if (rc < 0) { + fprintf(stderr, "error: %s: #"LPD64" - %d:%s\n", + jt_cmdname(argv[0]), i, errno, strerror(rc = errno)); + break; + } else { + if (be_verbose + (verbose, &next_time, i, &next_count, count)) + printf("%s: set attr #"LPD64"\n", + jt_cmdname(argv[0]), i); + } + } + + if (!rc) { + struct timeval end; + double diff; + + gettimeofday(&end, NULL); + + diff = difftime(&end, &start); + + --i; + if (verbose != 0) + printf("%s: "LPD64" attrs in %.3fs (%.3f attr/s): %s", + jt_cmdname(argv[0]), i, diff, i / diff, + ctime(&end.tv_sec)); + } + return rc; +} + int jt_obd_destroy(int argc, char **argv) { struct obd_ioctl_data data; @@ -1102,7 +1240,7 @@ int jt_obd_destroy(int argc, char **argv) IOC_PACK(argv[0], data); rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_DESTROY, buf); IOC_UNPACK(argv[0], data); - SHMEM_BUMP(); + shmem_bump(); if (rc < 0) { fprintf(stderr, "error: %s: objid "LPX64": %s\n", jt_cmdname(argv[0]), id, strerror(rc = errno)); @@ -1161,7 +1299,7 @@ int jt_obd_test_getattr(int argc, char **argv) char *end; int rc = 0; - if (argc < 2 && argc > 4) + if (argc < 2 || argc > 4) return CMD_HELP; IOC_INIT(data); @@ -1205,7 +1343,7 @@ int jt_obd_test_getattr(int argc, char **argv) data.ioc_obdo1.o_valid = 0xffffffff; IOC_PACK(argv[0], data); rc = l2_ioctl(OBD_DEV_ID, OBD_IOC_GETATTR, &data); - SHMEM_BUMP(); + shmem_bump(); if (rc < 0) { fprintf(stderr, "error: %s: #"LPD64" - %d:%s\n", jt_cmdname(argv[0]), i, errno, strerror(rc = errno)); @@ -1239,9 +1377,14 @@ int jt_obd_test_brw(int argc, char **argv) { struct obd_ioctl_data data; struct timeval start, next_time; - __u64 count, next_count, len, thr_offset = 0, objid = 3; + __u64 count, next_count, len, stride, thr_offset = 0, objid = 3; int write = 0, verbose = 1, cmd, i, rc = 0, pages = 1; + long n; int repeat_offset = 0; + unsigned long long ull; + int nthr_per_obj = 0; + int verify = 1; + int obj_idx = 0; char *end; if (argc < 2 || argc > 7) { @@ -1250,14 +1393,7 @@ int jt_obd_test_brw(int argc, char **argv) return CMD_HELP; } - /* make each thread write to a different offset */ - if (argv[1][0] == 't') { - count = strtoull(argv[1] + 1, &end, 0); - if (thread) - thr_offset = thread - 1; - } else - count = strtoull(argv[1], &end, 0); - + count = strtoull(argv[1], &end, 0); if (*end) { fprintf(stderr, "error: %s: bad iteration count '%s'\n", jt_cmdname(argv[0]), argv[1]); @@ -1269,9 +1405,22 @@ int jt_obd_test_brw(int argc, char **argv) write = 1; /* else it's a read */ - if (argv[2][0] != 0 && - argv[2][1] == 'r') - repeat_offset = 1; + if (argv[2][0] != 0) + for (i = 1; argv[2][i] != 0; i++) + switch (argv[2][i]) { + case 'r': + repeat_offset = 1; + break; + + case 'x': + verify = 0; + break; + + default: + fprintf (stderr, "Can't parse cmd '%s'\n", + argv[2]); + return CMD_HELP; + } } if (argc >= 4) { @@ -1288,13 +1437,23 @@ int jt_obd_test_brw(int argc, char **argv) return CMD_HELP; } } + if (argc >= 6) { - if (argv[5][0] == 't') { + if (thread && + (n = strtol(argv[5], &end, 0)) > 0 && + *end == 't' && + (ull = strtoull(end + 1, &end, 0)) > 0 && + *end == 0) { + nthr_per_obj = n; + objid = ull; + } else if (thread && + argv[5][0] == 't') { + nthr_per_obj = 1; objid = strtoull(argv[5] + 1, &end, 0); - if (thread) - objid += thread - 1; - } else + } else { + nthr_per_obj = 0; objid = strtoull(argv[5], &end, 0); + } if (*end) { fprintf(stderr, "error: %s: bad objid '%s'\n", jt_cmdname(argv[0]), argv[5]); @@ -1338,12 +1497,39 @@ int jt_obd_test_brw(int argc, char **argv) } len = pages * PAGE_SIZE; + stride = len; + + if (thread) { + pthread_mutex_lock (&shared_data->mutex); + if (nthr_per_obj != 0) { + /* threads interleave */ + obj_idx = (thread - 1)/nthr_per_obj; + objid += obj_idx; + stride *= nthr_per_obj; + thr_offset = ((thread - 1) % nthr_per_obj) * len; + if (thr_offset == 0) + shared_data->offsets[obj_idx] = stride; + } else { + /* threads disjoint */ + thr_offset = (thread - 1) * len; + } + + shared_data->barrier--; + if (shared_data->barrier == 0) + pthread_cond_broadcast(&shared_data->cond); + else + pthread_cond_wait(&shared_data->cond, + &shared_data->mutex); + + pthread_mutex_unlock (&shared_data->mutex); + } data.ioc_obdo1.o_id = objid; data.ioc_obdo1.o_mode = S_IFREG; - data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE; + data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE | OBD_MD_FLFLAGS; + data.ioc_obdo1.o_flags = (verify ? OBD_FL_DEBUG_CHECK : 0); data.ioc_count = len; - data.ioc_offset = thr_offset * len * count; + data.ioc_offset = (repeat_offset ? 0 : thr_offset); gettimeofday(&start, NULL); next_time.tv_sec = start.tv_sec - verbose; @@ -1359,18 +1545,28 @@ int jt_obd_test_brw(int argc, char **argv) data.ioc_obdo1.o_valid &= ~(OBD_MD_FLBLOCKS|OBD_MD_FLGRANT); IOC_PACK(argv[0], data); rc = l2_ioctl(OBD_DEV_ID, cmd, buf); - SHMEM_BUMP(); + shmem_bump(); if (rc) { fprintf(stderr, "error: %s: #%d - %s on %s\n", jt_cmdname(argv[0]), i, strerror(rc = errno), write ? "write" : "read"); break; } else if (be_verbose(verbose, &next_time,i, &next_count,count)) - printf("%s: %s number %dx%d\n", jt_cmdname(argv[0]), - write ? "write" : "read", i, pages); - - if (!repeat_offset) - data.ioc_offset += len; + printf("%s: %s number %d @ "LPD64":"LPU64" for %d\n", + jt_cmdname(argv[0]), write ? "write" : "read", i, + data.ioc_obdo1.o_id, data.ioc_offset, + (int)(pages * PAGE_SIZE)); + + if (!repeat_offset) { + if (stride == len) { + data.ioc_offset += stride; + } else if (i < count) { + pthread_mutex_lock (&shared_data->mutex); + data.ioc_offset = shared_data->offsets[obj_idx]; + shared_data->offsets[obj_idx] += len; + pthread_mutex_unlock (&shared_data->mutex); + } + } } if (!rc) { @@ -1988,7 +2184,7 @@ int jt_llog_remove(int argc, char **argv) if (argc == 3) fprintf(stdout, "log %s are removed.\n", argv[2]); else - fprintf(stdout, "the log in catlog %s are removed. \n", argv[1]); + fprintf(stdout, "the log in catalog %s are removed. \n", argv[1]); } else fprintf(stderr, "OBD_IOC_LLOG_REMOVE failed: %s\n", strerror(errno)); @@ -2007,7 +2203,7 @@ static void signal_server(int sig) int obd_initialize(int argc, char **argv) { - SHMEM_SETUP(); + shmem_setup(); register_ioc_dev(OBD_DEV_ID, OBD_DEV_PATH); return 0; diff --git a/lustre/utils/obdctl.h b/lustre/utils/obdctl.h index b68d02d..bec2e12 100644 --- a/lustre/utils/obdctl.h +++ b/lustre/utils/obdctl.h @@ -36,6 +36,7 @@ int jt_obd_abort_recovery(int argc, char **argv); int jt_obd_list(int argc, char **argv); int jt_obd_create(int argc, char **argv); int jt_obd_setattr(int argc, char **argv); +int jt_obd_test_setattr(int argc, char **argv); int jt_obd_destroy(int argc, char **argv); int jt_obd_getattr(int argc, char **argv); int jt_obd_test_getattr(int argc, char **argv); -- 1.8.3.1