/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
* vim:expandtab:shiftwidth=8:tabstop=8:
*
- * Copyright (C) 2006 Cluster File Systems, Inc.
- * Author: Nathan Rutman <nathan@clusterfs.com>
+ * GPL HEADER START
*
- * This file is part of Lustre, http://www.lustre.org.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
- * Lustre is free software; you can redistribute it and/or
- * modify it under the terms of version 2 of the GNU General Public
- * License as published by the Free Software Foundation.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
*
- * Lustre is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
*
- * You should have received a copy of the GNU General Public License
- * along with Lustre; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved
+ * Use is subject to license terms.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ *
+ * lustre/utils/loadgen.c
*
- * loadgen.c
* See how many local OSCs we can start whaling on a OST
* We're doing direct ioctls instead of going though a system() call to lctl
* to avoid the bash overhead.
* Adds an osc / echo client pair in each thread and starts echo transactions.
*
+ * Author: Nathan Rutman <nathan@clusterfs.com>
*/
#include <pthread.h>
#include <sys/time.h>
#include <lnet/lnetctl.h>
-#include "parser.h"
+#include <libcfs/libcfsutil.h>
#include "obdctl.h"
static char cmdname[512];
static char target[64] = "";
char nid[64] = "";
-static int live_threads = 0;
static int sig_received = 0;
static int o_verbose = 4; /* 0-5 */
+static int my_oss = 0;
+static int my_ecs = 0;
static int jt_quit(int argc, char **argv) {
Parser_quit(argc, argv);
static int loadgen_verbose(int argc, char **argv);
static int loadgen_target(int argc, char **argv);
+static int loadgen_start_echosrv(int argc, char **argv);
static int loadgen_start_clients(int argc, char **argv);
+static int loadgen_wait(int argc, char **argv);
static int loadgen_write(int argc, char **argv);
command_t cmdlist[] = {
"usage: device <name> [<nid>]"},
{"dl", jt_obd_list, 0, "show all devices\n"
"usage: dl"},
- {"start", loadgen_start_clients, 0,
- "set up echo clients\n"
+ {"echosrv", loadgen_start_echosrv, 0, "start an echo server\n"},
+ {"start", loadgen_start_clients, 0, "set up echo clients\n"
"usage: start_clients <num>"},
- {"verbose", loadgen_verbose, 0,
- "set verbosity level 0-5\n"
+ {"verbose", loadgen_verbose, 0, "set verbosity level 0-5\n"
"usage: verbose <level>"},
+ {"wait", loadgen_wait, 0,
+ "wait for all threads to finish\n"},
{"write", loadgen_write, 0,
"start a test_brw write test on X clients for Y iterations\n"
"usage: write <num_clients> <num_iter> [<delay>]"},
struct command_t {
int c_flags;
- int c_count;
+ int c_rpt;
int c_delay;
};
int k_dev;
};
+static int live_threads = 0;
static struct kid_t *kid_list = NULL;
+pthread_mutex_t m_list = PTHREAD_MUTEX_INITIALIZER;
static struct kid_t *push_kid(int tnum)
{
return NULL;
}
kid->k_pthread = pthread_self();
+ pthread_mutex_lock(&m_list);
kid->k_next = kid_list;
kid->k_id = tnum;
kid_list = kid;
live_threads++;
+ pthread_mutex_unlock(&m_list);
return kid;
}
int trigger_count = 0;
int waiting_count = 0;
int timer_on = 0;
+int all_done = 1;
struct timeval trigger_start;
-struct command_t *trigger_cmd = NULL;
+struct command_t trigger_cmd;
pthread_mutex_t m_trigger = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t cv_trigger = PTHREAD_COND_INITIALIZER;
unsigned long long write_bytes;
pthread_mutex_t m_count = PTHREAD_MUTEX_INITIALIZER;
-static void trigger(struct command_t *cmd, int count)
+static void trigger(int command, int threads, int repeat, int delay)
{
pthread_mutex_lock(&m_trigger);
- trigger_cmd = cmd;
- trigger_count = count;
+ trigger_cmd.c_flags = command;
+ trigger_cmd.c_rpt = repeat;
+ trigger_cmd.c_delay = delay;
+ trigger_count = threads;
+ if (o_verbose > 4)
+ printf("trigger %d cmd c=%d f=%x\n", trigger_count,
+ trigger_cmd.c_rpt, trigger_cmd.c_flags);
gettimeofday(&trigger_start, NULL);
timer_on = 1;
pthread_mutex_lock(&m_count);
static void kill_kids(void)
{
- struct command_t cmd;
struct kid_t *tmp = kid_list;
stop_all(SIGINT);
- cmd.c_flags = C_STOP;
- trigger(&cmd, 0);
+ trigger(C_STOP, 0, 0, 0);
while(tmp) {
pthread_kill(tmp->k_pthread, SIGTERM);
tmp = tmp->k_next;
return rc;
}
-static int read_proc(char *proc_path, long long *value)
+static int read_proc(char *proc_path, unsigned long long *value)
{
int fd, rc;
char buf[50];
rc = read(fd, buf, sizeof(buf));
close(fd);
+ if (errno == EOPNOTSUPP) {
+ /* probably an echo server */
+ return rc;
+ }
if (rc <= 0) {
fprintf(stderr, "read('%s') failed: %s (%d)\n",
- proc_path, strerror(errno), rc);
+ proc_path, strerror(errno), errno);
return rc;
}
*value = strtoull(buf, NULL, 10);
return rc;
}
-static int setup(char *oname, char *ename, int *dev)
+static int echocli_setup(char *oname, char *ename, int *dev)
{
char *args[5];
char proc_path[50];
data.ioc_dev = kid->k_dev;
data.ioc_obdo1.o_mode = 0100644;
data.ioc_obdo1.o_id = 0;
+ data.ioc_obdo1.o_gr = 2;
data.ioc_obdo1.o_uid = 0;
data.ioc_obdo1.o_gid = 0;
data.ioc_obdo1.o_valid = OBD_MD_FLTYPE | OBD_MD_FLMODE |
}
if (!(data.ioc_obdo1.o_valid & OBD_MD_FLID)) {
- fprintf(stderr,"%d: create oid not valid "LPX64"\n",
+ fprintf(stderr, "%d: create oid not valid "LPX64"\n",
kid->k_id, data.ioc_obdo1.o_valid);
return rc;
}
int rc;
if (o_verbose > 4)
- printf("%d: del "LPU64"\n", kid->k_id, kid->k_objid);
+ printf("%d: del "LPX64"\n", kid->k_id, kid->k_objid);
memset(&data, 0, sizeof(data));
data.ioc_dev = kid->k_dev;
rc = obj_ioctl(OBD_IOC_DESTROY, &data, 1);
if (rc)
- fprintf(stderr, "%s-%d: can't destroy obj "LPU64" (%d)\n",
+ fprintf(stderr, "%s-%d: can't destroy obj "LPX64" (%d)\n",
cmdname, kid->k_id, kid->k_objid, rc);
kid->k_objid = 0;
if (!(kid->k_cmd.c_flags & C_CREATE_EVERY))
rc = obj_create(kid);
- for (iter = 0; iter < kid->k_cmd.c_count; iter++) {
+ for (iter = 0; iter < kid->k_cmd.c_rpt; iter++) {
if (rc || sig_received)
break;
(write_bytes >> 20) / diff);
pthread_mutex_unlock(&m_count);
}
- timer_on = 0;
}
static void *run_one_child(void *threadvp)
{
struct kid_t *kid;
char oname[10], ename[10];
- int thread = (long)threadvp, dev;
+ int thread = (long)threadvp, dev = 0;
int rc = 0, err;
if (o_verbose > 2)
sprintf(oname, "o%.5d", thread);
sprintf(ename, "e%.5d", thread);
- rc = setup(oname, ename, &dev);
+ rc = echocli_setup(oname, ename, &dev);
if (rc) {
fprintf(stderr, "%s: can't setup '%s/%s' (%d)\n",
cmdname, oname, ename, rc);
while(!(rc || sig_received)) {
pthread_mutex_lock(&m_trigger);
+ pthread_mutex_lock(&m_list);
waiting_count++;
- if ((waiting_count == live_threads) && timer_on)
+ if ((waiting_count == live_threads) && timer_on) {
report_perf();
-
+ timer_on = 0;
+ all_done = 1;
+ }
+ pthread_mutex_unlock(&m_list);
pthread_cond_wait(&cv_trigger, &m_trigger);
waiting_count--;
+ all_done = 0;
/* First trigger_count threads will do the work, the rest
will block again */
if (o_verbose > 4)
printf("%d: trigger %d cmd %x\n",
kid->k_id, trigger_count,
- trigger_cmd->c_flags);
+ trigger_cmd.c_flags);
trigger_count--;
- kid->k_cmd = *trigger_cmd;
+ memcpy(&kid->k_cmd, &trigger_cmd, sizeof(trigger_cmd));
pthread_mutex_unlock(&m_trigger);
rc = do_work(kid);
} else {
pthread_exit((void *)(long)rc);
}
+/*
+ * PTHREAD_STACK_MIN is 16K minimal stack for threads. This
+ * is stack consumed by one thread, which executes NULL procedure.
+ * We need some more here and 20k stack for one client thread
+ * is enough to not overflow. In same time it does not consume
+ * a lot of memory for large number of threads.
+ *
+ * 20K virtual clients will only consume 320M + 400M. Still to
+ * create this number of virtual clients we need to fix 8192
+ * OBDs limit.
+ */
+#define CLIENT_THREAD_STACK_SIZE (PTHREAD_STACK_MIN + (20 * 1024))
+
static int loadgen_start_clients(int argc, char **argv)
{
int rc = 0, i, numt;
return CMD_HELP;
if (!target[0]) {
- fprintf(stderr,"%s: target OST is not defined, use 'device' "
+ fprintf(stderr, "%s: target OST is not defined, use 'device' "
"command\n", cmdname);
return -EINVAL;
}
rc = pthread_attr_init(&attr);
if (rc) {
- fprintf(stderr,"%s: pthread_attr_init:(%d) %s\n",
+ fprintf(stderr, "%s: pthread_attr_init:(%d) %s\n",
cmdname, rc, strerror(errno));
return -errno;
}
- pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN);
+ pthread_attr_setstacksize (&attr, CLIENT_THREAD_STACK_SIZE);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
numt += live_threads;
static int loadgen_write(int argc, char **argv)
{
- struct command_t cmd;
int threads;
if (argc < 3 || argc > 4)
return CMD_HELP;
threads = atoi(argv[1]);
+ pthread_mutex_lock(&m_list);
if (threads > live_threads) {
+ pthread_mutex_unlock(&m_list);
fprintf(stderr, "requested %d threads but only %d are running. "
"Use 'start' to start some more.\n",
threads, live_threads);
return -EOVERFLOW;
+ } else {
+ pthread_mutex_unlock(&m_list);
+ }
+ trigger(C_WRITE, threads, atoi(argv[2]),
+ (argc == 4) ? atoi(argv[3]) : 0);
+ return 0;
+}
+
+char ecsname[] = "echosrv";
+static int loadgen_stop_echosrv(int argc, char **argv)
+{
+ int verbose = (argc != 9);
+ if (my_oss) {
+ char name[]="OSS";
+ cleanup(name, verbose);
+ my_oss = 0;
+ }
+ if (my_ecs || (argc == 9)) {
+ cleanup(ecsname, verbose);
+ my_ecs = 0;
+ }
+ return 0;
+}
+
+static int loadgen_start_echosrv(int argc, char **argv)
+{
+ char *args[5];
+ int rc;
+
+ pthread_mutex_lock(&m_config);
+
+ args[0] = cmdname;
+
+ /* attach obdecho echosrv echosrv_UUID */
+ args[1] = "obdecho";
+ args[2] = args[3] = ecsname;
+ rc = jt_lcfg_attach(4, args);
+ if (rc) {
+ fprintf(stderr, "%s: can't attach echo server (%d)\n",
+ cmdname, rc);
+ /* Assume we want e.g. an old one cleaned anyhow. */
+ goto clean;
+ }
+ my_ecs = 1;
+
+ /* setup */
+ rc = jt_lcfg_setup(1, args);
+ if (rc) {
+ fprintf(stderr, "%s: can't setup echo server (%d)\n",
+ cmdname, rc);
+ goto clean;
+ }
+
+ /* Create an OSS to handle the communications */
+ /* attach ost OSS OSS_UUID */
+ args[1] = "ost";
+ args[2] = args[3] = "OSS";
+
+ rc = jt_lcfg_attach(4, args);
+ if (rc == EEXIST) {
+ /* Already set up for somebody else, that's fine. */
+ printf("OSS already set up, no problem.\n");
+ pthread_mutex_unlock(&m_config);
+ return 0;
+ }
+ if (rc) {
+ fprintf(stderr, "%s: can't attach OSS (%d)\n",
+ cmdname, rc);
+ goto clean;
+ }
+ my_oss = 1;
+
+ /* setup */
+ rc = jt_lcfg_setup(1, args);
+ if (rc) {
+ fprintf(stderr, "%s: can't setup OSS (%d)\n",
+ cmdname, rc);
+ goto clean;
+ }
+
+ pthread_mutex_unlock(&m_config);
+ return rc;
+
+clean:
+ pthread_mutex_unlock(&m_config);
+ loadgen_stop_echosrv(9, argv);
+ return rc;
+}
+
+static int loadgen_wait(int argc, char **argv)
+{
+ /* Give scripts a chance to start some threads */
+ sleep(1);
+ while (!all_done) {
+ sleep(1);
}
- cmd.c_flags = C_WRITE;
- cmd.c_count = atoi(argv[2]);
- if (argc == 4)
- cmd.c_delay = atoi(argv[3]);
- trigger(&cmd, threads);
return 0;
}
static int loadgen_init(int argc, char **argv)
{
- char *args[3];
+ char *args[4];
int rc;
sprintf(cmdname, "%s", argv[0]);
static int loadgen_exit()
{
int rc;
+
printf("stopping %d children\n", live_threads);
kill_kids();
rc = wait_for_threads();
+
+ loadgen_stop_echosrv(0, NULL);
+
return rc;
}
int rc;
setlinebuf(stdout);
+ /* without this threaded errors cause segfault */
+ setlinebuf(stderr);
if ((rc = ptl_initialize(argc, argv)) < 0)
exit(rc);
out:
obd_finalize(argc, argv);
- return rc;
+ return rc < 0 ? -rc : rc;
}
#ifndef LIBLUSTRE_TEST
return rc;
}
#endif
-