X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Futils%2Floadgen.c;h=e6cf984b36100462fbc9bbc7dd7b8e5cc8d7cee2;hp=9b974a1346fbd7cc379c79bf4da634b8cdb0995f;hb=fa507031d245b08c7f24efed32819daf2aa42ab3;hpb=0cc8befa64c34f9efba3714d455fd45662c0f12e diff --git a/lustre/utils/loadgen.c b/lustre/utils/loadgen.c index 9b974a1..e6cf984 100644 --- a/lustre/utils/loadgen.c +++ b/lustre/utils/loadgen.c @@ -1,30 +1,46 @@ /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- * vim:expandtab:shiftwidth=8:tabstop=8: * - * Copyright (C) 2006 Cluster File Systems, Inc. - * Author: Nathan Rutman + * GPL HEADER START * - * This file is part of Lustre, http://www.lustre.org. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * - * Lustre is free software; you can redistribute it and/or - * modify it under the terms of version 2 of the GNU General Public - * License as published by the Free Software Foundation. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. * - * Lustre is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). * - * You should have received a copy of the GNU General Public License - * along with Lustre; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/utils/loadgen.c * - * loadgen.c * See how many local OSCs we can start whaling on a OST * We're doing direct ioctls instead of going though a system() call to lctl * to avoid the bash overhead. * Adds an osc / echo client pair in each thread and starts echo transactions. * + * Author: Nathan Rutman */ #include @@ -42,15 +58,16 @@ #include #include -#include "parser.h" +#include #include "obdctl.h" static char cmdname[512]; static char target[64] = ""; char nid[64] = ""; -static int live_threads = 0; static int sig_received = 0; static int o_verbose = 4; /* 0-5 */ +static int my_oss = 0; +static int my_ecs = 0; static int jt_quit(int argc, char **argv) { Parser_quit(argc, argv); @@ -76,7 +93,9 @@ static int loadgen_usage(int argc, char **argv) static int loadgen_verbose(int argc, char **argv); static int loadgen_target(int argc, char **argv); +static int loadgen_start_echosrv(int argc, char **argv); static int loadgen_start_clients(int argc, char **argv); +static int loadgen_wait(int argc, char **argv); static int loadgen_write(int argc, char **argv); command_t cmdlist[] = { @@ -85,12 +104,13 @@ command_t cmdlist[] = { "usage: device []"}, {"dl", jt_obd_list, 0, "show all devices\n" "usage: dl"}, - {"start", loadgen_start_clients, 0, - "set up echo clients\n" + {"echosrv", loadgen_start_echosrv, 0, "start an echo server\n"}, + {"start", loadgen_start_clients, 0, "set up echo clients\n" "usage: start_clients "}, - {"verbose", loadgen_verbose, 0, - "set verbosity level 0-5\n" + {"verbose", loadgen_verbose, 0, "set verbosity level 0-5\n" "usage: verbose "}, + {"wait", loadgen_wait, 0, + "wait for all threads to finish\n"}, {"write", loadgen_write, 0, "start a test_brw write test on X clients for Y iterations\n" "usage: write []"}, @@ -111,7 +131,7 @@ command_t cmdlist[] = { struct command_t { int c_flags; - int c_count; + int c_rpt; int c_delay; }; @@ -124,7 +144,9 @@ struct kid_t { int k_dev; }; +static int live_threads = 0; static struct kid_t *kid_list = NULL; +pthread_mutex_t m_list = PTHREAD_MUTEX_INITIALIZER; static struct kid_t *push_kid(int tnum) { @@ -135,30 +157,38 @@ static struct kid_t *push_kid(int tnum) return NULL; } kid->k_pthread = pthread_self(); + pthread_mutex_lock(&m_list); kid->k_next = kid_list; kid->k_id = tnum; kid_list = kid; live_threads++; + pthread_mutex_unlock(&m_list); return kid; } int trigger_count = 0; int waiting_count = 0; int timer_on = 0; +int all_done = 1; struct timeval trigger_start; -struct command_t *trigger_cmd = NULL; +struct command_t trigger_cmd; pthread_mutex_t m_trigger = PTHREAD_MUTEX_INITIALIZER; pthread_cond_t cv_trigger = PTHREAD_COND_INITIALIZER; unsigned long long write_bytes; pthread_mutex_t m_count = PTHREAD_MUTEX_INITIALIZER; -static void trigger(struct command_t *cmd, int count) +static void trigger(int command, int threads, int repeat, int delay) { pthread_mutex_lock(&m_trigger); - trigger_cmd = cmd; - trigger_count = count; + trigger_cmd.c_flags = command; + trigger_cmd.c_rpt = repeat; + trigger_cmd.c_delay = delay; + trigger_count = threads; + if (o_verbose > 4) + printf("trigger %d cmd c=%d f=%x\n", trigger_count, + trigger_cmd.c_rpt, trigger_cmd.c_flags); gettimeofday(&trigger_start, NULL); timer_on = 1; pthread_mutex_lock(&m_count); @@ -176,12 +206,10 @@ static __inline__ void stop_all(int unused) static void kill_kids(void) { - struct command_t cmd; struct kid_t *tmp = kid_list; stop_all(SIGINT); - cmd.c_flags = C_STOP; - trigger(&cmd, 0); + trigger(C_STOP, 0, 0, 0); while(tmp) { pthread_kill(tmp->k_pthread, SIGTERM); tmp = tmp->k_next; @@ -239,7 +267,7 @@ static int write_proc(char *proc_path, char *value) return rc; } -static int read_proc(char *proc_path, long long *value) +static int read_proc(char *proc_path, unsigned long long *value) { int fd, rc; char buf[50]; @@ -253,9 +281,13 @@ static int read_proc(char *proc_path, long long *value) rc = read(fd, buf, sizeof(buf)); close(fd); + if (errno == EOPNOTSUPP) { + /* probably an echo server */ + return rc; + } if (rc <= 0) { fprintf(stderr, "read('%s') failed: %s (%d)\n", - proc_path, strerror(errno), rc); + proc_path, strerror(errno), errno); return rc; } *value = strtoull(buf, NULL, 10); @@ -326,7 +358,7 @@ static int cleanup(char *obdname, int quiet) return rc; } -static int setup(char *oname, char *ename, int *dev) +static int echocli_setup(char *oname, char *ename, int *dev) { char *args[5]; char proc_path[50]; @@ -448,6 +480,7 @@ static int obj_create(struct kid_t *kid) data.ioc_dev = kid->k_dev; data.ioc_obdo1.o_mode = 0100644; data.ioc_obdo1.o_id = 0; + data.ioc_obdo1.o_seq = FID_SEQ_ECHO; data.ioc_obdo1.o_uid = 0; data.ioc_obdo1.o_gid = 0; data.ioc_obdo1.o_valid = OBD_MD_FLTYPE | OBD_MD_FLMODE | @@ -461,7 +494,7 @@ static int obj_create(struct kid_t *kid) } if (!(data.ioc_obdo1.o_valid & OBD_MD_FLID)) { - fprintf(stderr,"%d: create oid not valid "LPX64"\n", + fprintf(stderr, "%d: create oid not valid "LPX64"\n", kid->k_id, data.ioc_obdo1.o_valid); return rc; } @@ -481,7 +514,7 @@ static int obj_delete(struct kid_t *kid) int rc; if (o_verbose > 4) - printf("%d: del "LPU64"\n", kid->k_id, kid->k_objid); + printf("%d: del "LPX64"\n", kid->k_id, kid->k_objid); memset(&data, 0, sizeof(data)); data.ioc_dev = kid->k_dev; @@ -491,7 +524,7 @@ static int obj_delete(struct kid_t *kid) rc = obj_ioctl(OBD_IOC_DESTROY, &data, 1); if (rc) - fprintf(stderr, "%s-%d: can't destroy obj "LPU64" (%d)\n", + fprintf(stderr, "%s-%d: can't destroy obj "LPX64" (%d)\n", cmdname, kid->k_id, kid->k_objid, rc); kid->k_objid = 0; @@ -580,7 +613,7 @@ static int do_work(struct kid_t *kid) if (!(kid->k_cmd.c_flags & C_CREATE_EVERY)) rc = obj_create(kid); - for (iter = 0; iter < kid->k_cmd.c_count; iter++) { + for (iter = 0; iter < kid->k_cmd.c_rpt; iter++) { if (rc || sig_received) break; @@ -631,14 +664,13 @@ static void report_perf() (write_bytes >> 20) / diff); pthread_mutex_unlock(&m_count); } - timer_on = 0; } static void *run_one_child(void *threadvp) { struct kid_t *kid; char oname[10], ename[10]; - int thread = (long)threadvp, dev; + int thread = (long)threadvp, dev = 0; int rc = 0, err; if (o_verbose > 2) @@ -646,7 +678,7 @@ static void *run_one_child(void *threadvp) sprintf(oname, "o%.5d", thread); sprintf(ename, "e%.5d", thread); - rc = setup(oname, ename, &dev); + rc = echocli_setup(oname, ename, &dev); if (rc) { fprintf(stderr, "%s: can't setup '%s/%s' (%d)\n", cmdname, oname, ename, rc); @@ -662,12 +694,17 @@ static void *run_one_child(void *threadvp) while(!(rc || sig_received)) { pthread_mutex_lock(&m_trigger); + pthread_mutex_lock(&m_list); waiting_count++; - if ((waiting_count == live_threads) && timer_on) + if ((waiting_count == live_threads) && timer_on) { report_perf(); - + timer_on = 0; + all_done = 1; + } + pthread_mutex_unlock(&m_list); pthread_cond_wait(&cv_trigger, &m_trigger); waiting_count--; + all_done = 0; /* First trigger_count threads will do the work, the rest will block again */ @@ -675,9 +712,9 @@ static void *run_one_child(void *threadvp) if (o_verbose > 4) printf("%d: trigger %d cmd %x\n", kid->k_id, trigger_count, - trigger_cmd->c_flags); + trigger_cmd.c_flags); trigger_count--; - kid->k_cmd = *trigger_cmd; + memcpy(&kid->k_cmd, &trigger_cmd, sizeof(trigger_cmd)); pthread_mutex_unlock(&m_trigger); rc = do_work(kid); } else { @@ -700,6 +737,19 @@ out: pthread_exit((void *)(long)rc); } +/* + * PTHREAD_STACK_MIN is 16K minimal stack for threads. This + * is stack consumed by one thread, which executes NULL procedure. + * We need some more here and 20k stack for one client thread + * is enough to not overflow. In same time it does not consume + * a lot of memory for large number of threads. + * + * 20K virtual clients will only consume 320M + 400M. Still to + * create this number of virtual clients we need to fix 8192 + * OBDs limit. + */ +#define CLIENT_THREAD_STACK_SIZE (PTHREAD_STACK_MIN + (20 * 1024)) + static int loadgen_start_clients(int argc, char **argv) { int rc = 0, i, numt; @@ -714,18 +764,18 @@ static int loadgen_start_clients(int argc, char **argv) return CMD_HELP; if (!target[0]) { - fprintf(stderr,"%s: target OST is not defined, use 'device' " + fprintf(stderr, "%s: target OST is not defined, use 'device' " "command\n", cmdname); return -EINVAL; } rc = pthread_attr_init(&attr); if (rc) { - fprintf(stderr,"%s: pthread_attr_init:(%d) %s\n", + fprintf(stderr, "%s: pthread_attr_init:(%d) %s\n", cmdname, rc, strerror(errno)); return -errno; } - pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN); + pthread_attr_setstacksize (&attr, CLIENT_THREAD_STACK_SIZE); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); numt += live_threads; @@ -825,29 +875,120 @@ static int loadgen_verbose(int argc, char **argv) static int loadgen_write(int argc, char **argv) { - struct command_t cmd; int threads; if (argc < 3 || argc > 4) return CMD_HELP; threads = atoi(argv[1]); + pthread_mutex_lock(&m_list); if (threads > live_threads) { + pthread_mutex_unlock(&m_list); fprintf(stderr, "requested %d threads but only %d are running. " "Use 'start' to start some more.\n", threads, live_threads); return -EOVERFLOW; + } else { + pthread_mutex_unlock(&m_list); + } + trigger(C_WRITE, threads, atoi(argv[2]), + (argc == 4) ? atoi(argv[3]) : 0); + return 0; +} + +char ecsname[] = "echosrv"; +static int loadgen_stop_echosrv(int argc, char **argv) +{ + int verbose = (argc != 9); + if (my_oss) { + char name[]="OSS"; + cleanup(name, verbose); + my_oss = 0; + } + if (my_ecs || (argc == 9)) { + cleanup(ecsname, verbose); + my_ecs = 0; + } + return 0; +} + +static int loadgen_start_echosrv(int argc, char **argv) +{ + char *args[5]; + int rc; + + pthread_mutex_lock(&m_config); + + args[0] = cmdname; + + /* attach obdecho echosrv echosrv_UUID */ + args[1] = "obdecho"; + args[2] = args[3] = ecsname; + rc = jt_lcfg_attach(4, args); + if (rc) { + fprintf(stderr, "%s: can't attach echo server (%d)\n", + cmdname, rc); + /* Assume we want e.g. an old one cleaned anyhow. */ + goto clean; + } + my_ecs = 1; + + /* setup */ + rc = jt_lcfg_setup(1, args); + if (rc) { + fprintf(stderr, "%s: can't setup echo server (%d)\n", + cmdname, rc); + goto clean; + } + + /* Create an OSS to handle the communications */ + /* attach ost OSS OSS_UUID */ + args[1] = "ost"; + args[2] = args[3] = "OSS"; + + rc = jt_lcfg_attach(4, args); + if (rc == EEXIST) { + /* Already set up for somebody else, that's fine. */ + printf("OSS already set up, no problem.\n"); + pthread_mutex_unlock(&m_config); + return 0; + } + if (rc) { + fprintf(stderr, "%s: can't attach OSS (%d)\n", + cmdname, rc); + goto clean; + } + my_oss = 1; + + /* setup */ + rc = jt_lcfg_setup(1, args); + if (rc) { + fprintf(stderr, "%s: can't setup OSS (%d)\n", + cmdname, rc); + goto clean; + } + + pthread_mutex_unlock(&m_config); + return rc; + +clean: + pthread_mutex_unlock(&m_config); + loadgen_stop_echosrv(9, argv); + return rc; +} + +static int loadgen_wait(int argc, char **argv) +{ + /* Give scripts a chance to start some threads */ + sleep(1); + while (!all_done) { + sleep(1); } - cmd.c_flags = C_WRITE; - cmd.c_count = atoi(argv[2]); - if (argc == 4) - cmd.c_delay = atoi(argv[3]); - trigger(&cmd, threads); return 0; } static int loadgen_init(int argc, char **argv) { - char *args[3]; + char *args[4]; int rc; sprintf(cmdname, "%s", argv[0]); @@ -877,9 +1018,13 @@ static int loadgen_init(int argc, char **argv) static int loadgen_exit() { int rc; + printf("stopping %d children\n", live_threads); kill_kids(); rc = wait_for_threads(); + + loadgen_stop_echosrv(0, NULL); + return rc; } @@ -889,6 +1034,8 @@ static int loadgen_main(int argc, char **argv) int rc; setlinebuf(stdout); + /* without this threaded errors cause segfault */ + setlinebuf(stderr); if ((rc = ptl_initialize(argc, argv)) < 0) exit(rc); @@ -913,7 +1060,7 @@ static int loadgen_main(int argc, char **argv) out: obd_finalize(argc, argv); - return rc; + return rc < 0 ? -rc : rc; } #ifndef LIBLUSTRE_TEST @@ -926,4 +1073,3 @@ int main (int argc, char **argv) return rc; } #endif -