13 #include <sys/syscall.h>
18 NSEC_PER_SEC = 1000000000L,
21 static int iabf_debug = false;
22 static const char iabf_delim[] = "---";
23 static __thread pid_t iabf_tid;
25 #define DEBUG(fmt, args...) \
28 fprintf(stderr, "DEBUG [%d] %s:%d: "fmt, iabf_tid, __func__, __LINE__, ##args); \
31 #define DEBUG_B(x) DEBUG("%s = %s\n", #x, (x) ? "true" : "false")
32 #define DEBUG_D(x) DEBUG("%s = %"PRIdMAX"\n", #x, (intmax_t)(x))
33 #define DEBUG_P(x) DEBUG("%s = %p\n", #x, (x))
34 #define DEBUG_S(x) DEBUG("%s = '%s'\n", #x, (x))
35 #define DEBUG_U(x) DEBUG("%s = %"PRIuMAX"\n", #x, (uintmax_t)(x))
36 #define DEBUG_X(x) DEBUG("%s = %"PRIxMAX"\n", #x, (uintmax_t)(x))
38 #define ERROR(fmt, args...) \
39 fprintf(stderr, "%s: "fmt, program_invocation_short_name, ##args)
41 #define FATAL(fmt, args...) \
43 ERROR("fatal: "fmt, ##args); \
47 #define xstrerror(e) strerror(abs(e))
49 static struct timespec
50 timespec_sub(struct timespec a, struct timespec b)
53 .tv_sec = a.tv_sec - b.tv_sec,
54 .tv_nsec = a.tv_nsec - b.tv_nsec,
57 while (r.tv_nsec >= NSEC_PER_SEC) {
59 r.tv_nsec -= NSEC_PER_SEC;
62 while (r.tv_nsec < 0) {
64 r.tv_nsec += NSEC_PER_SEC;
70 static struct timespec
71 timespec_from_ns(long ns)
73 return (struct timespec) {
74 .tv_sec = ns / NSEC_PER_SEC,
75 .tv_nsec = ns % NSEC_PER_SEC,
79 static long timespec_to_ns(struct timespec tv)
81 return tv.tv_sec * NSEC_PER_SEC + tv.tv_nsec;
87 cpu_set_t *ic_affinity;
88 long ic_delay_begin_ns;
90 long ic_delay_step_ns;
92 long ic_autotune_count;
94 pthread_barrier_t ic_barrier[2];
99 struct iabf_control *it_control;
102 struct timespec it_delay;
103 struct timespec it_elapsed;
107 long iabf_getenvl(const char *name, long def)
109 const char *s = getenv(name);
110 return s != NULL ? atol(s) : def;
113 static void iabf_barrier_wait(struct iabf_control *ic, int which)
117 assert(PTHREAD_BARRIER_SERIAL_THREAD == -1);
119 rc = pthread_barrier_wait(&ic->ic_barrier[which]);
121 FATAL("cannot wait on barrier: %s\n", xstrerror(rc));
124 static void *iabf_task_thread(void *data)
126 struct iabf_task *it = data;
127 struct iabf_control *ic = it->it_control;
130 iabf_tid = syscall(SYS_gettid);
132 assert(PTHREAD_BARRIER_SERIAL_THREAD == -1);
135 struct timespec ts[2];
139 iabf_barrier_wait(ic, 0);
141 DEBUG_D(ic->ic_should_stop);
142 if (ic->ic_should_stop)
145 rc = clock_nanosleep(CLOCK_MONOTONIC, 0, &it->it_delay, NULL);
147 FATAL("%s: cannot sleep: %s\n", it->it_name, xstrerror(rc));
149 rc = clock_gettime(CLOCK_MONOTONIC, &ts[0]);
151 FATAL("%s: cannot get time: %s\n", it->it_name, xstrerror(errno));
155 FATAL("%s: cannot fork: %s\n", it->it_name, strerror(errno));
158 execvpe(it->it_argv[0], it->it_argv, environ);
162 pid2 = waitpid(pid, &status, 0);
164 FATAL("%s: cannot wait for pid %d: %s\n", it->it_name, (int)pid, strerror(errno));
166 rc = clock_gettime(CLOCK_MONOTONIC, &ts[1]);
168 FATAL("%s: cannot get time: %s\n", it->it_name, xstrerror(errno));
170 it->it_elapsed = timespec_sub(ts[1], ts[0]);
174 DEBUG("%s: cmd = '%s', pid = %d, status = %d, elapsed_ns = %ld\n",
175 it->it_name, it->it_argv[0], pid, status, timespec_to_ns(it->it_elapsed));
177 if (WIFEXITED(status) && WEXITSTATUS(status) == 127)
178 FATAL("%s: command '%s' (pid %d) exited with status 127\n", it->it_name, it->it_argv[0], pid);
180 iabf_barrier_wait(ic, 1);
186 /* Run I, A+B, F once. Task threads must already be started.
188 * If delay_ns < 0 then
189 * delay exec of A by labs(delay_ns) nsec
191 * delay exec of B by labs(delay_ns) nsec.
193 static int iabf_step(struct iabf_control *ic,
194 struct iabf_task it[2],
199 if (ic->ic_init != NULL && ic->ic_init[0] != NULL) {
200 rc = callvpe(ic->ic_init[0], ic->ic_init, environ);
201 DEBUG_D(rc); /* waitpid status */
203 FATAL("initializer '%s' terminated with status %d\n", ic->ic_init[0], rc);
209 it[0].it_delay = timespec_from_ns(labs(delay_ns));
210 it[1].it_delay = timespec_from_ns(0);
212 it[0].it_delay = timespec_from_ns(0);
213 it[1].it_delay = timespec_from_ns(labs(delay_ns));
216 iabf_barrier_wait(ic, 0);
220 iabf_barrier_wait(ic, 1);
222 if (ic->ic_fini != NULL && ic->ic_fini[0] != NULL) {
223 rc = callvpe(ic->ic_fini[0], ic->ic_fini, environ);
224 DEBUG_D(rc); /* waitpid status */
226 FATAL("finalizer '%s' terminated with status %d\n", ic->ic_fini[0], rc);
232 /* Run (I, A+B, F) step $IABF_AUTOTUNE_COUNT times. Task threads must
233 * already be started. Get the average elapsed times for A and B. We
234 * want to choose delay_begin and delay_end to try to arrange all
235 * possible overlaps given the expected elapsed times of A and B.
237 * AAAAAAAAAA delay(A) is approx elapsed(B)
238 * BBBBB delay(B) == 0
240 * AAAAAAAAAA delay(A) == 0
241 * BBBBB delay(B) == 0
243 * AAAAAAAAAA delay(A) == 0
244 * BBBBB delay(B) is approx elapsed(A)
246 * Note that to delay task A we use a negative delay_ns.
248 static int iabf_autotune(struct iabf_control *ic,
249 struct iabf_task it[2])
251 long elapsed_ns[2] = { 0, 0 };
254 DEBUG("begin autotune\n");
256 assert(ic->ic_autotune_count >= 0);
258 if (ic->ic_autotune_count == 0)
261 for (i = 0; i < ic->ic_autotune_count; i++) {
262 iabf_step(ic, it, 0);
264 for (j = 0; j < 2; j++)
265 elapsed_ns[j] += timespec_to_ns(it[j].it_elapsed);
268 elapsed_ns[0] /= ic->ic_autotune_count;
269 elapsed_ns[1] /= ic->ic_autotune_count;
271 DEBUG_D(elapsed_ns[0]);
272 DEBUG_D(elapsed_ns[1]);
274 assert(0 <= elapsed_ns[0]);
275 assert(0 <= elapsed_ns[1]);
277 /* TODO Apply a multiplier to endpoints. */
279 if (ic->ic_delay_begin_ns == LONG_MIN)
280 ic->ic_delay_begin_ns = -elapsed_ns[1];
282 if (ic->ic_delay_end_ns == LONG_MAX)
283 ic->ic_delay_end_ns = +elapsed_ns[0];
285 assert(ic->ic_delay_begin_ns <= ic->ic_delay_end_ns);
286 assert(0 <= ic->ic_step_count);
288 if (ic->ic_step_count != 0)
289 ic->ic_delay_step_ns = (ic->ic_delay_end_ns - ic->ic_delay_begin_ns) / ic->ic_step_count;
291 if (ic->ic_delay_step_ns == 0)
292 ic->ic_delay_step_ns = 1; /* Or just leave it 0? */
294 DEBUG("end autotune\n");
299 /* Start A and B threads, autotune delay parameters if needed, run
300 * iabf_step() however many times. */
301 static int iabf(struct iabf_control *ic, char **a, char **b)
303 struct iabf_task it[2] = {
315 pthread_attr_t attr_, *attr = NULL;
319 rc = pthread_attr_init(&attr_);
321 FATAL("cannot initialize thread attributes: %s\n", xstrerror(rc));
325 for (i = 0; i < 2; i++) {
326 rc = pthread_barrier_init(&ic->ic_barrier[i], NULL, 3);
328 FATAL("cannot initialize barrier: %s\n", xstrerror(rc));
331 if (ic->ic_affinity != NULL) {
332 rc = pthread_setaffinity_np(pthread_self(), sizeof(ic->ic_affinity[2]), &ic->ic_affinity[2]);
334 FATAL("cannot set CPU affinity : %s\n", xstrerror(rc));
337 for (i = 0; i < 2; i++) {
338 if (ic->ic_affinity != NULL) {
339 rc = pthread_attr_setaffinity_np(attr, sizeof(ic->ic_affinity[i]), &ic->ic_affinity[i]);
341 FATAL("cannot set thread attr CPU affinity : %s\n", xstrerror(rc));
344 rc = pthread_create(&it[i].it_thread,
349 FATAL("cannot create thread: %s\n", xstrerror(rc));
352 if (ic->ic_delay_begin_ns == LONG_MIN ||
353 ic->ic_delay_end_ns == LONG_MAX ||
354 ic->ic_delay_step_ns == 0)
355 iabf_autotune(ic, it);
357 DEBUG_D(ic->ic_delay_begin_ns);
358 DEBUG_D(ic->ic_delay_end_ns);
359 DEBUG_D(ic->ic_delay_step_ns);
362 for (delay_ns = ic->ic_delay_begin_ns;
363 delay_ns < ic->ic_delay_end_ns;
364 delay_ns += ic->ic_delay_step_ns)
365 iabf_step(ic, it, delay_ns);
367 ic->ic_should_stop = 1;
368 DEBUG_D(ic->ic_should_stop);
370 iabf_barrier_wait(ic, 0);
372 for (i = 0; i < 2; i++) {
373 rc = pthread_join(it[i].it_thread, NULL);
375 FATAL("cannot join thread %s: %s\n", it[i].it_name, xstrerror(rc));
378 for (i = 0; i < 2; i++) {
379 rc = pthread_barrier_destroy(&ic->ic_barrier[i]);
381 FATAL("cannot destroy barrier: %s\n", xstrerror(rc));
385 pthread_attr_destroy(attr);
390 /* strsep() for argvs */
391 char **arg_sep(char ***pargs, const char *delim)
399 /* Find the end of the token. */
400 /* end = begin + strcspn (begin, delim); */
402 for (end = begin; *end != NULL && strcmp(*end, delim) != 0; end++)
406 /* Terminate the token and set *STRINGP past NUL character. */
410 /* No more delimiters; this is the last token. */
417 static cpu_set_t *iabf_affinity(const char *str)
419 cpu_set_t *cpu_sets = NULL;
429 cpu_sets = calloc(3, sizeof(cpu_sets[0]));
430 p = str1 = strdup(str);
432 for (i = 0; i < 3; i++) {
433 CPU_ZERO(&cpu_sets[i]);
437 FATAL("invalid affinity '%s'\n", str);
439 while ((r = strsep(&q, ",")) != NULL)
440 CPU_SET(atoi(r), &cpu_sets[i]);
444 FATAL("invalid affinity '%s'\n", str);
451 int main(int argc, char **argv)
453 struct iabf_control ic = {
456 char **args = argv + 1;
460 iabf_tid = syscall(SYS_gettid);
462 iabf_debug = atoi(getenv("IABF_DEBUG") ?: "0");
464 ic.ic_init = arg_sep(&args, iabf_delim);
465 a = arg_sep(&args, iabf_delim);
466 b = arg_sep(&args, iabf_delim);
467 ic.ic_fini = arg_sep(&args, iabf_delim);
469 if (ic.ic_init == NULL ||
473 FATAL("missing '%s' in argv\n", iabf_delim);
476 for (i = 0; ic.ic_init[i] != NULL; i++)
477 DEBUG_S(ic.ic_init[i]);
479 for (i = 0; a[i] != NULL; i++)
482 for (i = 0; b[i] != NULL; i++)
485 for (i = 0; ic.ic_fini[i] != NULL; i++)
486 DEBUG_S(ic.ic_fini[i]);
488 ic.ic_affinity = iabf_affinity(getenv("IABF_AFFINITY"));
489 ic.ic_delay_begin_ns = iabf_getenvl("IABF_DELAY_BEGIN_NS", LONG_MIN);
490 ic.ic_delay_end_ns = iabf_getenvl("IABF_DELAY_END_NS", LONG_MAX);
491 ic.ic_delay_step_ns = iabf_getenvl("IABF_DELAY_STEP_NS", 0);
492 ic.ic_step_count = iabf_getenvl("IABF_STEP_COUNT", 0);
493 ic.ic_autotune_count = iabf_getenvl("IABF_AUTOTUNE_COUNT", 16);
495 DEBUG_D(ic.ic_delay_begin_ns);
496 DEBUG_D(ic.ic_delay_end_ns);
497 DEBUG_D(ic.ic_delay_step_ns);
498 DEBUG_D(ic.ic_step_count);
499 DEBUG_D(ic.ic_autotune_count);
501 assert(ic.ic_delay_begin_ns <= ic.ic_delay_end_ns);
502 assert(0 <= ic.ic_delay_step_ns);
503 assert(0 <= ic.ic_step_count);