3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see
17 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
22 * Copyright (c) 2011 Whamcloud, Inc.
23 * Use is subject to license terms.
25 * Author: Niu Yawei <niu@whamcloud.com>
28 * lustre/obdclass/lprocfs_jobstats.c
32 # define EXPORT_SYMTAB
34 #define DEBUG_SUBSYSTEM S_CLASS
37 # include <liblustre.h>
40 #include <obd_class.h>
41 #include <lprocfs_status.h>
42 #include <lustre/lustre_idl.h>
47 * JobID formats & JobID environment variable names for supported
51 * JobID format: 32 bit integer.
52 * JobID env var: SLURM_JOB_ID.
54 * JobID format: Decimal integer range to 99999.
55 * JobID env var: JOB_ID.
57 * JobID format: 6 digit integer by default (up to 999999), can be
58 * increased to 10 digit (up to 2147483646).
59 * JobID env var: LSB_JOBID.
61 * JobID format: String of machine_name.cluster_id.process_id, for
62 * example: fr2n02.32.0
63 * JobID env var: LOADL_STEP_ID.
65 * JobID format: String of sequence_number[.server_name][@server].
66 * JobID env var: PBS_JOBID.
68 * JobID format: Same as PBS.
69 * JobID env var: Same as PBS.
73 cfs_hlist_node_t js_hash;
75 cfs_atomic_t js_refcount;
76 char js_jobid[JOBSTATS_JOBID_SIZE];
77 time_t js_timestamp; /* seconds */
78 struct lprocfs_stats *js_stats;
79 struct obd_job_stats *js_jobstats;
82 static unsigned job_stat_hash(cfs_hash_t *hs, const void *key, unsigned mask)
84 return cfs_hash_djb2_hash(key, strlen(key), mask);
87 static void *job_stat_key(cfs_hlist_node_t *hnode)
90 job = cfs_hlist_entry(hnode, struct job_stat, js_hash);
94 static int job_stat_keycmp(const void *key, cfs_hlist_node_t *hnode)
97 job = cfs_hlist_entry(hnode, struct job_stat, js_hash);
98 return (strlen(job->js_jobid) == strlen(key)) &&
99 !strncmp(job->js_jobid, key, strlen(key));
102 static void *job_stat_object(cfs_hlist_node_t *hnode)
104 return cfs_hlist_entry(hnode, struct job_stat, js_hash);
107 static void job_stat_get(cfs_hash_t *hs, cfs_hlist_node_t *hnode)
109 struct job_stat *job;
110 job = cfs_hlist_entry(hnode, struct job_stat, js_hash);
111 cfs_atomic_inc(&job->js_refcount);
114 static void job_free(struct job_stat *job)
116 LASSERT(atomic_read(&job->js_refcount) == 0);
117 LASSERT(job->js_jobstats);
119 cfs_write_lock(&job->js_jobstats->ojs_lock);
120 cfs_list_del_init(&job->js_list);
121 cfs_write_unlock(&job->js_jobstats->ojs_lock);
123 lprocfs_free_stats(&job->js_stats);
127 static void job_putref(struct job_stat *job)
129 LASSERT(atomic_read(&job->js_refcount) > 0);
130 if (atomic_dec_and_test(&job->js_refcount))
134 static void job_stat_put_locked(cfs_hash_t *hs, cfs_hlist_node_t *hnode)
136 struct job_stat *job;
137 job = cfs_hlist_entry(hnode, struct job_stat, js_hash);
141 static void job_stat_exit(cfs_hash_t *hs, cfs_hlist_node_t *hnode)
143 CERROR("Should not have any items!");
146 static cfs_hash_ops_t job_stats_hash_ops = {
147 .hs_hash = job_stat_hash,
148 .hs_key = job_stat_key,
149 .hs_keycmp = job_stat_keycmp,
150 .hs_object = job_stat_object,
151 .hs_get = job_stat_get,
152 .hs_put_locked = job_stat_put_locked,
153 .hs_exit = job_stat_exit,
156 static struct job_stat *job_alloc(char *jobid, struct obd_job_stats *jobs)
158 struct job_stat *job;
160 LASSERT(jobs->ojs_cntr_num && jobs->ojs_cntr_init_fn);
166 job->js_stats = lprocfs_alloc_stats(jobs->ojs_cntr_num, 0);
167 if (job->js_stats == NULL) {
172 jobs->ojs_cntr_init_fn(job->js_stats);
174 memcpy(job->js_jobid, jobid, JOBSTATS_JOBID_SIZE);
175 job->js_timestamp = cfs_time_current_sec();
176 job->js_jobstats = jobs;
177 CFS_INIT_HLIST_NODE(&job->js_hash);
178 CFS_INIT_LIST_HEAD(&job->js_list);
179 cfs_atomic_set(&job->js_refcount, 1);
184 int lprocfs_job_stats_log(struct obd_device *obd, char *jobid,
185 int event, long amount)
187 struct obd_job_stats *stats = &obd->u.obt.obt_jobstats;
188 struct job_stat *job, *job2;
191 LASSERT(stats && stats->ojs_hash);
193 if (!jobid || !strlen(jobid))
196 if (strlen(jobid) >= JOBSTATS_JOBID_SIZE) {
197 CERROR("Invalid jobid size (%lu), expect(%d)\n",
198 (unsigned long)strlen(jobid) + 1, JOBSTATS_JOBID_SIZE);
202 job = cfs_hash_lookup(stats->ojs_hash, jobid);
206 job = job_alloc(jobid, stats);
210 job2 = cfs_hash_findadd_unique(stats->ojs_hash, job->js_jobid,
215 LASSERT(!cfs_list_empty(&job->js_list));
217 LASSERT(cfs_list_empty(&job->js_list));
218 cfs_write_lock(&stats->ojs_lock);
219 cfs_list_add_tail(&job->js_list, &stats->ojs_list);
220 cfs_write_unlock(&stats->ojs_lock);
224 LASSERT(stats == job->js_jobstats);
225 LASSERT(stats->ojs_cntr_num > event);
226 job->js_timestamp = cfs_time_current_sec();
227 lprocfs_counter_add(job->js_stats, event, amount);
232 EXPORT_SYMBOL(lprocfs_job_stats_log);
234 static int job_iter_callback(cfs_hash_t *hs, cfs_hash_bd_t *bd,
235 cfs_hlist_node_t *hnode, void *data)
237 time_t oldest = *((time_t *)data);
238 struct job_stat *job;
240 job = cfs_hlist_entry(hnode, struct job_stat, js_hash);
241 if (!oldest || job->js_timestamp < oldest)
242 cfs_hash_bd_del_locked(hs, bd, hnode);
247 void lprocfs_job_stats_fini(struct obd_device *obd)
249 struct obd_job_stats *stats = &obd->u.obt.obt_jobstats;
252 if (stats->ojs_hash == NULL)
254 cfs_timer_disarm(&stats->ojs_cleanup_timer);
255 cfs_hash_for_each_safe(stats->ojs_hash, job_iter_callback, &oldest);
256 cfs_hash_putref(stats->ojs_hash);
257 stats->ojs_hash = NULL;
258 LASSERT(cfs_list_empty(&stats->ojs_list));
260 EXPORT_SYMBOL(lprocfs_job_stats_fini);
262 static void *lprocfs_jobstats_seq_start(struct seq_file *p, loff_t *pos)
264 struct obd_job_stats *stats = p->private;
266 struct job_stat *job;
268 cfs_read_lock(&stats->ojs_lock);
270 return SEQ_START_TOKEN;
272 cfs_list_for_each_entry(job, &stats->ojs_list, js_list) {
279 static void lprocfs_jobstats_seq_stop(struct seq_file *p, void *v)
281 struct obd_job_stats *stats = p->private;
283 cfs_read_unlock(&stats->ojs_lock);
286 static void *lprocfs_jobstats_seq_next(struct seq_file *p, void *v, loff_t *pos)
288 struct obd_job_stats *stats = p->private;
289 struct job_stat *job;
293 if (v == SEQ_START_TOKEN) {
294 next = stats->ojs_list.next;
296 job = (struct job_stat *)v;
297 next = job->js_list.next;
300 return next == &stats->ojs_list ? NULL :
301 cfs_list_entry(next, struct job_stat, js_list);
305 * Example of output on MDT:
308 * - job_id: test_id.222.25844
309 * snapshot_time: 1322494486
310 * open: { samples: 3, unit: reqs }
311 * close: { samples: 3, unit: reqs }
312 * mknod: { samples: 0, unit: reqs }
313 * link: { samples: 0, unit: reqs }
314 * unlink: { samples: 0, unit: reqs }
315 * mkdir: { samples: 0, unit: reqs }
316 * rmdir: { samples: 0, unit: reqs }
317 * rename: { samples: 1, unit: reqs }
318 * getattr: { samples: 7, unit: reqs }
319 * setattr: { samples: 0, unit: reqs }
320 * getxattr: { samples: 0, unit: reqs }
321 * setxattr: { samples: 0, unit: reqs }
322 * statfs: { samples: 0, unit: reqs }
323 * sync: { samples: 0, unit: reqs }
325 * Example of output on OST:
329 * snapshot_time: 1322494602
330 * read: { samples: 0, unit: bytes, min: 0, max: 0, sum: 0 }
331 * write: { samples: 1, unit: bytes, min: 10, max: 10, sum: 10 }
332 * setattr: { samples: 0, unit: reqs }
333 * punch: { samples: 0, unit: reqs }
334 * sync: { samples: 0, unit: reqs }
337 static const char spaces[] = " ";
339 static int inline width(const char *str, int len)
341 return len - min((int)strlen(str), 15);
344 static int lprocfs_jobstats_seq_show(struct seq_file *p, void *v)
346 struct job_stat *job = v;
347 struct lprocfs_stats *s;
348 struct lprocfs_counter ret, *cntr;
351 if (v == SEQ_START_TOKEN) {
352 seq_printf(p, "job_stats:\n");
356 seq_printf(p, "- %-16s %s\n", "job_id:", job->js_jobid);
357 seq_printf(p, " %-16s %ld\n", "snapshot_time:", job->js_timestamp);
360 for (i = 0; i < s->ls_num; i++) {
361 cntr = &(s->ls_percpu[0]->lp_cntr[i]);
362 lprocfs_stats_collect(s, i, &ret);
364 seq_printf(p, " %s:%.*s { samples: %11"LPF64"u",
365 cntr->lc_name, width(cntr->lc_name, 15), spaces,
367 if (cntr->lc_units[0] != '\0')
368 seq_printf(p, ", unit: %5s", cntr->lc_units);
370 if (cntr->lc_config & LPROCFS_CNTR_AVGMINMAX) {
371 seq_printf(p, ", min:%8"LPF64"u, max:%8"LPF64"u,"
373 ret.lc_count ? ret.lc_min : 0,
374 ret.lc_count ? ret.lc_max : 0,
375 ret.lc_count ? ret.lc_sum : 0);
377 if (cntr->lc_config & LPROCFS_CNTR_STDDEV) {
378 seq_printf(p, ", sumsq: %18"LPF64"u",
379 ret.lc_count ? ret.lc_sumsquare : 0);
382 seq_printf(p, " }\n");
388 struct seq_operations lprocfs_jobstats_seq_sops = {
389 start: lprocfs_jobstats_seq_start,
390 stop: lprocfs_jobstats_seq_stop,
391 next: lprocfs_jobstats_seq_next,
392 show: lprocfs_jobstats_seq_show,
395 static int lprocfs_jobstats_seq_open(struct inode *inode, struct file *file)
397 struct proc_dir_entry *dp = PDE(inode);
398 struct seq_file *seq;
401 if (LPROCFS_ENTRY_AND_CHECK(dp))
404 rc = seq_open(file, &lprocfs_jobstats_seq_sops);
409 seq = file->private_data;
410 seq->private = dp->data;
414 static ssize_t lprocfs_jobstats_seq_write(struct file *file, const char *buf,
415 size_t len, loff_t *off)
417 struct seq_file *seq = file->private_data;
418 struct obd_job_stats *stats = seq->private;
419 char jobid[JOBSTATS_JOBID_SIZE];
421 struct job_stat *job;
423 if (!memcmp(buf, "clear", strlen("clear"))) {
425 } else if (len < JOBSTATS_JOBID_SIZE) {
426 memset(jobid, 0, JOBSTATS_JOBID_SIZE);
427 /* Trim '\n' if any */
428 if (buf[len - 1] == '\n')
429 memcpy(jobid, buf, len - 1);
431 memcpy(jobid, buf, len);
436 LASSERT(stats->ojs_hash);
439 cfs_hash_for_each_safe(stats->ojs_hash, job_iter_callback,
447 job = cfs_hash_lookup(stats->ojs_hash, jobid);
451 cfs_hash_del_key(stats->ojs_hash, jobid);
457 struct file_operations lprocfs_jobstats_seq_fops = {
458 .owner = THIS_MODULE,
459 .open = lprocfs_jobstats_seq_open,
461 .write = lprocfs_jobstats_seq_write,
463 .release = lprocfs_seq_release,
466 static void job_cleanup_callback(unsigned long data)
468 struct obd_job_stats *stats = (struct obd_job_stats *)data;
471 if (stats->ojs_cleanup_interval) {
472 oldest = cfs_time_current_sec() - stats->ojs_cleanup_interval;
473 cfs_hash_for_each_safe(stats->ojs_hash, job_iter_callback,
475 cfs_timer_arm(&stats->ojs_cleanup_timer,
476 cfs_time_shift(stats->ojs_cleanup_interval));
480 int lprocfs_job_stats_init(struct obd_device *obd, int cntr_num,
481 cntr_init_callback init_fn)
483 struct proc_dir_entry *entry;
484 struct obd_job_stats *stats;
487 LASSERT(obd->obd_proc_entry != NULL);
488 LASSERT(obd->obd_type->typ_name);
490 if (strcmp(obd->obd_type->typ_name, LUSTRE_MDT_NAME) &&
491 strcmp(obd->obd_type->typ_name, LUSTRE_OST_NAME)) {
492 CERROR("Invalid obd device type.\n");
495 stats = &obd->u.obt.obt_jobstats;
497 LASSERT(stats->ojs_hash == NULL);
498 stats->ojs_hash = cfs_hash_create("JOB_STATS",
499 HASH_JOB_STATS_CUR_BITS,
500 HASH_JOB_STATS_MAX_BITS,
501 HASH_JOB_STATS_BKT_BITS, 0,
506 if (stats->ojs_hash == NULL)
509 CFS_INIT_LIST_HEAD(&stats->ojs_list);
510 cfs_rwlock_init(&stats->ojs_lock);
511 stats->ojs_cntr_num = cntr_num;
512 stats->ojs_cntr_init_fn = init_fn;
513 cfs_timer_init(&stats->ojs_cleanup_timer, job_cleanup_callback, stats);
514 stats->ojs_cleanup_interval = 600; /* 10 mins by default */
515 cfs_timer_arm(&stats->ojs_cleanup_timer,
516 cfs_time_shift(stats->ojs_cleanup_interval));
518 LPROCFS_WRITE_ENTRY();
519 entry = create_proc_entry("job_stats", 0644, obd->obd_proc_entry);
520 LPROCFS_WRITE_EXIT();
522 entry->proc_fops = &lprocfs_jobstats_seq_fops;
526 lprocfs_job_stats_fini(obd);
530 EXPORT_SYMBOL(lprocfs_job_stats_init);
532 int lprocfs_rd_job_interval(char *page, char **start, off_t off,
533 int count, int *eof, void *data)
535 struct obd_device *obd = (struct obd_device *)data;
536 struct obd_job_stats *stats;
538 LASSERT(obd != NULL);
539 stats = &obd->u.obt.obt_jobstats;
541 return snprintf(page, count, "%d\n", stats->ojs_cleanup_interval);
543 EXPORT_SYMBOL(lprocfs_rd_job_interval);
545 int lprocfs_wr_job_interval(struct file *file, const char *buffer,
546 unsigned long count, void *data)
548 struct obd_device *obd = (struct obd_device *)data;
549 struct obd_job_stats *stats;
552 LASSERT(obd != NULL);
553 stats = &obd->u.obt.obt_jobstats;
555 rc = lprocfs_write_helper(buffer, count, &val);
559 stats->ojs_cleanup_interval = val;
560 if (!stats->ojs_cleanup_interval)
561 cfs_timer_disarm(&stats->ojs_cleanup_timer);
563 cfs_timer_arm(&stats->ojs_cleanup_timer,
564 cfs_time_shift(stats->ojs_cleanup_interval));
569 EXPORT_SYMBOL(lprocfs_wr_job_interval);