3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see
17 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
22 * Copyright (c) 2012, 2013, Intel Corporation.
23 * Use is subject to license terms.
25 * Author: Niu Yawei <niu@whamcloud.com>
28 * lustre/obdclass/lprocfs_jobstats.c
31 #define DEBUG_SUBSYSTEM S_CLASS
34 # include <liblustre.h>
37 #include <obd_class.h>
38 #include <lprocfs_status.h>
39 #include <lustre/lustre_idl.h>
44 * JobID formats & JobID environment variable names for supported
48 * JobID format: 32 bit integer.
49 * JobID env var: SLURM_JOB_ID.
51 * JobID format: Decimal integer range to 99999.
52 * JobID env var: JOB_ID.
54 * JobID format: 6 digit integer by default (up to 999999), can be
55 * increased to 10 digit (up to 2147483646).
56 * JobID env var: LSB_JOBID.
58 * JobID format: String of machine_name.cluster_id.process_id, for
59 * example: fr2n02.32.0
60 * JobID env var: LOADL_STEP_ID.
62 * JobID format: String of sequence_number[.server_name][@server].
63 * JobID env var: PBS_JOBID.
65 * JobID format: Same as PBS.
66 * JobID env var: Same as PBS.
70 struct hlist_node js_hash;
71 struct list_head js_list;
73 char js_jobid[JOBSTATS_JOBID_SIZE];
74 time_t js_timestamp; /* seconds */
75 struct lprocfs_stats *js_stats;
76 struct obd_job_stats *js_jobstats;
79 static unsigned job_stat_hash(cfs_hash_t *hs, const void *key, unsigned mask)
81 return cfs_hash_djb2_hash(key, strlen(key), mask);
84 static void *job_stat_key(struct hlist_node *hnode)
87 job = hlist_entry(hnode, struct job_stat, js_hash);
91 static int job_stat_keycmp(const void *key, struct hlist_node *hnode)
94 job = hlist_entry(hnode, struct job_stat, js_hash);
95 return (strlen(job->js_jobid) == strlen(key)) &&
96 !strncmp(job->js_jobid, key, strlen(key));
99 static void *job_stat_object(struct hlist_node *hnode)
101 return hlist_entry(hnode, struct job_stat, js_hash);
104 static void job_stat_get(cfs_hash_t *hs, struct hlist_node *hnode)
106 struct job_stat *job;
107 job = hlist_entry(hnode, struct job_stat, js_hash);
108 atomic_inc(&job->js_refcount);
111 static void job_free(struct job_stat *job)
113 LASSERT(atomic_read(&job->js_refcount) == 0);
114 LASSERT(job->js_jobstats);
116 write_lock(&job->js_jobstats->ojs_lock);
117 list_del_init(&job->js_list);
118 write_unlock(&job->js_jobstats->ojs_lock);
120 lprocfs_free_stats(&job->js_stats);
124 static void job_putref(struct job_stat *job)
126 LASSERT(atomic_read(&job->js_refcount) > 0);
127 if (atomic_dec_and_test(&job->js_refcount))
131 static void job_stat_put_locked(cfs_hash_t *hs, struct hlist_node *hnode)
133 struct job_stat *job;
134 job = hlist_entry(hnode, struct job_stat, js_hash);
138 static void job_stat_exit(cfs_hash_t *hs, struct hlist_node *hnode)
140 CERROR("should not have any items\n");
143 static cfs_hash_ops_t job_stats_hash_ops = {
144 .hs_hash = job_stat_hash,
145 .hs_key = job_stat_key,
146 .hs_keycmp = job_stat_keycmp,
147 .hs_object = job_stat_object,
148 .hs_get = job_stat_get,
149 .hs_put_locked = job_stat_put_locked,
150 .hs_exit = job_stat_exit,
153 static int job_iter_callback(cfs_hash_t *hs, cfs_hash_bd_t *bd,
154 struct hlist_node *hnode, void *data)
156 time_t oldest = *((time_t *)data);
157 struct job_stat *job;
159 job = hlist_entry(hnode, struct job_stat, js_hash);
160 if (!oldest || job->js_timestamp < oldest)
161 cfs_hash_bd_del_locked(hs, bd, hnode);
166 static void lprocfs_job_cleanup(struct obd_job_stats *stats, bool force)
170 if (stats->ojs_cleanup_interval == 0)
173 now = cfs_time_current_sec();
174 if (!force && now < stats->ojs_last_cleanup +
175 stats->ojs_cleanup_interval)
178 oldest = now - stats->ojs_cleanup_interval;
179 cfs_hash_for_each_safe(stats->ojs_hash, job_iter_callback,
181 stats->ojs_last_cleanup = cfs_time_current_sec();
184 static struct job_stat *job_alloc(char *jobid, struct obd_job_stats *jobs)
186 struct job_stat *job;
188 LASSERT(jobs->ojs_cntr_num && jobs->ojs_cntr_init_fn);
194 job->js_stats = lprocfs_alloc_stats(jobs->ojs_cntr_num, 0);
195 if (job->js_stats == NULL) {
200 jobs->ojs_cntr_init_fn(job->js_stats);
202 memcpy(job->js_jobid, jobid, JOBSTATS_JOBID_SIZE);
203 job->js_timestamp = cfs_time_current_sec();
204 job->js_jobstats = jobs;
205 INIT_HLIST_NODE(&job->js_hash);
206 INIT_LIST_HEAD(&job->js_list);
207 atomic_set(&job->js_refcount, 1);
212 int lprocfs_job_stats_log(struct obd_device *obd, char *jobid,
213 int event, long amount)
215 struct obd_job_stats *stats = &obd->u.obt.obt_jobstats;
216 struct job_stat *job, *job2;
219 LASSERT(stats && stats->ojs_hash);
221 lprocfs_job_cleanup(stats, false);
223 if (!jobid || !strlen(jobid))
226 if (strlen(jobid) >= JOBSTATS_JOBID_SIZE) {
227 CERROR("Invalid jobid size (%lu), expect(%d)\n",
228 (unsigned long)strlen(jobid) + 1, JOBSTATS_JOBID_SIZE);
232 job = cfs_hash_lookup(stats->ojs_hash, jobid);
236 job = job_alloc(jobid, stats);
240 job2 = cfs_hash_findadd_unique(stats->ojs_hash, job->js_jobid,
245 /* We cannot LASSERT(!list_empty(&job->js_list)) here,
246 * since we just lost the race for inserting "job" into the
247 * ojs_list, and some other thread is doing it _right_now_.
248 * Instead, be content the other thread is doing this, since
249 * "job2" was initialized in job_alloc() already. LU-2163 */
251 LASSERT(list_empty(&job->js_list));
252 write_lock(&stats->ojs_lock);
253 list_add_tail(&job->js_list, &stats->ojs_list);
254 write_unlock(&stats->ojs_lock);
258 LASSERT(stats == job->js_jobstats);
259 LASSERT(stats->ojs_cntr_num > event);
260 job->js_timestamp = cfs_time_current_sec();
261 lprocfs_counter_add(job->js_stats, event, amount);
266 EXPORT_SYMBOL(lprocfs_job_stats_log);
268 void lprocfs_job_stats_fini(struct obd_device *obd)
270 struct obd_job_stats *stats = &obd->u.obt.obt_jobstats;
273 if (stats->ojs_hash == NULL)
275 cfs_hash_for_each_safe(stats->ojs_hash, job_iter_callback, &oldest);
276 cfs_hash_putref(stats->ojs_hash);
277 stats->ojs_hash = NULL;
278 LASSERT(list_empty(&stats->ojs_list));
280 EXPORT_SYMBOL(lprocfs_job_stats_fini);
282 static void *lprocfs_jobstats_seq_start(struct seq_file *p, loff_t *pos)
284 struct obd_job_stats *stats = p->private;
286 struct job_stat *job;
288 read_lock(&stats->ojs_lock);
290 return SEQ_START_TOKEN;
292 list_for_each_entry(job, &stats->ojs_list, js_list) {
299 static void lprocfs_jobstats_seq_stop(struct seq_file *p, void *v)
301 struct obd_job_stats *stats = p->private;
303 read_unlock(&stats->ojs_lock);
306 static void *lprocfs_jobstats_seq_next(struct seq_file *p, void *v, loff_t *pos)
308 struct obd_job_stats *stats = p->private;
309 struct job_stat *job;
310 struct list_head *next;
313 if (v == SEQ_START_TOKEN) {
314 next = stats->ojs_list.next;
316 job = (struct job_stat *)v;
317 next = job->js_list.next;
320 return next == &stats->ojs_list ? NULL :
321 list_entry(next, struct job_stat, js_list);
325 * Example of output on MDT:
328 * - job_id: test_id.222.25844
329 * snapshot_time: 1322494486
330 * open: { samples: 3, unit: reqs }
331 * close: { samples: 3, unit: reqs }
332 * mknod: { samples: 0, unit: reqs }
333 * link: { samples: 0, unit: reqs }
334 * unlink: { samples: 0, unit: reqs }
335 * mkdir: { samples: 0, unit: reqs }
336 * rmdir: { samples: 0, unit: reqs }
337 * rename: { samples: 1, unit: reqs }
338 * getattr: { samples: 7, unit: reqs }
339 * setattr: { samples: 0, unit: reqs }
340 * getxattr: { samples: 0, unit: reqs }
341 * setxattr: { samples: 0, unit: reqs }
342 * statfs: { samples: 0, unit: reqs }
343 * sync: { samples: 0, unit: reqs }
345 * Example of output on OST:
349 * snapshot_time: 1322494602
350 * read: { samples: 0, unit: bytes, min: 0, max: 0, sum: 0 }
351 * write: { samples: 1, unit: bytes, min: 10, max: 10, sum: 10 }
352 * setattr: { samples: 0, unit: reqs }
353 * punch: { samples: 0, unit: reqs }
354 * sync: { samples: 0, unit: reqs }
357 static const char spaces[] = " ";
359 static int inline width(const char *str, int len)
361 return len - min((int)strlen(str), 15);
364 static int lprocfs_jobstats_seq_show(struct seq_file *p, void *v)
366 struct job_stat *job = v;
367 struct lprocfs_stats *s;
368 struct lprocfs_counter ret;
369 struct lprocfs_counter_header *cntr_header;
372 if (v == SEQ_START_TOKEN) {
373 seq_printf(p, "job_stats:\n");
377 seq_printf(p, "- %-16s %s\n", "job_id:", job->js_jobid);
378 seq_printf(p, " %-16s %ld\n", "snapshot_time:", job->js_timestamp);
381 for (i = 0; i < s->ls_num; i++) {
382 cntr_header = &s->ls_cnt_header[i];
383 lprocfs_stats_collect(s, i, &ret);
385 seq_printf(p, " %s:%.*s { samples: %11"LPF64"u",
386 cntr_header->lc_name,
387 width(cntr_header->lc_name, 15), spaces,
389 if (cntr_header->lc_units[0] != '\0')
390 seq_printf(p, ", unit: %5s", cntr_header->lc_units);
392 if (cntr_header->lc_config & LPROCFS_CNTR_AVGMINMAX) {
393 seq_printf(p, ", min:%8"LPF64"u, max:%8"LPF64"u,"
395 ret.lc_count ? ret.lc_min : 0,
396 ret.lc_count ? ret.lc_max : 0,
397 ret.lc_count ? ret.lc_sum : 0);
399 if (cntr_header->lc_config & LPROCFS_CNTR_STDDEV) {
400 seq_printf(p, ", sumsq: %18"LPF64"u",
401 ret.lc_count ? ret.lc_sumsquare : 0);
404 seq_printf(p, " }\n");
410 struct seq_operations lprocfs_jobstats_seq_sops = {
411 start: lprocfs_jobstats_seq_start,
412 stop: lprocfs_jobstats_seq_stop,
413 next: lprocfs_jobstats_seq_next,
414 show: lprocfs_jobstats_seq_show,
417 static int lprocfs_jobstats_seq_open(struct inode *inode, struct file *file)
419 struct seq_file *seq;
422 if (LPROCFS_ENTRY_CHECK(PDE(inode)))
425 rc = seq_open(file, &lprocfs_jobstats_seq_sops);
428 seq = file->private_data;
429 seq->private = PDE_DATA(inode);
433 static ssize_t lprocfs_jobstats_seq_write(struct file *file,
434 const char __user *buf,
435 size_t len, loff_t *off)
437 struct seq_file *seq = file->private_data;
438 struct obd_job_stats *stats = seq->private;
439 char jobid[JOBSTATS_JOBID_SIZE];
441 struct job_stat *job;
443 if (len == 0 || len >= JOBSTATS_JOBID_SIZE)
446 if (copy_from_user(jobid, buf, len))
450 /* Trim '\n' if any */
451 if (jobid[len - 1] == '\n')
454 if (strcmp(jobid, "clear") == 0)
457 LASSERT(stats->ojs_hash);
460 cfs_hash_for_each_safe(stats->ojs_hash, job_iter_callback,
468 job = cfs_hash_lookup(stats->ojs_hash, jobid);
472 cfs_hash_del_key(stats->ojs_hash, jobid);
478 struct file_operations lprocfs_jobstats_seq_fops = {
479 .owner = THIS_MODULE,
480 .open = lprocfs_jobstats_seq_open,
482 .write = lprocfs_jobstats_seq_write,
484 .release = lprocfs_seq_release,
487 int lprocfs_job_stats_init(struct obd_device *obd, int cntr_num,
488 cntr_init_callback init_fn)
490 struct proc_dir_entry *entry;
491 struct obd_job_stats *stats;
494 LASSERT(obd->obd_proc_entry != NULL);
495 LASSERT(obd->obd_type->typ_name);
497 if (strcmp(obd->obd_type->typ_name, LUSTRE_MDT_NAME) &&
498 strcmp(obd->obd_type->typ_name, LUSTRE_OST_NAME)) {
499 CERROR("Invalid obd device type.\n");
502 stats = &obd->u.obt.obt_jobstats;
504 LASSERT(stats->ojs_hash == NULL);
505 stats->ojs_hash = cfs_hash_create("JOB_STATS",
506 HASH_JOB_STATS_CUR_BITS,
507 HASH_JOB_STATS_MAX_BITS,
508 HASH_JOB_STATS_BKT_BITS, 0,
513 if (stats->ojs_hash == NULL)
516 INIT_LIST_HEAD(&stats->ojs_list);
517 rwlock_init(&stats->ojs_lock);
518 stats->ojs_cntr_num = cntr_num;
519 stats->ojs_cntr_init_fn = init_fn;
520 stats->ojs_cleanup_interval = 600; /* 10 mins by default */
521 stats->ojs_last_cleanup = cfs_time_current_sec();
523 LPROCFS_WRITE_ENTRY();
524 entry = proc_create_data("job_stats", 0644, obd->obd_proc_entry,
525 &lprocfs_jobstats_seq_fops, stats);
526 LPROCFS_WRITE_EXIT();
528 lprocfs_job_stats_fini(obd);
533 EXPORT_SYMBOL(lprocfs_job_stats_init);
535 #ifndef HAVE_ONLY_PROCFS_SEQ
536 int lprocfs_rd_job_interval(char *page, char **start, off_t off,
537 int count, int *eof, void *data)
539 struct obd_device *obd = (struct obd_device *)data;
540 struct obd_job_stats *stats;
542 LASSERT(obd != NULL);
543 stats = &obd->u.obt.obt_jobstats;
545 return snprintf(page, count, "%d\n", stats->ojs_cleanup_interval);
547 EXPORT_SYMBOL(lprocfs_rd_job_interval);
549 int lprocfs_wr_job_interval(struct file *file, const char __user *buffer,
550 unsigned long count, void *data)
552 struct obd_device *obd = (struct obd_device *)data;
553 struct obd_job_stats *stats;
556 LASSERT(obd != NULL);
557 stats = &obd->u.obt.obt_jobstats;
559 rc = lprocfs_write_helper(buffer, count, &val);
563 stats->ojs_cleanup_interval = val;
564 lprocfs_job_cleanup(stats, true);
569 EXPORT_SYMBOL(lprocfs_wr_job_interval);
571 int lprocfs_job_interval_seq_show(struct seq_file *m, void *data)
573 struct obd_device *obd = m->private;
574 struct obd_job_stats *stats;
576 LASSERT(obd != NULL);
577 stats = &obd->u.obt.obt_jobstats;
578 return seq_printf(m, "%d\n", stats->ojs_cleanup_interval);
580 EXPORT_SYMBOL(lprocfs_job_interval_seq_show);
583 lprocfs_job_interval_seq_write(struct file *file, const char *buffer,
584 size_t count, loff_t *off)
586 struct obd_device *obd = ((struct seq_file *)file->private_data)->private;
587 struct obd_job_stats *stats;
590 LASSERT(obd != NULL);
591 stats = &obd->u.obt.obt_jobstats;
593 rc = lprocfs_write_helper(buffer, count, &val);
597 stats->ojs_cleanup_interval = val;
598 lprocfs_job_cleanup(stats, true);
601 EXPORT_SYMBOL(lprocfs_job_interval_seq_write);