#include "ls3_merge_stats.h"
+static void ls3_m_upd_str2fid(const char *fid_str, struct lu_fid *fid)
+{
+ char *end_ptr;
+
+ /* if was not deleted, in one of the reports FID will be 100% */
+ if (strcmp(fid_str, LS3_M_EMPTY_FID_STR) == 0 ||
+ strcmp(fid_str, "") == 0)
+ return; /* Skip */
+
+ fid->f_seq = strtoull(fid_str, &end_ptr, 16);
+ if (*end_ptr != '\0') {
+ fid->f_oid = strtoul(end_ptr + 1, &end_ptr, 16);
+ if (*end_ptr != '\0')
+ fid->f_ver = strtoul(end_ptr + 1, NULL, 16);
+ }
+}
static struct range_report_template *ls3_m_alloc_new_id_range(
struct report_template *report_ptr,
return range_ptr;
}
-
static void ls3_m_parsing_range(struct range_report_template *range,
json_object *json,
const char *dev_name,
double min;
double max;
uint64_t l_atime;
- int last_dev_idx;
if (user_report) {
json_object_object_get_ex(json, "CountFilesInRange", &j_count);
min = json_object_get_int64(j_min);
max = json_object_get_int64(j_max);
- last_dev_idx = range->rrt_dev_cnt + 1;
- range->rrt_dev[last_dev_idx] = xstrdup(dev_name);
- range->rrt_dev_cnt++;
-
if (range->rrt_min > min)
range->rrt_min = min;
}
static void ls3_m_parsing_report(struct report_template **r_template_ptr,
- json_object *report, const char *dev_name,
- bool user_report)
+ json_object *report,
+ const char *dev_name, bool user_report)
{
struct report_template *mem_report;
json_object *general_info;
table_idx = json_object_get_int(table_id);
mem_report = r_template_ptr[table_idx];
if (!mem_report)
- LS3_FATAL("Table %d not found in memory\n", table_idx);
+ return;
if (user_report) {
range_types = LS3_STATS_RANGE_TYPE_INT;
}
}
-static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd,
- const char *path)
+static struct ls3_stats_dir_obj *ls3_m_get_child_dir_ptr(
+ struct ls3_stats_dir_general *m_dir_stats,
+ struct ls3_stats_dir_obj *parent_ptr, const char *dir_name, int depth)
+{
+ struct ls3_stats_dir_obj *dir_ptr;
+ uint64_t current_index = 0;
+ bool was_found = false;
+ int i;
+
+ if (!parent_ptr)
+ LS3_FATAL("parent directory is not defined (NULL)\n");
+
+
+ for (i = 0; i <= parent_ptr->lsdo_last_child; i++) {
+ dir_ptr = parent_ptr->lsdo_child[i];
+
+ if (dir_ptr == NULL)
+ continue;
+
+ if (strcmp(dir_ptr->lsdo_name, dir_name) == 0) {
+ was_found = true;
+ break;
+ }
+ }
+
+ if (was_found)
+ return dir_ptr;
+
+ /* Get exactly index in the array since it's a new catalog
+ * Expand memory if necessary and create/allocate a new directory.
+ */
+ current_index = parent_ptr->lsdo_last_child + 1;
+ if (current_index == 1 && parent_ptr->lsdo_child[0] == NULL)
+ current_index = 0; /* This is first catalog with 0 index */
+
+ if (current_index == parent_ptr->lsdo_child_max)
+ ls3_expand_num_of_dirs(parent_ptr);
+
+ dir_ptr = parent_ptr->lsdo_child[current_index];
+ if (dir_ptr == NULL) {
+ dir_ptr = ls3_stats_allocate_new_dir(m_dir_stats, NULL,
+ dir_name, depth);
+ parent_ptr->lsdo_child[current_index] = dir_ptr;
+ parent_ptr->lsdo_last_child = current_index;
+ }
+
+ return dir_ptr;
+}
+
+static void ls3_m_read_dir_in_tree(struct ls3_stats_dir_general *m_dir_stats,
+ json_object *j_dir,
+ struct ls3_stats_dir_obj *dir)
+{
+ json_object *j_total_size;
+ json_object *j_total_alloc_size;
+ json_object *j_uid;
+ json_object *j_projid;
+ json_object *j_gid;
+ json_object *j_fid;
+ json_object *j_files_cnt;
+ json_object *j_dirs_cnt;
+ json_object *j_child_dirs;
+ json_object *j_mtime;
+ json_object *j_atime;
+ json_object *j_ctime;
+ json_object *j_crtime;
+ int cnt_child = 0;
+ int i;
+
+ if (!j_dir)
+ return;
+
+ if (json_object_object_get_ex(j_dir, "SizeBytes", &j_total_size))
+ dir->lsdo_size += json_object_get_int64(j_total_size);
+
+ if (json_object_object_get_ex(j_dir, "AllocatedSizeBytes",
+ &j_total_alloc_size)) {
+ uint64_t alloc_size_in_sec =
+ json_object_get_int64(j_total_alloc_size) / 512;
+
+ dir->lsdo_alloc_size_in_sectors += alloc_size_in_sec;
+ }
+
+ if (json_object_object_get_ex(j_dir, "UserID", &j_uid))
+ dir->lsdo_uid = json_object_get_int64(j_uid);
+
+ if (json_object_object_get_ex(j_dir, "GroupID", &j_gid))
+ dir->lsdo_gid = json_object_get_int64(j_gid);
+
+ if (json_object_object_get_ex(j_dir, "ProjID", &j_projid))
+ dir->lsdo_projid = json_object_get_int64(j_gid);
+
+ if (json_object_object_get_ex(j_dir, "FilesCount", &j_files_cnt))
+ dir->lsdo_files_count += json_object_get_int64(j_files_cnt);
+
+ if (json_object_object_get_ex(j_dir, "DirsCount", &j_dirs_cnt))
+ dir->lsdo_dirs_count += json_object_get_int64(j_dirs_cnt);
+
+ if (json_object_object_get_ex(j_dir, "Atime", &j_atime)) {
+ if (dir->lsdo_atime < json_object_get_int64(j_atime))
+ dir->lsdo_atime = json_object_get_int64(j_atime);
+ }
+
+ if (json_object_object_get_ex(j_dir, "Mtime", &j_mtime)) {
+ if (dir->lsdo_mtime < json_object_get_int64(j_mtime))
+ dir->lsdo_mtime = json_object_get_int64(j_mtime);
+ }
+
+ if (json_object_object_get_ex(j_dir, "Ctime", &j_ctime)) {
+ if (dir->lsdo_ctime < json_object_get_int64(j_ctime))
+ dir->lsdo_ctime = json_object_get_int64(j_ctime);
+ }
+
+ if (json_object_object_get_ex(j_dir, "Crtime", &j_crtime)) {
+ if (dir->lsdo_crtime < json_object_get_int64(j_crtime))
+ dir->lsdo_crtime = json_object_get_int64(j_crtime);
+ }
+
+ if (json_object_object_get_ex(j_dir, "FID", &j_fid)) {
+ const char *fid_str = json_object_get_string(j_fid);
+
+ ls3_m_upd_str2fid(fid_str, &dir->lsdo_fid);
+ }
+
+ if (json_object_object_get_ex(j_dir, "ChildDirectories", &j_child_dirs))
+ cnt_child = json_object_array_length(j_child_dirs);
+
+ for (i = 0; i < cnt_child; i++) {
+ struct ls3_stats_dir_obj *child_dir_ptr;
+ json_object *j_dir_name;
+ json_object *j_depth;
+ json_object *j_child;
+ const char *dir_name;
+
+ j_child = json_object_array_get_idx(j_child_dirs, i);
+ if (!j_child) {
+ continue;
+ }
+
+ json_object_object_get_ex(j_child, "DirectoryName", &j_dir_name);
+ dir_name = json_object_get_string(j_dir_name);
+
+ json_object_object_get_ex(j_child, "Depth", &j_depth);
+ child_dir_ptr =
+ ls3_m_get_child_dir_ptr(m_dir_stats, dir, dir_name,
+ json_object_get_int(j_depth));
+ /* recursion */
+ ls3_m_read_dir_in_tree(m_dir_stats, j_child, child_dir_ptr);
+ }
+
+}
+
+static int ls3_m_check_first_dir(struct ls3_stats_dir_general *m_dir_stats,
+ json_object *j_dirs)
+{
+ struct ls3_stats_dir_obj *new_dir_ptr;
+ json_object *j_first_dir;
+ json_object *j_total_size;
+ json_object *j_total_alloc_size;
+ json_object *j_r_limit;
+ json_object *j_max_depth;
+ const char *dir_name;
+ uint64_t alloc_size_in_sec;
+
+ json_object_object_get_ex(j_dirs, "SourceDirectory", &j_first_dir);
+ json_object_object_get_ex(j_dirs, "TotalSizeBytes", &j_total_size);
+ json_object_object_get_ex(j_dirs, "TotalAllocatedSizeBytes",
+ &j_total_alloc_size);
+
+ dir_name = json_object_get_string(j_first_dir);
+ alloc_size_in_sec = json_object_get_int64(j_total_alloc_size) / 512;
+
+ if (m_dir_stats->lsdg_start_dir == NULL) {
+ new_dir_ptr = xcalloc(1, sizeof(struct ls3_stats_dir_obj));
+ new_dir_ptr->lsdo_child_max = LS3_STATS_DIR_COUNT_BY_DEFAULT;
+ new_dir_ptr->lsdo_depth = -1;
+ new_dir_ptr->lsdo_size = json_object_get_int64(j_total_size);
+ new_dir_ptr->lsdo_name = xstrdup(dir_name);
+ new_dir_ptr->lsdo_alloc_size_in_sectors = alloc_size_in_sec;
+ new_dir_ptr->lsdo_child = (struct ls3_stats_dir_obj**)xcalloc(
+ new_dir_ptr->lsdo_child_max,
+ sizeof(struct ls3_stats_dir_obj*));
+
+ m_dir_stats->lsdg_start_dir = new_dir_ptr;
+ m_dir_stats->lsdg_start_path = xstrdup("");
+ } else {
+ /* This is not the first report we have read. Just update */
+ new_dir_ptr = m_dir_stats->lsdg_start_dir;
+
+ if (strcmp(new_dir_ptr->lsdo_name, dir_name) != 0) {
+ LS3_ERROR("starting directory entry points do not match. expect:[%s] got:[%s]\n",
+ new_dir_ptr->lsdo_name, dir_name);
+ return -EINVAL;
+ }
+
+ new_dir_ptr->lsdo_size += json_object_get_int64(j_total_size);
+ new_dir_ptr->lsdo_alloc_size_in_sectors += alloc_size_in_sec;
+ }
+
+ if (json_object_object_get_ex(j_dirs, "TopRatingLimit", &j_r_limit))
+ m_dir_stats->lsdg_top_rating_limit =
+ json_object_get_int(j_r_limit);
+
+ if (json_object_object_get_ex(j_dirs, "MaxDepth", &j_max_depth))
+ m_dir_stats->lsdg_max_depth = json_object_get_int(j_max_depth);
+
+ return 0;
+}
+
+static void ls3_m_parsing_dirs_tree(struct fstats_report *stats,
+ struct ls3_stats_dir_general *m_dir_stats,
+ json_object *j_dirs, const char *dev_name)
+{
+ json_object *j_main_tree;
+ json_object *j_child_dirs;
+ int dirs_cnt;
+ int rc;
+
+ rc = ls3_m_check_first_dir(m_dir_stats, j_dirs);
+ if (rc)
+ return; /* Skip this report */
+
+ if (json_object_object_get_ex(j_dirs, "MainTree", &j_main_tree)) {
+ if (!json_object_object_get_ex(j_main_tree, "ChildDirectories",
+ &j_child_dirs))
+ return;
+
+ /* Get the number of directories with depth equal to 0. */
+ dirs_cnt = json_object_array_length(j_child_dirs);
+ if (dirs_cnt == 0) {
+ LS3_WARNING("no directories found. device: %s",
+ dev_name);
+ /* It’s better to know this before we get into recursion
+ * Skip it.
+ */
+ return;
+ }
+
+ /* Recourse reading child dirs starting from the first dir*/
+ ls3_m_read_dir_in_tree(m_dir_stats, j_main_tree,
+ m_dir_stats->lsdg_start_dir);
+ }
+}
+
+static int ls3_m_parsing_json(struct fstats_report *stats,
+ struct ls3_stats_dir_general *m_dirs_stats,
+ FILE *fd, const char *path)
{
json_object *json;
json_object *j_reports;
json_object *ju_reports;
+ json_object *j_dir_stats;
json_object *j_dev_name;
+ json_object *j_mount_point;
const char *dev_name;
- char *ftext;
long length;
+ char *ftext;
int i;
+ if (!fd || !stats || !m_dirs_stats)
+ return -1;
+
fseek(fd, 0, SEEK_END);
length = ftell(fd);
fseek(fd, 0, SEEK_SET);
json_object_object_get_ex(json, "DeviceName", &j_dev_name);
dev_name = json_object_get_string(j_dev_name);
+ if (json_object_object_get_ex(json, "ClientMountPath", &j_mount_point)) {
+ const char *mount_point;
+
+ mount_point = json_object_get_string(j_mount_point);
+ stats->client_mount_path = xstrdup(mount_point);
+ } else {
+ stats->client_mount_path = xstrdup("MergedReport");
+ }
+
if (json_object_object_get_ex(json, "Reports", &j_reports)) {
int reports_cnt = json_object_array_length(j_reports);
ls3_m_parsing_user_report(stats, user, dev_name);
}
}
+
+ if (json_object_object_get_ex(json, "DirectoriesStats", &j_dir_stats))
+ ls3_m_parsing_dirs_tree(stats, m_dirs_stats, j_dir_stats,
+ dev_name);
+
+ return 0;
}
static FILE *ls3_m_open_json_file(const char *filename)
return false;
}
-void ls3_m_read_reports(struct fstats_report *stats,
- const char *dir_with_reports)
+static void ls3_m_read_reports(struct fstats_report *stats,
+ struct ls3_stats_dir_general *dirs_stats,
+ const char *dir_with_reports)
{
struct dirent *entry;
DIR *dir;
sprintf(full_path, "%s/%s", dir_with_reports, entry->d_name);
fd = ls3_m_open_json_file(full_path);
- ls3_m_parsing_json(stats, fd, full_path);
+ if (!fd)
+ continue; /* Skip if NULL */
+
+ ls3_m_parsing_json(stats, dirs_stats, fd, full_path);
fclose(fd);
}
}
return stats;
}
+static struct ls3_stats_dir_general *ls3_m_dirs_report_init(void)
+{
+ struct ls3_stats_dir_general *dirs_tree;
+
+ /* This is the first and only place with initialization. */
+ dirs_tree = xcalloc(1, sizeof(struct ls3_stats_dir_general));
+
+ return dirs_tree;
+}
+
void ls3_stats_merge_reports(const char *dir_with_reports)
{
- struct fstats_report *main_report;
- struct ls3_stats_dir_general *dirs_report = NULL;
-
- main_report = ls3_m_report_init();
- main_report->merging = true;
- main_report->report_file_name = xstrdup("merged_report.all");
- ls3_m_read_reports(main_report, dir_with_reports);
- ls3_stats_printf(main_report, dirs_report);
- ls3_stats_destroy(main_report);
+ struct fstats_report *files_report;
+ struct ls3_stats_dir_general *dirs_report;
+
+ files_report = ls3_m_report_init();
+ dirs_report = ls3_m_dirs_report_init();
+
+ files_report->merging = true;
+ files_report->device_name = xstrdup("MergedReport");
+ files_report->report_file_name = xstrdup("merged_report.all");
+ dirs_report->lsdg_files_report = files_report;
+ ls3_m_read_reports(files_report, dirs_report, dir_with_reports);
+ ls3_stats_printf(files_report, dirs_report);
+ ls3_stats_destroy(files_report);
}
git://git.whamcloud.com / fs / lustre-release.git / commitdiff