From b6fee2b803e7ae817f52911c9475e21227260d1d Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Thu, 15 Jul 2021 09:25:52 -0500 Subject: [PATCH] Squashed 'lipe/' changes from 8251fae..39ab2a0 39ab2a0 EX-3379 lipe: update EXT2_ET_EA_NAME_NOT_FOUND kluge 7c8d939 EX-3388 lipe: use correct timestamps e1ec215 Update lipe version to 1.18. 38a55f7 EX-3058 lamigo: drop NO_ACCT tag upon replication completion 79c9922 EX-3138 lipe: lpurge slot mutex handling d0ddcc9 EX-3100 lamigo: check rj_agent pointer 587b1ef EX-3030 lamigo: more stats 87fd5d6 EX-3046 lipe: remove IML sockets 954971d EX-2659 tests: add sanity-lipe.sh to test LiPE utilities git-subtree-dir: lipe git-subtree-split: 39ab2a09b51500ad850472110b31fa42576e1551 --- lipe.spec.in | 2 +- src/general_policy.c | 2 +- src/lamigo.c | 258 ++++++++++++++++++++++----------------------- src/lamigo.h | 4 - src/lipe_expression_test.c | 2 +- src/lpurge.c | 87 +++------------ src/lustre_ea_ldiskfs.c | 9 +- version-gen.sh | 2 +- 8 files changed, 154 insertions(+), 212 deletions(-) diff --git a/lipe.spec.in b/lipe.spec.in index a18d53a..161bd4d 100644 --- a/lipe.spec.in +++ b/lipe.spec.in @@ -28,7 +28,7 @@ Group: Applications/System Source0: @PACKAGE@-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires: lipe-pylustre = %{version}-%{release} -Requires: rsync json-c libyaml +Requires: rsync json-c libyaml PyYAML python2-filelock python-dateutil Requires: e2fsprogs >= 1.42.13.wc6 Provides: lipe = %{version}-%{release} %if %{with systemd} diff --git a/src/general_policy.c b/src/general_policy.c index a36a427..348ed2b 100644 --- a/src/general_policy.c +++ b/src/general_policy.c @@ -201,7 +201,7 @@ int64_t lipe_policy_attribute_evaluate(enum lipe_policy_attribute_type type, lipe_policy_attributes[type]); LRETURN(0); } - result = attrs->loa_atime_ms; + result = attrs->loa_mtime_ms; break; case LIPE_POLICY_ATTRIBUTE_CTIME: if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR)) { diff --git a/src/lamigo.c b/src/lamigo.c index 323a5aa..90fc2e2 100644 --- a/src/lamigo.c +++ b/src/lamigo.c @@ -51,7 +51,6 @@ #include #include "policy.h" #include "list.h" -#include "flist.h" #include "debug.h" #include "lipe_object_attrs.h" #include "lipe_ssh.h" @@ -95,11 +94,6 @@ __u64 lamigo_rule_attrs; /* attributes needed to evalute the rules */ static void usage(void) { - /* - * internal option: - * --iml-re-socket, unix domain socket to dump FIDs to resync - * --iml-ex-socket, unix domain socket to dump FIDs to extend - */ printf("\nUsage: %s [options]\n" "options:\n" "\t-a, --min-age, seconds before record handled (default: %u)\n" @@ -194,7 +188,8 @@ struct resync_job { char *rj_mirror_opts; void (*rj_callback)(struct resync_job *, void *, int rc); void *rj_callback_data; - unsigned long rj_start; + time_t rj_start; + time_t rj_done_timestamp; }; enum amigo_resync_type { @@ -226,6 +221,11 @@ struct options opt = { .o_alr_hot_after_idle = DEF_HOT_AFTER_IDLE, }; +struct history { + struct lu_fid h_fid; + enum amigo_resync_type h_result; +}; + struct stats { unsigned long s_read; /* llog records read */ unsigned long s_skipped; /* llog records skipped */ @@ -240,10 +240,31 @@ struct stats { unsigned long s_replicate_ro2hot; unsigned long s_replicate_rw2hot; unsigned long s_replicate_rw2cold; + unsigned long s_skip_by_rule; + unsigned long s_extend_by_pool; + unsigned long s_extend_by_objects; + unsigned long s_skip_unknown; + unsigned long s_resync_stale; + unsigned long s_skip_insync; + unsigned long s_skip_by_source; + unsigned long s_extend_by_target; + struct history s_hist[100]; + int s_hist_cur; }; struct stats stats = { 0 }; +static void lamigo_hist_add(struct lu_fid *fid, enum amigo_resync_type result) +{ + struct history *h = stats.s_hist; + + h[stats.s_hist_cur].h_fid = *fid; + h[stats.s_hist_cur].h_result = result; + stats.s_hist_cur++; + if (stats.s_hist_cur == ARRAY_SIZE(stats.s_hist)) + stats.s_hist_cur = 0; +} + struct fid_rec { struct fid_hash fr_fh; struct lipe_list_head fr_link; @@ -293,15 +314,9 @@ struct pool_list *src_pools; /* source pools */ struct pool_list *tgt_pools; /* target pool */ static void *lamigo_refresh_statfs_thread(void *arg); -inline int are_agents_busy(void) +static inline bool are_agents_busy(void) { - if (opt.o_iml_re_socket) - return 0; - - if (lamigo_jobs_running >= lamigo_max_jobs) - return 1; - - return 0; + return lamigo_jobs_running >= lamigo_max_jobs; } /* Convert human readable size string to and int; "1k" -> 1000 */ @@ -358,25 +373,58 @@ static void systemf(const char *fmt, ...) free(cmd); } -static struct lipe_flist *lamigo_flist_init(char *socket) +#define JOB_FMT \ + " job%u: { tid: %lu, fid: "DFID", index: %llu, agent: %d, start: %lu, done: %lu, command: %s }\n" + +static void lamigo_dump_jobs(FILE *out, struct lipe_list_head *jlist) { - struct lipe_flist *flist = flist_alloc(NULL, 1024, - socket, LDT_UNIX_SOCKET); - if (flist == NULL) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, - "failed to alloc fid list for socket %s: %s\n", - socket, - strerror(errno)); - return NULL; + time_t now = time(NULL); + struct resync_job *j; + int i = 0; + + lipe_list_for_each_entry(j, jlist, rj_list) { + char *cmd; + + if (j->rj_setprefer) + cmd = "setprefer"; + else if (j->rj_resync == AMIGO_RESYNC_EXTEND) + cmd = "extend"; + else + cmd = "resync"; + + fprintf(out, JOB_FMT, i++, j->rj_pid, PFID(&j->rj_fid), + j->rj_index, j->rj_agent ? j->rj_agent->rag_index : -1, + now - j->rj_start, now - j->rj_done_timestamp, cmd); } +} + +static char *lamigo_resync2str[] = { "none", "extend", "resync" }; + +static void lamigo_dump_history(FILE *out) +{ + int i = 0, cur = stats.s_hist_cur; + char *action; - return flist; + for (i = 0; i < ARRAY_SIZE(stats.s_hist); i++) { + if (--cur < 0) + cur = ARRAY_SIZE(stats.s_hist) - 1; + + if (stats.s_hist[cur].h_fid.f_seq == 0) + break; + + if (stats.s_hist[cur].h_result > AMIGO_RESYNC_RESYNC) + action = "unknown"; + else + action = lamigo_resync2str[stats.s_hist[cur].h_result]; + + fprintf(out, " hist%d: { fid: "DFID", result: %s }\n", cur, + PFID(&stats.s_hist[cur].h_fid), action); + } } void lamigo_usr1_handle(int sig) { struct resync_agent *a; - struct resync_job *j; struct pool_list *pl; struct tm *tmtime; char timestr[32]; @@ -444,11 +492,6 @@ void lamigo_usr1_handle(int sig) " open: %d\n", pl->pl_pool, pl->pl_ostnr, pl->pl_avail, pl->pl_total, (int)pl->pl_open); - if (opt.o_iml_re_socket) - fprintf(f, " iml_re_socket: %s\n", opt.o_iml_re_socket); - if (opt.o_iml_ex_socket) - fprintf(f, " iml_ex_socket: %s\n", opt.o_iml_ex_socket); - fprintf(f, "stats:\n" " read: %lu\n" " skipped: %lu\n" @@ -462,12 +505,24 @@ void lamigo_usr1_handle(int sig) " skip_hot: %lu\n" " ro2hot: %lu\n" " rw2hot: %lu\n" - " rw2cold: %lu\n", + " rw2cold: %lu\n" + " skip-by-rule: %lu\n" + " extend-by-pool: %lu\n" + " extend-by-objects: %lu\n" + " skip_unknown: %lu\n" + " resync-stale: %lu\n" + " skip-insync: %lu\n" + " skip-by-source: %lu\n" + " extend-by-target: %lu\n", stats.s_read, stats.s_skipped, stats.s_processed, stats.s_removed, stats.s_dups, stats.s_spawned, stats.s_replicated, stats.s_busy, head.lh_cached_count, stats.s_skip_hot, stats.s_replicate_ro2hot, - stats.s_replicate_rw2hot, stats.s_replicate_rw2cold); + stats.s_replicate_rw2hot, stats.s_replicate_rw2cold, + stats.s_skip_by_rule, stats.s_extend_by_pool, + stats.s_extend_by_objects, stats.s_skip_unknown, + stats.s_resync_stale, stats.s_skip_insync, + stats.s_skip_by_source, stats.s_extend_by_target); tmtime = localtime(&lamigo_last_cleared); strftime(timestr, sizeof(timestr), "%c", tmtime); @@ -489,17 +544,14 @@ void lamigo_usr1_handle(int sig) a->rag_bad ? "inactive" : "active"); } -#define JOB_FMT \ - " job%u: { tid: %lu, fid: "DFID", index: %llu, agent: %d, start: %lu, command: %s }\n" - - i = 0; fprintf(f, "jobs:\n"); - lipe_list_for_each_entry(j, &lamigo_job_list, rj_list) { - fprintf(f, JOB_FMT, i++, j->rj_pid, PFID(&j->rj_fid), - j->rj_index, j->rj_agent->rag_index, j->rj_start, - j->rj_resync == AMIGO_RESYNC_EXTEND ? "extend" : - "resync"); - } + lamigo_dump_jobs(f, &lamigo_job_list); + + fprintf(f, "failed jobs:\n"); + lamigo_dump_jobs(f, &lamigo_failed_job_list); + + fprintf(f, "history:\n"); + lamigo_dump_history(f); fflush(f); fclose(f); @@ -541,15 +593,6 @@ static int lamigo_init_cache(void) if (rc) return rc; - if (opt.o_iml_re_socket) { - opt.o_re_flist = lamigo_flist_init(opt.o_iml_re_socket); - if (opt.o_re_flist == NULL) - return -errno; - opt.o_ex_flist = lamigo_flist_init(opt.o_iml_ex_socket); - if (opt.o_ex_flist == NULL) - return -errno; - } - return 0; } @@ -592,31 +635,6 @@ static void lamigo_cleanup(void) extern char **environ; -/* json format: { "fid": "[FID]" } - * written to either MIRROR or EXTEND socket - */ -static int lamigo_flist_add_fid(struct lu_fid *fid, int resync) -{ - int rc; - const char *output; - char fid_buf[FID_LEN + 1]; - struct json_object *obj_top; - - obj_top = json_object_new_object(); - snprintf(fid_buf, sizeof(fid_buf), DFID, PFID(fid)); - json_object_object_add(obj_top, "fid", - json_object_new_string(fid_buf)); - output = json_object_to_json_string_ext(obj_top, - JSON_C_TO_STRING_PLAIN); - if (resync == AMIGO_RESYNC_RESYNC) - rc = flist_add_one(opt.o_re_flist, output); - else // AMIGO_RESYNC_EXTEND - rc = flist_add_one(opt.o_ex_flist, output); - json_object_put(obj_top); - - return rc; -} - static int lamigo_exec_cmd(struct resync_agent *a, char *cmd) { struct resync_ssh_session *rss; @@ -851,6 +869,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, resync = AMIGO_RESYNC_EXTEND; mo->mo_src_pool = pl; mo->mo_need_prefer = true; + stats.s_extend_by_pool++; goto out; } } @@ -868,6 +887,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, mo->mo_src_pool = found_on; mo->mo_need_prefer = true; resync = AMIGO_RESYNC_EXTEND; + stats.s_extend_by_objects++; } mo->mo_stripes = stripes; goto out; @@ -875,10 +895,10 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, if (v3->lmm_magic != LOV_USER_MAGIC_COMP_V1) { resync = AMIGO_RESYNC_NONE; + stats.s_skip_unknown++; goto out; } - resync = AMIGO_RESYNC_NONE; mirid = -1; stripes = 0; @@ -915,11 +935,13 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, * stale */ resync = AMIGO_RESYNC_RESYNC; + stats.s_resync_stale++; goto out; } else { /* this is complete uptodate replica * nothing to do... */ resync = AMIGO_RESYNC_NONE; + stats.s_skip_insync++; goto out; } } else { @@ -935,8 +957,10 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, } rc = lamigo_get_objects(v3, &objects, &stripes); - if (rc) + if (rc) { + stats.s_skip_unknown++; goto out; + } /* total objects in this mirror */ objs_in_mirror += stripes; @@ -975,8 +999,10 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, mo->mo_stripes = stripes; } - if (!onsrc) + if (!onsrc) { + stats.s_skip_by_source++; goto out; + } /* subject to replication */ if (ontgt) { @@ -985,16 +1011,19 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, * we can't count it as a replica */ resync = AMIGO_RESYNC_EXTEND; + stats.s_extend_by_target++; } else if (stale) { /* all mirror's object are on the target pool * but some component is stale */ resync = AMIGO_RESYNC_RESYNC; + stats.s_resync_stale++; goto out; } } else { /* no replica found, make one */ resync = AMIGO_RESYNC_EXTEND; + stats.s_extend_by_target++; } out: if (resync == AMIGO_RESYNC_EXTEND) @@ -1203,12 +1232,15 @@ static int lamigo_is_in_sync(struct lu_fid *fid, llapi_printf(LLAPI_MSG_DEBUG, "skip "DFID" due to rules\n", PFID(fid)); + stats.s_skip_by_rule++; goto out; } resync = lamigo_striping_is_in_sync(attrs.loa_lum, src, tgt, mo); out: + lamigo_hist_add(fid, resync); + llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "check "DFID" stripes=%d: resync=%d\n", PFID(fid), mo->mo_stripes, resync); @@ -1304,6 +1336,9 @@ void lamigo_schedule_setprefer(struct resync_job *rj, void *cbdata, int rc) { struct resync_job *srj; + lamigo_alr_mark(&rj->rj_fid, + rc == 0 ? ALR_TAG_REPLICATED : ALR_TAG_FAILED, + ALR_TAG_NO_ACCT); if (rc) return; @@ -1368,17 +1403,6 @@ static int lamigo_update_one(struct fid_rec *f) return 0; } - if (opt.o_iml_re_socket) { - rc = lamigo_flist_add_fid(&f->fr_fh.fh_fid, resync); - if (rc < 0) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "can't add "DFID": rc=%d\n", - PFID(&f->fr_fh.fh_fid), rc); - return 1; - } - return 0; - } - rj = calloc(1, sizeof(struct resync_job)); if (rj == NULL) { llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "can't allocate for a job\n"); @@ -1392,8 +1416,12 @@ static int lamigo_update_one(struct fid_rec *f) if (mo.mo_need_prefer && mo.mo_src_pool) { rj->rj_callback_data = mo.mo_src_pool; rj->rj_callback = lamigo_schedule_setprefer; + } else { + /* just to drop NO_ACCT tag */ + rj->rj_callback = lamigo_alr_mirror_cb; } + return lamigo_submit_job(rj); } @@ -1418,11 +1446,6 @@ static int lamigo_start_update(int count) } } - if (opt.o_iml_re_socket) { - flist_write(opt.o_re_flist, true); - flist_write(opt.o_ex_flist, true); - } - return rc; } @@ -1632,6 +1655,8 @@ static void lamigo_job_fini(struct resync_job *rj, intptr_t retval) rj->rj_pid, PFID(&rj->rj_fid), time(NULL) - rj->rj_start, retval, rj->rj_agent->rag_bad); + rj->rj_done_timestamp = time(NULL); + if (rj->rj_check_job) { rj->rj_agent->rag_check_in_progress = 0; if (retval == 0) { @@ -1788,8 +1813,6 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) lamigo_agent_count++; } -#define LAMIGO_OPT_IML_RE_SOCKET 1 -#define LAMIGO_OPT_IML_EX_SOCKET 2 #define LAMIGO_OPT_POOL_REFRESH 3 #define LAMIGO_OPT_PROGRESS_INTV 4 #define LAMIGO_OPT_MIRROR_CMD 5 @@ -1825,8 +1848,6 @@ static struct option options[] = { { "heatfn", required_argument, NULL, 'H'}, { "hot_after_idle", required_argument, NULL, 'I'}, { "heat-dump", required_argument, NULL, 'W'}, - { "iml-re-socket", required_argument, NULL, LAMIGO_OPT_IML_RE_SOCKET}, - { "iml-ex-socket", required_argument, NULL, LAMIGO_OPT_IML_EX_SOCKET}, { "max-cache", required_argument, NULL, 'c'}, { "min-age", required_argument, NULL, 'a'}, { "mirror-cmd", required_argument, NULL, LAMIGO_OPT_MIRROR_CMD}, @@ -2091,12 +2112,6 @@ void lamigo_process_opt(int c, char *optarg) exit(1); } break; - case LAMIGO_OPT_IML_RE_SOCKET: - opt.o_iml_re_socket = strdup(optarg); - break; - case LAMIGO_OPT_IML_EX_SOCKET: - opt.o_iml_ex_socket = strdup(optarg); - break; case LAMIGO_OPT_MIRROR_CMD: opt.o_mirror_cmd = strdup(optarg); break; @@ -2501,16 +2516,7 @@ void lamigo_parse_opts(int argc, char **argv) exit(1); } - // Indepenedent operation vs IML integration - if (opt.o_iml_re_socket || opt.o_iml_ex_socket) { - if (opt.o_iml_re_socket == NULL || - opt.o_iml_ex_socket == NULL) { - llapi_err_noerrno(LLAPI_MSG_ERROR, - "Both or neither sockets must be defined"); - exit(1); - } - - } else if (lipe_list_empty(&lamigo_agent_list)) { + if (lipe_list_empty(&lamigo_agent_list)) { llapi_err_noerrno(LLAPI_MSG_ERROR, "no agents configured?\n"); exit(1); } @@ -2566,18 +2572,6 @@ static int lamigo_create_job(struct lu_fid *fid, struct mirror_opts *mo) { struct resync_job *rj; - int rc; - - if (opt.o_iml_re_socket) { - rc = lamigo_flist_add_fid(fid, resync); - if (rc < 0) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "can't add "DFID": rc=%d\n", - PFID(fid), rc); - return 1; - } - return 0; - } rj = calloc(1, sizeof(struct resync_job)); if (rj == NULL) { @@ -2643,13 +2637,17 @@ int lamigo_lipe_callback(struct lipe_instance *instance, } resync = lamigo_check_user_rules(attrs, &sysattrs); - if (resync == AMIGO_RESYNC_NONE) + if (resync == AMIGO_RESYNC_NONE) { + lamigo_hist_add(&attrs->loa_fid, resync); goto out; + } resync = lamigo_striping_is_in_sync((void *)attrs->loa_lum, src_pools, tgt_pools, &mo); - if (resync == AMIGO_RESYNC_NONE) + if (resync == AMIGO_RESYNC_NONE) { + lamigo_hist_add(&attrs->loa_fid, resync); return 0; + } lamigo_check_jobs(); diff --git a/src/lamigo.h b/src/lamigo.h index 0e7f0ec..bb7421e 100644 --- a/src/lamigo.h +++ b/src/lamigo.h @@ -81,8 +81,6 @@ struct options { bool o_rescan; char *o_mirror_cmd; int o_num_threads; - char *o_iml_re_socket; - char *o_iml_ex_socket; int o_pool_refresh; int o_statfs_refresh; int o_src_free; @@ -102,8 +100,6 @@ struct options { int o_alr_ignore_reads; int o_alr_ignore_writes; char *o_heat_file; - struct lipe_flist *o_re_flist; - struct lipe_flist *o_ex_flist; }; extern struct options opt; extern int enable_heat; diff --git a/src/lipe_expression_test.c b/src/lipe_expression_test.c index 44d06b5..b8b10e7 100644 --- a/src/lipe_expression_test.c +++ b/src/lipe_expression_test.c @@ -54,7 +54,7 @@ static int set_attribute_value(enum lipe_policy_attribute_type type, attrs->loa_atime_ms = value; break; case LIPE_POLICY_ATTRIBUTE_MTIME: - attrs->loa_atime_ms = value; + attrs->loa_mtime_ms = value; break; case LIPE_POLICY_ATTRIBUTE_CTIME: attrs->loa_ctime_ms = value; diff --git a/src/lpurge.c b/src/lpurge.c index c1361c1..ca4493f 100644 --- a/src/lpurge.c +++ b/src/lpurge.c @@ -132,7 +132,6 @@ struct options { char *o_device; char *o_dumpfile; char *o_fids_dumpfile; - char *o_iml_socket; char *o_mountpoint; char *o_pool; unsigned int o_max_jobs; @@ -174,36 +173,11 @@ char *prog; void load_config(char *name); -struct lipe_flist *lflist; #define LPURGE_FLIST_SIZE (1024 * 1024) -static int lpurge_init_flist(void) -{ - if (opt.o_iml_socket) { - lflist = flist_alloc(NULL, LPURGE_FLIST_SIZE, - opt.o_iml_socket, - LDT_UNIX_SOCKET); - } else { - llapi_printf(LLAPI_MSG_FATAL, - "no valid output found, exit\n"); - return -1; - } - - if (lflist == NULL) { - llapi_printf(LLAPI_MSG_FATAL, - "failed to alloc fid list, %s\n", - strerror(errno)); - return -errno; - } - - return 0; -} - static void sig_handler(int signal) { psignal(signal, "exiting"); - if (lflist) - flist_free(lflist); _exit(0); } @@ -271,7 +245,6 @@ void usage(void) "\t-h, --freehi, high watermark, %% of space (default: %u)\n" "\t--help, print this help message\n" "\t-i, --interval, seconds to next check (default: %u)\n" - "\t--iml_socket, do not purge, just dump found objects to domain socket\n" "\t-j, --max_jobs, max.jobs to release replicas (default: %u)\n" "\t-l, --freelo, low watermark, %% of space (default: %u)\n" "\t-m, --mds, MDS idx:host:mountpoint (can specify multiple)\n" @@ -305,12 +278,6 @@ void usage(void) "\tpool=\n" "\tmds=0:mds1host:/mnt/lustre\n" "\tmds=1:mds2host:/mnt/lustre2\n"); - printf(" config example dryrun:\n" - "\tdevice=lustre-OST0000\n" - "\timl_socket=/tmp/socket.lpurge\n" - "\tfreelo=20\n" - "\tfreehi=30\n" - "\tpool=\n"); exit(0); } @@ -640,8 +607,7 @@ int lpurge_lipe_callback(struct lipe_instance *instance, if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_FILTER_FID) == 0) { ls->ls_nopfid_objs++; ls->ls_nopfid_space += attrs->loa_blocks >> 10; - pthread_mutex_unlock(&ls->ls_mutex); - goto out; + goto out_ls_mutex; } /* to avoid N OSTs to 1 MDT scalability issue we only consider @@ -650,8 +616,7 @@ int lpurge_lipe_callback(struct lipe_instance *instance, if (attrs->loa_filter_fid.ff_parent.f_ver != 0) { ls->ls_notfirst_objs++; ls->ls_notfirst_space += attrs->loa_blocks >> 10; - pthread_mutex_unlock(&ls->ls_mutex); - goto out; + goto out_ls_mutex; } /* if the object has got ost_layout structure which encodes @@ -663,8 +628,7 @@ int lpurge_lipe_callback(struct lipe_instance *instance, == 0) { ls->ls_nomirror_objs++; ls->ls_nomirror_space += attrs->loa_blocks >> 10; - pthread_mutex_unlock(&ls->ls_mutex); - goto out; + goto out_ls_mutex; } } @@ -687,8 +651,7 @@ int lpurge_lipe_callback(struct lipe_instance *instance, if (ls->ls_stored > opt.o_slot_size) { /* the youngest slot is full */ if (index == 0) { - pthread_mutex_unlock(&ls->ls_mutex); - goto out; + goto out_ls_mutex; } else { ls->ls_found--; ls->ls_space -= attrs->loa_blocks >> 10; @@ -698,8 +661,8 @@ int lpurge_lipe_callback(struct lipe_instance *instance, llapi_printf(LLAPI_MSG_DEBUG, "reclaiming one slot\n"); lpurge_reclaim_slot(index); - pthread_mutex_lock(&ls->ls_mutex); ls = lpurge_hist + index; + pthread_mutex_lock(&ls->ls_mutex); ls->ls_found++; ls->ls_space += attrs->loa_blocks >> 10; } @@ -707,7 +670,7 @@ int lpurge_lipe_callback(struct lipe_instance *instance, lo = calloc(1, sizeof(*lo)); if (lo == NULL) - goto out; + goto out_ls_mutex; lo->lo_fid = attrs->loa_filter_fid.ff_parent; lo->lo_blocks = attrs->loa_blocks >> 10; @@ -722,15 +685,16 @@ int lpurge_lipe_callback(struct lipe_instance *instance, lipe_list_add(&lo->lo_list, &ls->ls_obj_list); ls->ls_stored++; + llapi_printf(LLAPI_MSG_DEBUG, "add "DFID" to %p/%lu\n", + PFID(&attrs->loa_filter_fid.ff_parent), ls, ls->ls_stored); + if (ls->ls_max_utime < last_used) ls->ls_max_utime = last_used; if (ls->ls_min_utime > last_used) ls->ls_min_utime = last_used; - pthread_mutex_unlock(&ls->ls_mutex); - - llapi_printf(LLAPI_MSG_DEBUG, "add "DFID" to %p/%lu\n", - PFID(&attrs->loa_filter_fid.ff_parent), ls, ls->ls_stored); +out_ls_mutex: + pthread_mutex_unlock(&ls->ls_mutex); out: /* * in lazy crawling mode scan for some limited number @@ -1308,9 +1272,8 @@ void parse_mountpoint(const char *name) } } -#define LPURGE_INTERNAL_DUMP_FIDS 1 -#define LPURGE_OPT_IML_SOCKET 2 -#define LPURGE_OPT_VERSION 3 +#define LPURGE_INTERNAL_DUMP_FIDS 1 +#define LPURGE_OPT_VERSION 2 static struct option options[] = { { "device", required_argument, NULL, 'D'}, @@ -1322,8 +1285,6 @@ static struct option options[] = { { "freelo", required_argument, NULL, 'l'}, { "help", no_argument, NULL, 'H' }, { "interval", required_argument, NULL, 'i'}, - { "iml_socket", required_argument, NULL, LPURGE_OPT_IML_SOCKET}, - { "listen_socket", required_argument, NULL, 's'}, { "max_jobs", required_argument, NULL, 'j'}, { "mds", required_argument, NULL, 'm'}, { "mount", required_argument, NULL, 'M'}, @@ -1436,12 +1397,6 @@ void lpurge_process_opt(int c, char *optarg) } opt.o_scan_rate = value; break; - case 's': - llapi_printf(LLAPI_MSG_WARN, - "'-s' and '--listen-socket' are deprecated, use '--iml-socket' instead\n"); - case LPURGE_OPT_IML_SOCKET: - opt.o_iml_socket = strdup(optarg); - break; case 'S': value = strtol(optarg, &endptr, 10); if (*endptr != '\0' || value < 0) { @@ -1584,7 +1539,7 @@ void lpurge_verify_opts(void) opt.o_fids_dumpfile = strdup(buf); } - if (opt.o_iml_socket == NULL && lustre_fd < 0) { + if (lustre_fd < 0) { llapi_printf(LLAPI_MSG_ERROR, "client mountpoint is not defined\n"); exit(1); @@ -1620,16 +1575,6 @@ void lpurge_verify_opts(void) sysconf(_SC_NPROCESSORS_ONLN)); } - // IML Integration - if (opt.o_iml_socket) { - int rc; - - rc = lpurge_init_flist(); - if (rc) - exit(rc); - return; - } - if (opt.o_max_jobs < 1 || opt.o_max_jobs > 1024) { llapi_printf(LLAPI_MSG_FATAL, "invalid max_jobs: %d\n", opt.o_max_jobs); @@ -1643,7 +1588,7 @@ void lpurge_parse_opts(int argc, char **argv) int c; while ((c = getopt_long(argc, argv, - "dD:b::nh:l:i:s:j:M:m:p:R:t:S:f:w:?", + "dD:b::nh:l:i:j:M:m:p:R:t:S:f:w:?", options, NULL)) != EOF) { /* Don't do further process if invalid option found */ @@ -1698,8 +1643,6 @@ void lpurge_usr1_handle(int sig) opt.o_device, opt.o_mountpoint, opt.o_pool, opt.o_max_jobs, opt.o_interval, opt.o_scan_rate, opt.o_scan_threads, opt.o_slot_size); - if (opt.o_iml_socket) - fprintf(f, " iml_socket: %s\n", opt.o_iml_socket); fprintf(f, "stats:\n" " scans: %lu\n" diff --git a/src/lustre_ea_ldiskfs.c b/src/lustre_ea_ldiskfs.c index 0bc798b..dbf0bf1 100644 --- a/src/lustre_ea_ldiskfs.c +++ b/src/lustre_ea_ldiskfs.c @@ -54,13 +54,18 @@ int get_link_ea_ldiskfs(struct lipe_object *object, * here is not super critical, since it is mostly "does xattr exist or not", * but this should be flexible in what is accepted now and in the future. */ +#define EXT2_ET_EA_NAME_NOT_FOUND_1445WC1 (2133571512L) #define EXT2_ET_EA_NAME_NOT_FOUND_1456WC1 (2133571513L) #define EXT2_ET_EA_NAME_NOT_FOUND_1456WC3 (2133571514L) +#define EXT2_ET_EA_NAME_NOT_FOUND_1462WC3 (2133571515L) + static bool xattr_name_not_found(long errcode) { return errcode == EXT2_ET_EA_NAME_NOT_FOUND || - errcode == EXT2_ET_EA_NAME_NOT_FOUND_1456WC1 || - errcode == EXT2_ET_EA_NAME_NOT_FOUND_1456WC3; + errcode == EXT2_ET_EA_NAME_NOT_FOUND_1445WC1 || + errcode == EXT2_ET_EA_NAME_NOT_FOUND_1456WC1 || + errcode == EXT2_ET_EA_NAME_NOT_FOUND_1456WC3 || + errcode == EXT2_ET_EA_NAME_NOT_FOUND_1462WC3; } int get_lmv_ea_ldiskfs(struct lipe_object *object, diff --git a/version-gen.sh b/version-gen.sh index d731ccb..edcd684 100755 --- a/version-gen.sh +++ b/version-gen.sh @@ -1,6 +1,6 @@ #!/bin/sh -VERSION="1.17" +VERSION="1.18" # if [ -d .git ]; then # VERSION=$(git describe|sed 's/-[0-9]*-/./') # fi -- 1.8.3.1