From 08e741bcfc17f3f3de5aa66d8b3c62ee3ce5f275 Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Wed, 27 Oct 2021 13:35:46 -0500 Subject: [PATCH] EX-3002 lipe: lpurge units Use kb suffixes consistently in lpurge. Add loa_used_kb() to encapsulate the poor naming otf the loa_blocks member in struct lipe_object_attrs. Remove some misleading comments. Rename structure members lo_blocks to lo_used_kb, and ls_*_space to ls_*_used_kb. In lpurge_purge_slot() rename target to target_kb, and total to queued_kb. Rename lpurge_free_space() to lpurge_purge(). Signed-off-by: John L. Hammond Test-Parameters: trivial testlist=hot-pools Change-Id: I0bbd4f0ea815e3a95eb36d1dc590f03f76eeccd3 Reviewed-on: https://review.whamcloud.com/45388 Reviewed-by: Jian Yu Reviewed-by: Alexandre Ioffe Tested-by: Maloo Tested-by: jenkins Reviewed-by: Alex Zhuravlev Reviewed-on: https://review.whamcloud.com/46125 --- lipe/src/lpurge.c | 127 +++++++++++++++++++++++++++--------------------------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/lipe/src/lpurge.c b/lipe/src/lpurge.c index dcea77c..120dc95 100644 --- a/lipe/src/lpurge.c +++ b/lipe/src/lpurge.c @@ -77,8 +77,7 @@ struct lpurge_object { struct lipe_list_head lo_list; struct lu_fid lo_fid; - /* Used space in bytes */ - __u32 lo_blocks; + __u32 lo_used_kb; __u32 lo_mirror_id; /* Last use time */ time_t lo_last_utime; @@ -89,19 +88,18 @@ struct lpurge_slot { time_t ls_max_utime; unsigned long ls_age; unsigned long ls_found; - /* Used space in bytes */ - unsigned long ls_space; + unsigned long ls_used_kb; unsigned long ls_stored; /* FIDs stored */ unsigned int ls_scan; struct lipe_list_head ls_obj_list; pthread_mutex_t ls_mutex; /* stats for objects we can't release */ unsigned long ls_nomirror_objs; /* no replicated objects */ - unsigned long ls_nomirror_space; + unsigned long ls_nomirror_used_kb; unsigned long ls_nopfid_objs; /* no PFID, can't find parent */ - unsigned long ls_nopfid_space; + unsigned long ls_nopfid_used_kb; unsigned long ls_notfirst_objs; /* not a first stripe */ - unsigned long ls_notfirst_space; + unsigned long ls_notfirst_used_kb; }; #define LPURGE_HIST_MAX 16 @@ -181,6 +179,12 @@ static void load_config(char *name); #define LPURGE_FLIST_SIZE (1024 * 1024) +static inline unsigned long long loa_used_kb(const struct lipe_object_attrs *loa) +{ + /* XXX XXX XXX loa_blocks is in units of bytes. */ + return loa->loa_blocks >> 10; +} + static void sig_handler(int signal) { psignal(signal, "exiting"); @@ -209,7 +213,7 @@ static void lpurge_init_result(void) lpurge_hist[i].ls_min_utime = ~0UL; lpurge_hist[i].ls_max_utime = 0; lpurge_hist[i].ls_found = 0; - lpurge_hist[i].ls_space = 0; + lpurge_hist[i].ls_used_kb = 0; lpurge_hist[i].ls_stored = 0; LIPE_INIT_LIST_HEAD(&lpurge_hist[i].ls_obj_list); age = age << 1; @@ -224,17 +228,17 @@ static void lpurge_reset_result(void) for (i = 0; i < LPURGE_HIST_MAX; i++) { lpurge_hist[i].ls_found = 0; - lpurge_hist[i].ls_space = 0; + lpurge_hist[i].ls_used_kb = 0; lpurge_hist[i].ls_stored = 0; lpurge_hist[i].ls_scan = 1; lpurge_hist[i].ls_min_utime = ~0UL; lpurge_hist[i].ls_max_utime = 0; lpurge_hist[i].ls_nopfid_objs = 0; - lpurge_hist[i].ls_nopfid_space = 0; + lpurge_hist[i].ls_nopfid_used_kb = 0; lpurge_hist[i].ls_nomirror_objs = 0; - lpurge_hist[i].ls_nomirror_space = 0; + lpurge_hist[i].ls_nomirror_used_kb = 0; lpurge_hist[i].ls_notfirst_objs = 0; - lpurge_hist[i].ls_notfirst_space = 0; + lpurge_hist[i].ls_notfirst_used_kb = 0; lipe_list_for_each_entry_safe(s, t, &lpurge_hist[i].ls_obj_list, lo_list) { lipe_list_del(&s->lo_list); @@ -507,7 +511,7 @@ static void lpurge_reclaim_slot(unsigned int index) pthread_mutex_lock(&ls_1->ls_mutex); ls_1->ls_found = 0; - ls_1->ls_space = 0; + ls_1->ls_used_kb = 0; ls_1->ls_stored = 0; ls_1->ls_max_utime = 0; ls_1->ls_min_utime = ~0UL; @@ -534,7 +538,7 @@ static void lpurge_reclaim_slot(unsigned int index) &ls_1->ls_obj_list); ls_1->ls_stored = ls_2->ls_stored; ls_1->ls_found = ls_2->ls_found; - ls_1->ls_space = ls_2->ls_space; + ls_1->ls_used_kb = ls_2->ls_used_kb; ls_1->ls_age = ls_2->ls_age; ls_1->ls_max_utime = ls_2->ls_max_utime; ls_1->ls_min_utime = ls_2->ls_min_utime; @@ -550,7 +554,7 @@ static void lpurge_reclaim_slot(unsigned int index) pthread_mutex_lock(&lpurge_hist[index].ls_mutex); lpurge_hist[index].ls_stored = 0; lpurge_hist[index].ls_found = 0; - lpurge_hist[index].ls_space = 0; + lpurge_hist[index].ls_used_kb = 0; lpurge_hist[index].ls_min_utime = ~0UL; lpurge_hist[index].ls_max_utime = 0; lpurge_hist[index].ls_age = lpurge_hist[index + 1].ls_age; @@ -575,7 +579,7 @@ static int lpurge_check_mdt_object(struct lpurge_slot *ls, if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LOVEA) == 0) { ls->ls_nomirror_objs++; - ls->ls_nomirror_space += attrs->loa_blocks >> 10; + ls->ls_nomirror_used_kb += loa_used_kb(attrs); return 0; } @@ -583,7 +587,7 @@ static int lpurge_check_mdt_object(struct lpurge_slot *ls, if (comp_v1->lcm_magic != LOV_USER_MAGIC_COMP_V1 || comp_v1->lcm_mirror_count == 0) { ls->ls_nomirror_objs++; - ls->ls_nomirror_space += attrs->loa_blocks >> 10; + ls->ls_nomirror_used_kb += loa_used_kb(attrs); return 0; } @@ -614,7 +618,7 @@ static int lpurge_check_mdt_object(struct lpurge_slot *ls, } ls->ls_nomirror_objs++; - ls->ls_nomirror_space += attrs->loa_blocks >> 10; + ls->ls_nomirror_used_kb += loa_used_kb(attrs); return 0; } @@ -623,7 +627,7 @@ static int lpurge_check_ost_object(struct lpurge_slot *ls, { if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_FILTER_FID) == 0) { ls->ls_nopfid_objs++; - ls->ls_nopfid_space += attrs->loa_blocks >> 10; + ls->ls_nopfid_used_kb += loa_used_kb(attrs); return 0; } @@ -632,7 +636,7 @@ static int lpurge_check_ost_object(struct lpurge_slot *ls, */ if (attrs->loa_filter_fid.ff_parent.f_ver != 0) { ls->ls_notfirst_objs++; - ls->ls_notfirst_space += attrs->loa_blocks >> 10; + ls->ls_notfirst_used_kb += loa_used_kb(attrs); return 0; } @@ -644,7 +648,7 @@ static int lpurge_check_ost_object(struct lpurge_slot *ls, if (mirror_id_of(attrs->loa_filter_fid.ff_layout.ol_comp_id) == 0) { ls->ls_nomirror_objs++; - ls->ls_nomirror_space += attrs->loa_blocks >> 10; + ls->ls_nomirror_used_kb += loa_used_kb(attrs); return 0; } } @@ -721,13 +725,15 @@ static int lpurge_lipe_callback(struct lipe_instance *instance, if (!rc) goto out_ls_mutex; - LX_DEBUG("found under "DFID": size %ld block %ld age %ld slot %d\n", + LX_DEBUG("found under "DFID": size_kb %llu, used_kb %llu, age %ld, slot %d\n", PFID(&attrs->loa_filter_fid.ff_parent), - (unsigned long)attrs->loa_size, - (unsigned long)attrs->loa_blocks >> 10, age, index); + (unsigned long long)(attrs->loa_size >> 10), + loa_used_kb(attrs), + age, + index); ls->ls_found++; - ls->ls_space += attrs->loa_blocks >> 10; + ls->ls_used_kb += loa_used_kb(attrs); lpurge_scanned_since++; /* @@ -742,7 +748,7 @@ static int lpurge_lipe_callback(struct lipe_instance *instance, goto out_ls_mutex; } else { ls->ls_found--; - ls->ls_space -= attrs->loa_blocks >> 10; + ls->ls_used_kb -= loa_used_kb(attrs); pthread_mutex_unlock(&ls->ls_mutex); index--; @@ -752,13 +758,13 @@ static int lpurge_lipe_callback(struct lipe_instance *instance, ls = lpurge_hist + index; pthread_mutex_lock(&ls->ls_mutex); ls->ls_found++; - ls->ls_space += attrs->loa_blocks >> 10; + ls->ls_used_kb += loa_used_kb(attrs); } } lo = xcalloc(1, sizeof(*lo)); lo->lo_fid = attrs->loa_filter_fid.ff_parent; - lo->lo_blocks = attrs->loa_blocks >> 10; + lo->lo_used_kb = loa_used_kb(attrs); lo->lo_last_utime = last_used; if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid)) { __u32 id; @@ -1102,18 +1108,17 @@ static void lpurge_work_submit(struct lpurge_object *lo) } #endif -static void lpurge_purge_slot(struct lpurge_slot *ls, long long target) +static void lpurge_purge_slot(struct lpurge_slot *ls, long long target_kb) { struct lpurge_object *lo; - unsigned long long total, prev_used_kb, used_kb; + unsigned long long queued_kb, prev_used_kb, used_kb; long long purged_kb; int i, rc; - /* try to remove some replicas */ again: - LX_DEBUG("release upto %llu (expect %lu in %lu)\n", - target, ls->ls_space, ls->ls_found); - total = 0; + LX_DEBUG("releasing upto %llu KB: expect %lu KB from %lu objects\n", + target_kb, ls->ls_used_kb, ls->ls_found); + queued_kb = 0; assert(!lipe_list_empty(&ls->ls_obj_list)); @@ -1124,9 +1129,8 @@ again: lo = lipe_list_entry(ls->ls_obj_list.next, struct lpurge_object, lo_list); - /* how many blocks we expect to free */ - total += lo->lo_blocks; - ls->ls_space -= lo->lo_blocks; + queued_kb += lo->lo_used_kb; + ls->ls_used_kb -= lo->lo_used_kb; ls->ls_found--; ls->ls_stored--; @@ -1134,7 +1138,7 @@ again: stats.s_queued++; /* if current collection of objects may free target space, then stop */ - if (total >= target) + if (queued_kb >= target_kb) break; } @@ -1146,14 +1150,14 @@ again: /* estimate how much space has been released */ - rc = lpurge_get_used_kb(&prev_used_kb); + rc = lpurge_get_used_kb(&used_kb); if (rc) return; - /* FIXME Clarify units of total. */ - LX_DEBUG("spawn, expect %llu back\n", total); - LX_DEBUG_D(target); - LX_DEBUG_U(total); + LX_DEBUG_U(queued_kb); + LX_DEBUG_D(target_kb); + LX_DEBUG_U(used_kb); + prev_used_kb = used_kb; /* Wait for purge threads to complete all submitted work. */ pthread_mutex_lock(&lpurge_work_lock); @@ -1173,19 +1177,16 @@ again: return; purged_kb = prev_used_kb - used_kb; - LX_DEBUG_U(prev_used_kb); - LX_DEBUG_U(used_kb); LX_DEBUG_D(purged_kb); - /* XXX purged_kb is signed. */ - if (purged_kb > 0 && purged_kb >= total) + if (purged_kb > 0 && purged_kb >= queued_kb) break; } if (purged_kb > 0) - target -= purged_kb; + target_kb -= purged_kb; - if (target <= 0 || used_kb <= lpurge_min_used_kb) { + if (target_kb <= 0 || used_kb <= lpurge_min_used_kb) { /* got enough space back, relax */ LX_DEBUG("relax\n"); return; @@ -1209,7 +1210,7 @@ again: for (i = 0; i < LPURGE_HIST_MAX; i++) { if (&lpurge_hist[i] == ls) { lpurge_hist[i].ls_found = 0; - lpurge_hist[i].ls_space = 0; + lpurge_hist[i].ls_used_kb = 0; lpurge_hist[i].ls_stored = 0; lpurge_hist[i].ls_scan = 1; } else { @@ -1234,7 +1235,7 @@ again: */ } -static void lpurge_free_space(void) +static void lpurge_purge(void) { unsigned long long used_kb; int i, rc; @@ -1249,7 +1250,7 @@ static void lpurge_free_space(void) for (i = LPURGE_HIST_MAX - 1; i >= 0; i--) { struct lpurge_slot *ls = lpurge_hist + i; - if (ls->ls_found == 0/* || ls->ls_space == 0*/) + if (ls->ls_found == 0/* || ls->ls_used_kb == 0*/) continue; rc = lpurge_get_used_kb(&used_kb); @@ -1282,7 +1283,7 @@ static void lpurge_scan(void) for (i = 0; i < LPURGE_HIST_MAX; i++) { if (!lipe_list_empty(&lpurge_hist[i].ls_obj_list)) { LX_DEBUG("slot %d isnt empty: %lu in %lu\n", - i, lpurge_hist[i].ls_space, + i, lpurge_hist[i].ls_used_kb, lpurge_hist[i].ls_found); return; } @@ -1342,7 +1343,7 @@ static void lpurge_scan(void) continue; LX_DEBUG("%d (< %lu): %lu in %lu objects\n", i, lpurge_hist[i].ls_age, - lpurge_hist[i].ls_space, + lpurge_hist[i].ls_used_kb, lpurge_hist[i].ls_found); } } @@ -1723,7 +1724,7 @@ static void lpurge_usr1_handle(int sig) total_kb); #define HIST_FMT \ - " hist%u: { age: %lu, found: %lu, space: %lu, stored: %lu, nomirror_cnt: %lu, nomirror_space: %lu, nopfid_cnt: %lu, nopfid_space: %lu, notfirst_cnt: %lu, notfirst_space: %lu }\n" + " hist%u: { age: %lu, found: %lu, space: %lu, stored: %lu, nomirror_cnt: %lu, nomirror_used_kb: %lu, nopfid_cnt: %lu, nopfid_used_kb: %lu, notfirst_cnt: %lu, notfirst_used_kb: %lu }\n" fprintf(f, "hlists:\n"); for (i = LPURGE_HIST_MAX - 1; i >= 0; i--) { @@ -1733,10 +1734,10 @@ static void lpurge_usr1_handle(int sig) continue; fprintf(f, HIST_FMT, i, ls->ls_age, ls->ls_found, - ls->ls_space, ls->ls_stored, - ls->ls_nomirror_objs,ls->ls_nomirror_space, - ls->ls_nopfid_objs, ls->ls_nopfid_space, - ls->ls_notfirst_objs, ls->ls_notfirst_space); + ls->ls_used_kb, ls->ls_stored, + ls->ls_nomirror_objs, ls->ls_nomirror_used_kb, + ls->ls_nopfid_objs, ls->ls_nopfid_used_kb, + ls->ls_notfirst_objs, ls->ls_notfirst_used_kb); } fflush(f); @@ -1766,7 +1767,7 @@ static void lpurge_usr2_handle(int sig) /* * dump summary first - * {Summary: {"min_utime": 12345, "max_utime": 345669, "total_found": 100000, "total_stored": 100000, "total_space": 456780000}} + * {Summary: {"min_utime": 12345, "max_utime": 345669, "total_found": 100000, "total_stored": 100000, "total_used_kb": 456780000}} */ for (i = LPURGE_HIST_MAX - 1; i >= 0; i--) { struct lpurge_slot *ls = lpurge_hist + i; @@ -1778,7 +1779,7 @@ static void lpurge_usr2_handle(int sig) if (min_utime > ls->ls_min_utime) min_utime = ls->ls_min_utime; found_sum += ls->ls_found; - space_sum += ls->ls_space; + space_sum += ls->ls_used_kb; stored_sum += ls->ls_stored; } @@ -1838,7 +1839,7 @@ static void lpurge_usr2_handle(int sig) json_object_object_add(slot_stats, "stored", json_object_new_int64(ls->ls_stored)); json_object_object_add(slot_stats, "space", - json_object_new_int64(ls->ls_space)); + json_object_new_int64(ls->ls_used_kb)); snprintf(buff, sizeof(buff), "slot_%d", i); json_object_object_add(obj_slot, buff, slot_stats); @@ -2012,7 +2013,7 @@ int main(int argc, char **argv) lpurge_scan_rate = opt.o_scan_rate; lpurge_scan(); - lpurge_free_space(); + lpurge_purge(); /* device size can change runtime.. */ lpurge_configure_thresholds(); -- 1.8.3.1