From: Alexandre Ioffe Date: Fri, 8 Dec 2023 22:17:37 +0000 (-0800) Subject: EX-1878 lipe: resync all stale files X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=3daed518df8e21672e785987c065f9424bb5de5e;p=fs%2Flustre-release.git EX-1878 lipe: resync all stale files Add --resync-all-stales option (default is on). This option allows lamigo by default to resync all files if any component of a file is stale regardless to pool or OST location. If the option is off, lamigo works the old way Test-Parameters: trivial testlist=hot-pools Signed-off-by: Alexandre Ioffe Change-Id: Ibc26a21fa99f75de87a8e0328b183d96b7548c1f Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53391 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/lipe/src/lamigo.c b/lipe/src/lamigo.c index 07f96aa..d463209 100644 --- a/lipe/src/lamigo.c +++ b/lipe/src/lamigo.c @@ -96,6 +96,7 @@ #define DEF_SSH_EXEC_TO_SECS (10 * 60) /* ssh exec timeout */ #define DEF_MIRROR_STATS_SEC 6 /* mirror extend/resync stats interval */ #define DEF_STATS_MISSED 50 /* stats missed before timeout */ +#define DEF_RESYNC_ALL_STALES true /* resync regardless of tgt and src pools */ #define LAMIGO_USERFILE "/var/lib/lamigo-%s.chlg" #define LAMIGO_DUMPFILE "/var/run/lamigo-%s.stats" @@ -153,6 +154,7 @@ static void usage(void) "\t--hot-after-idle, hot files after N idle periods to be mirrored (default: %d)\n" /* --src-free, deprecated */ /* --tgt-free, deprecated */ + "\t--resync-all-stales=0|1 (default: %u)\n" "\t--slow-pool=POOL (default '%s')\n" "\t--slow-pool-max-used=MAX stop mirroring to POOL when %% used reaches MAX (default %d)\n" "\t--ssh-exec-timeout=SECS, ssh connection timeout for remote exec command (default: %d)\n" @@ -177,6 +179,7 @@ static void usage(void) DEF_ALR_PERIOD_SECS, DEF_HOT_FRACTION, DEF_HOT_AFTER_IDLE, + DEF_RESYNC_ALL_STALES, DEF_SLOW_POOL, DEF_SLOW_POOL_MAX_USED, DEF_SSH_EXEC_TO_SECS); @@ -289,6 +292,7 @@ struct options opt = { .o_alr_hot_after_idle = DEF_HOT_AFTER_IDLE, .o_include_dom = 0, .o_json_file = NULL, + .o_resync_all_stales = DEF_RESYNC_ALL_STALES, }; struct history { @@ -662,6 +666,7 @@ static void lamigo_dump_stats_file(void) " fast_pool_max_used: %d\n" " slow_pool_max_used: %d\n" " include_dom: %d\n" + " resync_all_stales: %u\n" " ssh_exec_to: %ld\n", opt.o_slow_pool, opt.o_min_age, opt.o_cache_size, opt.o_rescan, opt.o_num_threads, opt.o_pool_refresh, @@ -672,6 +677,7 @@ static void lamigo_dump_stats_file(void) opt.o_fast_pool_max_used, opt.o_slow_pool_max_used, opt.o_include_dom, + opt.o_resync_all_stales, opt.o_ssh_exec_to); for (pl = fast_pools; pl != NULL; pl = pl->pl_next, i++) fprintf(f, "pool %s:\n" @@ -1234,12 +1240,223 @@ static int lamigo_objs_on_pool(struct pool_list *pool, return nr; } -static int lamigo_striping_is_in_sync(struct lov_user_md *lum, - struct pool_list *src, - struct pool_list *tgt, - struct mirror_opts *mo) +/* Resync all stale files + * Return: + * AMIGO_RESYNC_NONE - Stale component was not found. + * Possibly need to extend. + * AMIGO_RESYNC_RESYNC - Stale found. + * Resync unconditionally regardless of src and tgt + */ +static enum amigo_resync_type lamigo_find_stale(struct lov_user_md *lum) +{ + struct lov_comp_md_v1 *comp_v1; + struct lov_user_md_v3 *v3 = (struct lov_user_md_v3 *)lum; + int i; + + /* + * for each mirror id: + * - check it has even a single component on the fast pool, + * then it's a subject to replication + * - replica is suppposed to have all components on the slow pool + */ + comp_v1 = (void *)v3; + + for (i = 0; i < comp_v1->lcm_entry_count; i++) { + struct lov_comp_md_entry_v1 *entry = &comp_v1->lcm_entries[i]; + + if ((entry->lcme_flags & LCME_FL_INIT) == 0) + continue; + + v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i); + + if (lamigo_entry_needs_resync(entry)) { + stats.s_resync_stale++; + return AMIGO_RESYNC_RESYNC; + } + } + + return AMIGO_RESYNC_NONE; +} + +/* Does file require mirror extend or resync or none? + * Resync is required regardless src or tgt + * + * mo parameter is used only when AMIGO_RESYNC_EXTEND is returned + */ +static enum amigo_resync_type +striping_is_in_sync_stales(struct lov_user_md *lum, + struct pool_list *src, + struct pool_list *tgt, + struct mirror_opts *mo) +{ + struct lov_user_ost_data_v1 *objects; + struct lov_comp_md_v1 *comp_v1; + struct lov_user_md_v3 *v3 = (struct lov_user_md_v3 *)lum; + enum amigo_resync_type resync; + int i, k, rc; + int onsrc = 0, ontgt = 0, stripes = 0, stripesz = 0; + int mirid = -1; + int objs_in_mirror; + struct pool_list *pl, *found_on = NULL; + struct pool_list *src_pool = NULL; + bool need_prefer = false; + + assert(mo); + + if (v3->lmm_magic == LOV_USER_MAGIC_V3) { + /* + * fast path - if pool is specified in the striping, + * then we don't need to check each individual OST + * do replication only for specific pool + */ + pl = lamigo_lookup_fast_pool(v3->lmm_pool_name); + if (pl) { + stripes = v3->lmm_stripe_count; + stripesz = v3->lmm_stripe_size; + src_pool = pl; + need_prefer = true; + stats.s_extend_by_pool++; + goto extend; + } + } + + /* slow path - check individual stripes against OSTs we track */ + if (v3->lmm_magic == LOV_USER_MAGIC_V1 || + v3->lmm_magic == LOV_USER_MAGIC_V3) { + rc = lamigo_get_objects(v3, &objects, &stripes, &stripesz); + if (rc) + return AMIGO_RESYNC_NONE; + + + k = lamigo_objs_on_pool(src, objects, stripes, &found_on); + if (k) { + need_prefer = true; + stats.s_extend_by_objects++; + goto extend; + } + return AMIGO_RESYNC_NONE; + } + + if (v3->lmm_magic != LOV_USER_MAGIC_COMP_V1) { + stats.s_skip_unknown++; + return AMIGO_RESYNC_NONE; + } + + resync = lamigo_find_stale(lum); + + if (resync != AMIGO_RESYNC_NONE) + return resync; + + objs_in_mirror = 0; + + /* + * for each mirror id: + * - check it has even a single component on the fast pool, + * then it's a subject to replication + * - replica is suppposed to have all components on the slow pool + */ + comp_v1 = (void *)v3; + for (i = 0; i < comp_v1->lcm_entry_count; i++) { + struct lov_comp_md_entry_v1 *entry = &comp_v1->lcm_entries[i]; + + if ((entry->lcme_flags & LCME_FL_INIT) == 0) + continue; + v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i); + + if (mirid != mirror_id_of(entry->lcme_id)) { + if (ontgt) { + if (ontgt != objs_in_mirror) { + /* part of this mirror is out of the + * slow pool we can't count it as a + * replica + */ + } else { + /* this is complete uptodate replica + * nothing to do... + */ + stats.s_skip_insync++; + return AMIGO_RESYNC_NONE; + } + } else { + /* not a replica, scan for another mirror */ + } + + /* new mirror starts */ + mirid = mirror_id_of(entry->lcme_id); + ontgt = 0; + objs_in_mirror = 0; + /* do not reset onsrc as any object matters */ + } + + if (opt.o_include_dom && (v3->lmm_pattern & LOV_PATTERN_MDT)) { + LX_DEBUG("DoM component\n"); + onsrc++; + continue; + } + + rc = lamigo_get_objects(v3, &objects, &stripes, &stripesz); + if (rc) { + stats.s_skip_unknown++; + return AMIGO_RESYNC_NONE; + } + + /* total objects in this mirror */ + objs_in_mirror += stripes; + + /* if any object is located on the fast pool, then + * the file is subject to replication + */ + k = lamigo_objs_on_pool(src, objects, stripes, &found_on); + onsrc += k; + if (k && (entry->lcme_flags & LCME_FL_PREF_RW) != LCME_FL_PREF_RW) { + src_pool = found_on; + need_prefer = true; + } + + /* mirror has all objects on the slow pool - check for resync + * mirror has no all objects on the slow pool - create new + * mirror + */ + if (v3->lmm_magic == LOV_USER_MAGIC_V3 && + strcmp(v3->lmm_pool_name, tgt->pl_pool) == 0) { + /* trust pool name */ + k = stripes; + } else { + k = lamigo_objs_on_pool(tgt, objects, stripes, NULL); + } + + ontgt += k; + } /* for */ + + if (!onsrc) { + stats.s_skip_by_source++; + return AMIGO_RESYNC_NONE; + } + + /* subject to replication */ + if (!ontgt) { + /* no replica found, make one */ + stats.s_extend_by_target++; + goto extend; + } + + return AMIGO_RESYNC_NONE; + +extend: + mo->mo_stripes = stripes; + mo->mo_stripe_size = stripesz; + mo->mo_src_pool = found_on; + mo->mo_need_prefer = need_prefer; + return AMIGO_RESYNC_EXTEND; +} + +/* mo is used only when AMIGO_RESYNC_EXTEND is returned */ +static enum amigo_resync_type +striping_is_in_sync_non_stales(struct lov_user_md *lum, + struct pool_list *src, + struct pool_list *tgt, + struct mirror_opts *mo) { - struct lov_comp_md_entry_v1 *entry; struct lov_user_ost_data_v1 *objects; struct lov_comp_md_v1 *comp_v1; struct lov_user_md_v3 *v3 = (struct lov_user_md_v3 *)lum; @@ -1263,6 +1480,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, */ mo->mo_stripes = stripes = v3->lmm_stripe_count; mo->mo_stripe_size = stripesz = v3->lmm_stripe_size; + if ((pl = lamigo_lookup_fast_pool(v3->lmm_pool_name))) { resync = AMIGO_RESYNC_EXTEND; mo->mo_src_pool = pl; @@ -1293,13 +1511,10 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, } if (v3->lmm_magic != LOV_USER_MAGIC_COMP_V1) { - resync = AMIGO_RESYNC_NONE; stats.s_skip_unknown++; goto out; } - resync = AMIGO_RESYNC_NONE; - mirid = -1; stripes = 0; objs_in_mirror = 0; stale = 0; @@ -1313,15 +1528,13 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, */ comp_v1 = (void *)v3; for (i = 0; i < comp_v1->lcm_entry_count; i++) { - - entry = &comp_v1->lcm_entries[i]; + struct lov_comp_md_entry_v1 *entry = &comp_v1->lcm_entries[i]; if ((entry->lcme_flags & LCME_FL_INIT) == 0) continue; v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i); if (mirid != mirror_id_of(entry->lcme_id)) { - if (ontgt) { if (ontgt != objs_in_mirror) { /* part of this mirror is out of the @@ -1396,7 +1609,6 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, ontgt += k; if (lamigo_entry_needs_resync(entry)) stale++; - } /* find largest stripe count */ @@ -1404,7 +1616,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, mo->mo_stripes = stripes; mo->mo_stripe_size = stripesz; } - } + } /* for */ if (!onsrc) { stats.s_skip_by_source++; @@ -1441,6 +1653,18 @@ out: } static enum amigo_resync_type +lamigo_striping_is_in_sync(struct lov_user_md *lum, + struct pool_list *src, + struct pool_list *tgt, + struct mirror_opts *mo) +{ + if (opt.o_resync_all_stales) + return striping_is_in_sync_stales(lum, src, tgt, mo); + else + return striping_is_in_sync_non_stales(lum, src, tgt, mo); +} + +static enum amigo_resync_type lamigo_check_user_rules(struct lipe_object_attrs *attrs, struct lipe_policy_sysattrs *sysattrs) { @@ -2229,6 +2453,7 @@ enum { LAMIGO_OPT_PERIOD_TIME, LAMIGO_OPT_POOL_REFRESH, LAMIGO_OPT_PROGRESS_INTV, + LAMIGO_OPT_RESYNC_ALL_STALES, LAMIGO_OPT_SLOW_POOL, LAMIGO_OPT_SLOW_POOL_MAX_USED, LAMIGO_OPT_SRC_DOM, /* == LAMIGO_OPT_INCLUDE_DOM + warning */ @@ -2276,6 +2501,7 @@ static struct option options[] = { { "pool-refresh", required_argument, NULL, LAMIGO_OPT_POOL_REFRESH }, { "progress-interval", required_argument, NULL, LAMIGO_OPT_PROGRESS_INTV }, { "rescan", no_argument, NULL, 'r'}, + { "resync-all-stales", required_argument, NULL, LAMIGO_OPT_RESYNC_ALL_STALES }, { "slow-pool", required_argument, NULL, LAMIGO_OPT_SLOW_POOL }, { "slow-pool-max-used", required_argument, NULL, LAMIGO_OPT_SLOW_POOL_MAX_USED }, { "src", required_argument, NULL, 's'}, @@ -2513,6 +2739,9 @@ static void lamigo_process_opt(int c, char *optarg) case LAMIGO_OPT_RESYNC_CMD: opt.o_resync_cmd = xstrdup(optarg); break; + case LAMIGO_OPT_RESYNC_ALL_STALES: + opt.o_resync_all_stales = atoi(optarg) ? true : false; + break; case LAMIGO_OPT_POOL_REFRESH: opt.o_pool_refresh = strtol(optarg, &endptr, 10); if (*endptr != '\0' || opt.o_pool_refresh < 1) diff --git a/lipe/src/lamigo.h b/lipe/src/lamigo.h index 0048527..0ec019b 100644 --- a/lipe/src/lamigo.h +++ b/lipe/src/lamigo.h @@ -134,6 +134,7 @@ struct options { int o_alr_ignore_reads; int o_alr_ignore_writes; int o_include_dom; + bool o_resync_all_stales; /* Resync stale regardless presencing in src pool */ char *o_heat_file; }; extern struct options opt; /* opt is not a good global variable name. */ diff --git a/lustre/tests/hot-pools.sh b/lustre/tests/hot-pools.sh index 5c74fcd..4f33e9c 100755 --- a/lustre/tests/hot-pools.sh +++ b/lustre/tests/hot-pools.sh @@ -2583,6 +2583,69 @@ test_76() { } run_test 76 "lpurge to use 2nd-stripe objects" +test_77() { + local file=${DIR}/${tfile} + local min_age=10 + local tpool + + init_hot_pools_env + + LAMIGO_EXTRA_OPT="--heatfn=none" \ + LAMIGO_AGE="${min_age}" \ + start_lamigo_cmd + check_lamigo_is_started || error "failed to start lamigo" + stack_trap stop_lamigo_cmd + + verify_one_lamigo_param 0 resync_all_stales 1 + + $LFS setstripe -N -E eof --pool $LAMIGO_TGT \ + -N -E eof --pool $LAMIGO_SRC $file-1 || \ + error "$LFS setstripe $file-1 failed" + + dd if=/dev/zero of=$file-1 bs=64KB count=1 oflag=direct || + error "cannot create '$file-1'" + + wait_file_resync $file-1 $((min_age * 2)) || error "failed to resync $file-1" + verify_one_lamigo_param 0 resync-stale 1 + + $LFS setstripe -N3 $file-2 || error "setstripe $file-2 failed" + dd if=/dev/zero of=$file-2 bs=64KB count=1 oflag=direct || + error "cannot create '$file-2'" + + wait_file_resync $file-2 $((min_age * 2)) || error "failed to resync $file-2" + verify_one_lamigo_param 0 resync-stale 2 + + tpool=${TESTSUITE}_${testnum} + pool_add $tpool || error "failed to create OST pool '$tpool'" + pool_add_targets $tpool 0 0 || + error "failed to add targets to OST pool '$tpool'" + + $LFS setstripe -N -E eof --pool $LAMIGO_TGT \ + -N -E eof --pool $LAMIGO_SRC \ + -N -E eof --pool $tpool $file-3 || \ + error "setstripe for $file-3 failed" + + $LFS setstripe --comp-set --comp-flags=stale --pool=$tpool $file-3 || + error "setstripe stale for $file-3 failed" + + wait_file_resync $file-3 $((min_age * 2)) || error "failed to resync $file-3" + verify_one_lamigo_param 0 resync-stale 3 + + $LFS setstripe -N -E eof --pool $LAMIGO_TGT \ + -N -E eof --pool $tpool $file-4 || \ + error "setstripe for $file-4 failed" + + $LFS setstripe --comp-set --comp-flags=prefer --pool=$LAMIGO_TGT $file-4 || + error "setstripe prefer for $file-4 failed" + + dd if=/dev/zero of=$file-4 bs=64KB count=1 oflag=direct || + error "cannot create '$file-4'" + + wait_file_resync $file-4 $((min_age * 2)) || error "failed to resync $file-4" + verify_one_lamigo_param 0 resync-stale 4 +} +run_test 77 "lamigo: verify --resync-all-stales option" + complete_test $SECONDS check_and_cleanup_lustre exit_status