EX-1878 lipe: resync all stale files

Add --resync-all-stales option (default is on).
This option allows lamigo by default to resync
all files if any component of a file
is stale regardless to pool or OST location.
If the option is off, lamigo works the old way

Test-Parameters: trivial testlist=hot-pools
Signed-off-by: Alexandre Ioffe <aioffe@ddn.com>
Change-Id: Ibc26a21fa99f75de87a8e0328b183d96b7548c1f
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53391
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lamigo.c b/lipe/src/lamigo.c
index 07f96aa..d463209 100644 (file)
--- a/lipe/src/lamigo.c
+++ b/lipe/src/lamigo.c
@@ -96,6 +96,7 @@
 #define DEF_SSH_EXEC_TO_SECS   (10 * 60) /* ssh exec timeout */
 #define DEF_MIRROR_STATS_SEC   6       /* mirror extend/resync stats interval */
 #define DEF_STATS_MISSED       50      /* stats missed before timeout */
+#define DEF_RESYNC_ALL_STALES  true    /* resync regardless of tgt and src pools */
 
 #define LAMIGO_USERFILE        "/var/lib/lamigo-%s.chlg"
 #define LAMIGO_DUMPFILE        "/var/run/lamigo-%s.stats"
@@ -153,6 +154,7 @@ static void usage(void)
               "\t--hot-after-idle, hot files after N idle periods to be mirrored (default: %d)\n"
               /* --src-free, deprecated  */
               /* --tgt-free, deprecated */
+              "\t--resync-all-stales=0|1 (default: %u)\n"
               "\t--slow-pool=POOL (default '%s')\n"
               "\t--slow-pool-max-used=MAX stop mirroring to POOL when %% used reaches MAX (default %d)\n"
               "\t--ssh-exec-timeout=SECS, ssh connection timeout for remote exec command (default: %d)\n"
@@ -177,6 +179,7 @@ static void usage(void)
               DEF_ALR_PERIOD_SECS,
               DEF_HOT_FRACTION,
               DEF_HOT_AFTER_IDLE,
+              DEF_RESYNC_ALL_STALES,
               DEF_SLOW_POOL,
               DEF_SLOW_POOL_MAX_USED,
               DEF_SSH_EXEC_TO_SECS);
@@ -289,6 +292,7 @@ struct options opt = {
        .o_alr_hot_after_idle = DEF_HOT_AFTER_IDLE,
        .o_include_dom = 0,
        .o_json_file = NULL,
+       .o_resync_all_stales = DEF_RESYNC_ALL_STALES,
 };
 
 struct history {
@@ -662,6 +666,7 @@ static void lamigo_dump_stats_file(void)
                "    fast_pool_max_used: %d\n"
                "    slow_pool_max_used: %d\n"
                "    include_dom: %d\n"
+               "    resync_all_stales: %u\n"
                "    ssh_exec_to: %ld\n",
                opt.o_slow_pool, opt.o_min_age, opt.o_cache_size,
                opt.o_rescan, opt.o_num_threads, opt.o_pool_refresh,
@@ -672,6 +677,7 @@ static void lamigo_dump_stats_file(void)
                opt.o_fast_pool_max_used,
                opt.o_slow_pool_max_used,
                opt.o_include_dom,
+               opt.o_resync_all_stales,
                opt.o_ssh_exec_to);
        for (pl = fast_pools; pl != NULL; pl = pl->pl_next, i++)
                fprintf(f, "pool %s:\n"
@@ -1234,12 +1240,223 @@ static int lamigo_objs_on_pool(struct pool_list *pool,
        return nr;
 }
 
-static int lamigo_striping_is_in_sync(struct lov_user_md *lum,
-                                     struct pool_list *src,
-                                     struct pool_list *tgt,
-                                     struct mirror_opts *mo)
+/* Resync all stale files
+ * Return:
+ *     AMIGO_RESYNC_NONE - Stale component was not found.
+ *                         Possibly need to extend.
+ *     AMIGO_RESYNC_RESYNC - Stale found.
+ *                           Resync unconditionally regardless of src and tgt
+ */
+static enum amigo_resync_type lamigo_find_stale(struct lov_user_md *lum)
+{
+       struct lov_comp_md_v1 *comp_v1;
+       struct lov_user_md_v3 *v3 = (struct lov_user_md_v3 *)lum;
+       int i;
+
+       /*
+        * for each mirror id:
+        *  - check it has even a single component on the fast pool,
+        *    then it's a subject to replication
+        *  - replica is suppposed to have all components on the slow pool
+        */
+       comp_v1 = (void *)v3;
+
+       for (i = 0; i < comp_v1->lcm_entry_count; i++) {
+               struct lov_comp_md_entry_v1 *entry = &comp_v1->lcm_entries[i];
+
+               if ((entry->lcme_flags & LCME_FL_INIT) == 0)
+                       continue;
+
+               v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i);
+
+               if (lamigo_entry_needs_resync(entry)) {
+                       stats.s_resync_stale++;
+                       return AMIGO_RESYNC_RESYNC;
+               }
+       }
+
+       return AMIGO_RESYNC_NONE;
+}
+
+/*      Does file require mirror extend or resync or none?
+ *     Resync is required regardless src or tgt
+ *
+ *     mo parameter is used only when AMIGO_RESYNC_EXTEND is returned
+ */
+static enum amigo_resync_type
+striping_is_in_sync_stales(struct lov_user_md *lum,
+                          struct pool_list *src,
+                          struct pool_list *tgt,
+                          struct mirror_opts *mo)
+{
+       struct lov_user_ost_data_v1 *objects;
+       struct lov_comp_md_v1 *comp_v1;
+       struct lov_user_md_v3 *v3 = (struct lov_user_md_v3 *)lum;
+       enum amigo_resync_type resync;
+       int i, k, rc;
+       int onsrc = 0, ontgt = 0, stripes = 0, stripesz = 0;
+       int mirid = -1;
+       int objs_in_mirror;
+       struct pool_list *pl, *found_on = NULL;
+       struct pool_list *src_pool = NULL;
+       bool need_prefer = false;
+
+       assert(mo);
+
+       if (v3->lmm_magic == LOV_USER_MAGIC_V3) {
+               /*
+                * fast path - if pool is specified in the striping,
+                * then we don't need to check each individual OST
+                * do replication only for specific pool
+                */
+               pl = lamigo_lookup_fast_pool(v3->lmm_pool_name);
+               if (pl) {
+                       stripes = v3->lmm_stripe_count;
+                       stripesz = v3->lmm_stripe_size;
+                       src_pool = pl;
+                       need_prefer = true;
+                       stats.s_extend_by_pool++;
+                       goto extend;
+               }
+       }
+
+       /* slow path - check individual stripes against OSTs we track */
+       if (v3->lmm_magic == LOV_USER_MAGIC_V1 ||
+           v3->lmm_magic == LOV_USER_MAGIC_V3) {
+               rc = lamigo_get_objects(v3, &objects, &stripes, &stripesz);
+               if (rc)
+                       return AMIGO_RESYNC_NONE;
+
+
+               k = lamigo_objs_on_pool(src, objects, stripes, &found_on);
+               if (k) {
+                       need_prefer = true;
+                       stats.s_extend_by_objects++;
+                       goto extend;
+               }
+               return AMIGO_RESYNC_NONE;
+       }
+
+       if (v3->lmm_magic != LOV_USER_MAGIC_COMP_V1) {
+               stats.s_skip_unknown++;
+               return AMIGO_RESYNC_NONE;
+       }
+
+       resync = lamigo_find_stale(lum);
+
+       if (resync != AMIGO_RESYNC_NONE)
+               return resync;
+
+       objs_in_mirror = 0;
+
+       /*
+        * for each mirror id:
+        *  - check it has even a single component on the fast pool,
+        *    then it's a subject to replication
+        *  - replica is suppposed to have all components on the slow pool
+        */
+       comp_v1 = (void *)v3;
+       for (i = 0; i < comp_v1->lcm_entry_count; i++) {
+               struct lov_comp_md_entry_v1 *entry = &comp_v1->lcm_entries[i];
+
+               if ((entry->lcme_flags & LCME_FL_INIT) == 0)
+                       continue;
+               v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i);
+
+               if (mirid != mirror_id_of(entry->lcme_id)) {
+                       if (ontgt) {
+                               if (ontgt != objs_in_mirror) {
+                                       /* part of this mirror is out of the
+                                        * slow pool we can't count it as a
+                                        * replica
+                                        */
+                               } else {
+                                       /* this is complete uptodate replica
+                                        * nothing to do...
+                                        */
+                                       stats.s_skip_insync++;
+                                       return AMIGO_RESYNC_NONE;
+                               }
+                       } else {
+                               /* not a replica, scan for another mirror */
+                       }
+
+                       /* new mirror starts */
+                       mirid = mirror_id_of(entry->lcme_id);
+                       ontgt = 0;
+                       objs_in_mirror = 0;
+                       /* do not reset onsrc as any object matters */
+               }
+
+               if (opt.o_include_dom && (v3->lmm_pattern & LOV_PATTERN_MDT)) {
+                       LX_DEBUG("DoM component\n");
+                       onsrc++;
+                       continue;
+               }
+
+               rc = lamigo_get_objects(v3, &objects, &stripes, &stripesz);
+               if (rc) {
+                       stats.s_skip_unknown++;
+                       return AMIGO_RESYNC_NONE;
+               }
+
+               /* total objects in this mirror */
+               objs_in_mirror += stripes;
+
+               /* if any object is located on the fast pool, then
+                * the file is subject to replication
+                */
+               k = lamigo_objs_on_pool(src, objects, stripes, &found_on);
+               onsrc += k;
+               if (k && (entry->lcme_flags & LCME_FL_PREF_RW) != LCME_FL_PREF_RW) {
+                       src_pool = found_on;
+                       need_prefer = true;
+               }
+
+               /* mirror has all objects on the slow pool - check for resync
+                * mirror has no all objects on the slow pool - create new
+                * mirror
+                */
+               if (v3->lmm_magic == LOV_USER_MAGIC_V3 &&
+                   strcmp(v3->lmm_pool_name, tgt->pl_pool) == 0) {
+                       /* trust pool name */
+                       k = stripes;
+               } else {
+                       k = lamigo_objs_on_pool(tgt, objects, stripes, NULL);
+               }
+
+               ontgt += k;
+       } /* for */
+
+       if (!onsrc) {
+               stats.s_skip_by_source++;
+               return AMIGO_RESYNC_NONE;
+       }
+
+       /* subject to replication */
+       if (!ontgt) {
+               /* no replica found, make one */
+               stats.s_extend_by_target++;
+               goto extend;
+       }
+
+       return AMIGO_RESYNC_NONE;
+
+extend:
+       mo->mo_stripes = stripes;
+       mo->mo_stripe_size = stripesz;
+       mo->mo_src_pool = found_on;
+       mo->mo_need_prefer = need_prefer;
+       return AMIGO_RESYNC_EXTEND;
+}
+
+/* mo is used only when AMIGO_RESYNC_EXTEND is returned */
+static enum amigo_resync_type
+striping_is_in_sync_non_stales(struct lov_user_md *lum,
+                              struct pool_list *src,
+                              struct pool_list *tgt,
+                              struct mirror_opts *mo)
 {
-       struct lov_comp_md_entry_v1 *entry;
        struct lov_user_ost_data_v1 *objects;
        struct lov_comp_md_v1 *comp_v1;
        struct lov_user_md_v3 *v3 = (struct lov_user_md_v3 *)lum;
@@ -1263,6 +1480,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum,
                 */
                mo->mo_stripes = stripes = v3->lmm_stripe_count;
                mo->mo_stripe_size = stripesz = v3->lmm_stripe_size;
+
                if ((pl = lamigo_lookup_fast_pool(v3->lmm_pool_name))) {
                        resync = AMIGO_RESYNC_EXTEND;
                        mo->mo_src_pool = pl;
@@ -1293,13 +1511,10 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum,
        }
 
        if (v3->lmm_magic != LOV_USER_MAGIC_COMP_V1) {
-               resync = AMIGO_RESYNC_NONE;
                stats.s_skip_unknown++;
                goto out;
        }
 
-       resync = AMIGO_RESYNC_NONE;
-       mirid = -1;
        stripes = 0;
        objs_in_mirror = 0;
        stale = 0;
@@ -1313,15 +1528,13 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum,
         */
        comp_v1 = (void *)v3;
        for (i = 0; i < comp_v1->lcm_entry_count; i++) {
-
-               entry = &comp_v1->lcm_entries[i];
+               struct lov_comp_md_entry_v1 *entry = &comp_v1->lcm_entries[i];
 
                if ((entry->lcme_flags & LCME_FL_INIT) == 0)
                        continue;
                v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i);
 
                if (mirid != mirror_id_of(entry->lcme_id)) {
-
                        if (ontgt) {
                                if (ontgt != objs_in_mirror) {
                                        /* part of this mirror is out of the
@@ -1396,7 +1609,6 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum,
                        ontgt += k;
                        if (lamigo_entry_needs_resync(entry))
                                stale++;
-
                }
 
                /* find largest stripe count */
@@ -1404,7 +1616,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum,
                        mo->mo_stripes = stripes;
                        mo->mo_stripe_size = stripesz;
                }
-       }
+       } /* for */
 
        if (!onsrc) {
                stats.s_skip_by_source++;
@@ -1441,6 +1653,18 @@ out:
 }
 
 static enum amigo_resync_type
+lamigo_striping_is_in_sync(struct lov_user_md *lum,
+                          struct pool_list *src,
+                          struct pool_list *tgt,
+                          struct mirror_opts *mo)
+{
+       if (opt.o_resync_all_stales)
+               return striping_is_in_sync_stales(lum, src, tgt, mo);
+       else
+               return striping_is_in_sync_non_stales(lum, src, tgt, mo);
+}
+
+static enum amigo_resync_type
 lamigo_check_user_rules(struct lipe_object_attrs *attrs,
                        struct lipe_policy_sysattrs *sysattrs)
 {
@@ -2229,6 +2453,7 @@ enum {
        LAMIGO_OPT_PERIOD_TIME,
        LAMIGO_OPT_POOL_REFRESH,
        LAMIGO_OPT_PROGRESS_INTV,
+       LAMIGO_OPT_RESYNC_ALL_STALES,
        LAMIGO_OPT_SLOW_POOL,
        LAMIGO_OPT_SLOW_POOL_MAX_USED,
        LAMIGO_OPT_SRC_DOM, /* == LAMIGO_OPT_INCLUDE_DOM + warning */
@@ -2276,6 +2501,7 @@ static struct option options[] = {
        { "pool-refresh", required_argument, NULL, LAMIGO_OPT_POOL_REFRESH },
        { "progress-interval", required_argument, NULL, LAMIGO_OPT_PROGRESS_INTV },
        { "rescan", no_argument, NULL, 'r'},
+       { "resync-all-stales", required_argument, NULL, LAMIGO_OPT_RESYNC_ALL_STALES },
        { "slow-pool", required_argument, NULL, LAMIGO_OPT_SLOW_POOL },
        { "slow-pool-max-used", required_argument, NULL, LAMIGO_OPT_SLOW_POOL_MAX_USED },
        { "src", required_argument, NULL, 's'},
@@ -2513,6 +2739,9 @@ static void lamigo_process_opt(int c, char *optarg)
        case LAMIGO_OPT_RESYNC_CMD:
                opt.o_resync_cmd = xstrdup(optarg);
                break;
+       case LAMIGO_OPT_RESYNC_ALL_STALES:
+               opt.o_resync_all_stales = atoi(optarg) ? true : false;
+               break;
        case LAMIGO_OPT_POOL_REFRESH:
                opt.o_pool_refresh = strtol(optarg, &endptr, 10);
                if (*endptr != '\0' || opt.o_pool_refresh < 1)

diff --git a/lipe/src/lamigo.h b/lipe/src/lamigo.h
index 0048527..0ec019b 100644 (file)
--- a/lipe/src/lamigo.h
+++ b/lipe/src/lamigo.h
@@ -134,6 +134,7 @@ struct options {
        int              o_alr_ignore_reads;
        int              o_alr_ignore_writes;
        int              o_include_dom;
+       bool             o_resync_all_stales; /* Resync stale regardless presencing in src pool */
        char            *o_heat_file;
 };
 extern struct options opt; /* opt is not a good global variable name. */

diff --git a/lustre/tests/hot-pools.sh b/lustre/tests/hot-pools.sh
index 5c74fcd..4f33e9c 100755 (executable)
--- a/lustre/tests/hot-pools.sh
+++ b/lustre/tests/hot-pools.sh
@@ -2583,6 +2583,69 @@ test_76() {
 }
 run_test 76 "lpurge to use 2nd-stripe objects"
 
+test_77() {
+       local file=${DIR}/${tfile}
+       local min_age=10
+       local tpool
+
+       init_hot_pools_env
+
+       LAMIGO_EXTRA_OPT="--heatfn=none" \
+       LAMIGO_AGE="${min_age}" \
+               start_lamigo_cmd
+       check_lamigo_is_started || error "failed to start lamigo"
+       stack_trap stop_lamigo_cmd
+
+       verify_one_lamigo_param 0 resync_all_stales 1
+
+       $LFS setstripe -N -E eof --pool $LAMIGO_TGT \
+                      -N -E eof --pool $LAMIGO_SRC $file-1 || \
+                      error "$LFS setstripe $file-1 failed"
+
+       dd if=/dev/zero of=$file-1 bs=64KB count=1 oflag=direct ||
+               error "cannot create '$file-1'"
+
+       wait_file_resync $file-1 $((min_age * 2)) || error "failed to resync $file-1"
+       verify_one_lamigo_param 0 resync-stale 1
+
+       $LFS setstripe -N3 $file-2 || error "setstripe $file-2 failed"
+       dd if=/dev/zero of=$file-2 bs=64KB count=1 oflag=direct ||
+               error "cannot create '$file-2'"
+
+       wait_file_resync $file-2 $((min_age * 2)) || error "failed to resync $file-2"
+       verify_one_lamigo_param 0 resync-stale 2
+
+       tpool=${TESTSUITE}_${testnum}
+       pool_add $tpool || error "failed to create OST pool '$tpool'"
+       pool_add_targets $tpool 0 0 ||
+               error "failed to add targets to OST pool '$tpool'"
+
+       $LFS setstripe -N -E eof --pool $LAMIGO_TGT \
+                      -N -E eof --pool $LAMIGO_SRC \
+                      -N -E eof --pool $tpool $file-3 || \
+                      error "setstripe for $file-3 failed"
+
+       $LFS setstripe --comp-set --comp-flags=stale --pool=$tpool $file-3 ||
+               error "setstripe stale for $file-3 failed"
+
+       wait_file_resync $file-3 $((min_age * 2)) || error "failed to resync $file-3"
+       verify_one_lamigo_param 0 resync-stale 3
+
+       $LFS setstripe -N -E eof --pool $LAMIGO_TGT \
+                      -N -E eof --pool $tpool $file-4 || \
+                      error "setstripe for $file-4 failed"
+
+       $LFS setstripe --comp-set --comp-flags=prefer --pool=$LAMIGO_TGT $file-4 ||
+               error "setstripe prefer for $file-4 failed"
+
+       dd if=/dev/zero of=$file-4 bs=64KB count=1 oflag=direct ||
+               error "cannot create '$file-4'"
+
+       wait_file_resync $file-4 $((min_age * 2)) || error "failed to resync $file-4"
+       verify_one_lamigo_param 0 resync-stale 4
+}
+run_test 77 "lamigo: verify --resync-all-stales option"
+
 complete_test $SECONDS
 check_and_cleanup_lustre
 exit_status