From d60b862aa9a4497dd604ecbbc2ebb9d37b8fa3cd Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Thu, 10 Mar 2022 11:00:04 -0600 Subject: [PATCH] EX-4963 lipe: rescan after registering a new changelog user In lamigo, if we register a new changelog user then force a rescan of the device. Add hot-pools test_6c() to verify. Adjust hot-pools to be more precise about param matching. Test-Parameters: trivial testlist=hot-pools Test-Parameters: trivial testlist=hot-pools env=ONLY=6c,ONLY_REPEAT=20 Signed-off-by: John L. Hammond Change-Id: I9576d50f87ffe0eb5e44c62087fca317ddce50c7 Reviewed-on: https://review.whamcloud.com/46781 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alex Zhuravlev Reviewed-by: Alexandre Ioffe --- lipe/src/lamigo.c | 26 +++++++++++++++++++++++--- lustre/tests/hot-pools.sh | 28 ++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lipe/src/lamigo.c b/lipe/src/lamigo.c index 5260ec6..6fb66bb 100644 --- a/lipe/src/lamigo.c +++ b/lipe/src/lamigo.c @@ -258,6 +258,9 @@ struct history { }; struct stats { + time_t s_scan_begin; + time_t s_scan_end; + unsigned long s_scan_replicated; /* s_replicated at end of scan */ unsigned long s_read; /* llog records read */ unsigned long s_skipped; /* llog records skipped */ unsigned long s_processed; /* llog records processed */ @@ -524,6 +527,9 @@ static void lamigo_dump_stats_file(void) pl->pl_used_kb, pl->pl_total_kb, (int)pl->pl_is_open); fprintf(f, "stats:\n" + " scan_begin: %ld\n" + " scan_end: %ld\n" + " scan_replicated: %ld\n" " read: %lu\n" " skipped: %lu\n" " processed: %lu\n" @@ -545,6 +551,9 @@ static void lamigo_dump_stats_file(void) " skip-insync: %lu\n" " skip-by-source: %lu\n" " extend-by-target: %lu\n", + stats.s_scan_begin, + stats.s_scan_end, + stats.s_scan_replicated, stats.s_read, stats.s_skipped, stats.s_processed, stats.s_removed, stats.s_dups, stats.s_spawned, stats.s_replicated, stats.s_busy, head.lh_cached_count, @@ -2645,6 +2654,8 @@ static int lamigo_rescan(void) memset(&result, 0, sizeof(result)); gettimeofday(&result.sr_time_start, NULL); + stats.s_scan_begin = time(NULL); + rc = lipe_scan(&instance, &policy, &result, @@ -2654,6 +2665,10 @@ static int lamigo_rescan(void) "test" /* workspace */, true /* abort_failure */, &ldd_err); + + stats.s_scan_end = time(NULL); + stats.s_scan_replicated = stats.s_replicated; + if (rc) { LX_ERROR("cannot scan '%s': %s\n", instance.li_device, strerror(-rc)); return rc; @@ -2862,10 +2877,15 @@ static void lamigo_changelog_user_register(void) lamigo_changelog_user_find_any(); - if (lamigo_changelog_user != NULL) - return; + if (lamigo_changelog_user == NULL) + LX_FATAL("cannot find just registered changelog user\n"); - LX_FATAL("cannot find just registered changelog user\n"); + /* If a new changelog user was just registered then either this is the + * first time lamigo was run on the filesystem, or it has been some + * time since it was last run and changes were not logged. Run a + * full device scan to look for any files that need processing. + */ + opt.o_rescan = true; } static unsigned long progress_timestamp; diff --git a/lustre/tests/hot-pools.sh b/lustre/tests/hot-pools.sh index 62fff2e..0ec9bc1 100755 --- a/lustre/tests/hot-pools.sh +++ b/lustre/tests/hot-pools.sh @@ -542,8 +542,9 @@ verify_one_lamigo_param() { local j for ((j = 0; j < $max; j++)); do + # FIXME Use yq to extract param value. value=$(dump_one_lamigo_stats $i | - awk '/'$param':/ { print $2 }') + awk -v param="$param" '$1 == param ":" { print $2 }') [[ -n "$expected" ]] || { echo "Default value '$value' is used for '$param'." @@ -919,7 +920,7 @@ verify_one_lpurge_param() { for ((j = 0; j < $max; j++)); do value=$(dump_one_lpurge_stats $i | - awk '/'$param':/ { print $2 }') + awk -v param="$param" '$1 == param ":" { print $2 }') [[ -n "$expected" ]] || { echo "Default value '$value' is used for '$param'." @@ -1208,6 +1209,29 @@ test_6b() { } run_test 6b "lamigo: start with no changelog user" +test_6c() { + local real_user + local scan_begin + + INIT_HOT_POOLS_CHANGELOG=false init_hot_pools_env + + LAMIGO_USR= start_one_lamigo_cmd + check_one_lamigo_is_started || error "failed to start lamigo" + stack_trap stop_one_lamigo_cmd + + real_user=$(get_lamigo_chlg) + echo "New Changelog user is '$real_user'" + stack_trap "__changelog_deregister mds1 $real_user" + stack_trap "rm -f ${LAMIGO_USERFILE[0]}" + + sleep 5 + scan_begin=$(dump_one_lamigo_stats | yq .stats.scan_begin) + echo "scan_begin = '$scan_begin'" + ((scan_begin != 0)) || + error "lamigo did not rescan" +} +run_test 6c "lamigo should rescan after registering a new changelog user" + test_7() { init_hot_pools_env -- 1.8.3.1