EX-3002 lipe: update lamigo command line options

Deprecate the '-s', '--src', '--src-dom' '--src-free' '-t', '--tgt',
and '--tgt-free' options to lamigo in favor of new options
  --fast-pool=POOL (default 'ddn_ssd')
  --fast-pool-max-used=MAX stop mirroring to POOL when % used reaches MAX (default 30)
  --slow-pool=POOL (default 'ddn_hdd')
  --slow-pool-max-used=MAX stop mirroring to POOL when % used reaches MAX (default 90)
  --include-dom treat DoM components as if they belong to fast pool

We continue to accept the old options with the same meanings but they
are removed from the help message and we print a warning when they are
used. Update hot-pools.sh to use the new options by default and add a
test that the old options still work.

Test-Parameters: trivial testlist=hot-pools
Signed-off-by: John L. Hammond <jhammond@whamcloud.com>
Change-Id: Ib940c188ccdd57710b68f5e97e725a26d0833356
Reviewed-on: https://review.whamcloud.com/45379
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Reviewed-by: Alexandre Ioffe <aioffe@ddn.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/46121
Reviewed-by: Jian Yu <yujian@whamcloud.com>

diff --git a/lipe/src/lamigo.c b/lipe/src/lamigo.c
index ab52d26..49011af 100644 (file)
--- a/lipe/src/lamigo.c
+++ b/lipe/src/lamigo.c
@@ -103,6 +103,7 @@ __u64 lamigo_rule_attrs; /* attributes needed to evalute the rules */
 
 static void usage(void)
 {
+       /* FIXME Deprecate all short options other than '-h'. */
        printf("\nUsage: %s [options]\n"
               "options:\n"
               "\t-a, --min-age, seconds before record handled (default: %u)\n"
@@ -119,13 +120,16 @@ static void usage(void)
               "\t-n, --thread-number, threads scanning MDT (default: %u)\n"
               "\t-o, --oss, OSS to fetch access logs from\n"
               "\t-r, --rescan, full MDT scan and replicate matched files\n"
-              "\t-s, --src, source OST pool (default: %s)\n"
-              "\t-t, --tgt, target OST pool (default: %s)\n"
+              /* -s, --src, source OST pool (default: %s) deprecated */
+              /* -t, --tgt, target OST pool (default: %s) deprecated */
               "\t-u, --user, changelog id (default: read from %s)\n"
               "\t-v, --verbose, produce more verbose ouput\n"
               "\t-w, --dump, stats file (via USR1 signal, default: %s)\n"
               "\t-W, --heat-dump, heat table file (via USR2 signal, default: %s)\n"
               "\t--alr-extra-args=ARGS, extra arguments for ALR (default: '%s')\n"
+              "\t--fast-pool=POOL (default '%s')\n"
+              "\t--fast-pool-max-used=MAX stop mirroring to POOL when %% used reaches MAX (default %d)\n"
+              "\t--include-dom treat DoM components as if they belong to fast pool\n"
               "\t--ignore-reads, ignore reads when calculating heat (default: no)\n"
               "\t--ignore-writes, ignore writes when calculating heat (default: no)\n"
               "\t--large-io, files with average IO greater won't be mirrored (default: 0 - disabled)\n"
@@ -134,8 +138,10 @@ static void usage(void)
               "\t--ofd-interval, seconds between gathering IO stats (default: period-time / 2)\n"
               "\t--hot-fraction, files with heat >= <hottest*fraction/100> are hot (default: %d)\n"
               "\t--hot-after-idle, hot files after N idle periods to be mirrored (default: %d)\n"
-              "\t--src-free, mirroring to source OST pool is prohibited if pool has less space (default: %d%%)\n"
-              "\t--tgt-free, mirroring to target OST pool is prohibited if pool has less space (default: %d%%)\n"
+              /* --src-free, deprecated  */
+              /* --tgt-free, deprecated */
+              "\t--slow-pool=POOL (default '%s')\n"
+              "\t--slow-pool-max-used=MAX stop mirroring to POOL when %% used reaches MAX (default %d)\n"
               "\t--ssh-log-verbosity=V, set SSH_OPTIONS_LOG_VERBOSITY to V (default: 0)\n"
               "\t--version, print version information and exit\n",
               program_invocation_short_name,
@@ -144,18 +150,19 @@ static void usage(void)
               DEF_MIRROR_CMD,
               DEF_PROGRESS_INTV >> 10,
               DEF_THREAD_COUNT,
-              DEF_FAST_POOL,
-              DEF_SLOW_POOL,
               LAMIGO_USERFILE,
               LAMIGO_DUMPFILE,
               LAMIGO_HEAT_FILE,
               DEF_ALR_EXTRA_ARGS,
+              DEF_FAST_POOL,
+              DEF_FAST_POOL_MAX_USED,
               DEF_ALR_PERIODS,
               DEF_ALR_PERIOD_SECS,
               DEF_HOT_FRACTION,
               DEF_HOT_AFTER_IDLE,
-              100 - DEF_FAST_POOL_MAX_USED,
-              100 - DEF_SLOW_POOL_MAX_USED);
+              DEF_SLOW_POOL,
+              DEF_SLOW_POOL_MAX_USED);
+
        exit(EXIT_SUCCESS);
 }
 
@@ -1747,11 +1754,14 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs)
 }
 
 enum {
-       LAMIGO_OPT_ALR_EXTRA_ARGS = 3,
+       LAMIGO_OPT_ALR_EXTRA_ARGS,
+       LAMIGO_OPT_FAST_POOL,
+       LAMIGO_OPT_FAST_POOL_MAX_USED,
        LAMIGO_OPT_HOT_AFTER_IDLE,
        LAMIGO_OPT_HOT_FRACTION,
        LAMIGO_OPT_IGNORE_READS,
        LAMIGO_OPT_IGNORE_WRITES,
+       LAMIGO_OPT_INCLUDE_DOM,
        LAMIGO_OPT_LARGE_IO,
        LAMIGO_OPT_MIRROR_CMD,
        LAMIGO_OPT_OFD_INTERVAL,
@@ -1759,51 +1769,57 @@ enum {
        LAMIGO_OPT_PERIOD_TIME,
        LAMIGO_OPT_POOL_REFRESH,
        LAMIGO_OPT_PROGRESS_INTV,
-       LAMIGO_OPT_SRC_DOM,
-       LAMIGO_OPT_SRC_FREE,
+       LAMIGO_OPT_SLOW_POOL,
+       LAMIGO_OPT_SLOW_POOL_MAX_USED,
+       LAMIGO_OPT_SRC_DOM, /* == LAMIGO_OPT_INCLUDE_DOM + warning */
+       LAMIGO_OPT_SRC_FREE, /* == LAMIGO_OPT_FAST_POOL_MAX_USED + math + warning */
        LAMIGO_OPT_SSH_LOG_VERBOSITY,
        LAMIGO_OPT_STATFS_REFRESH,
-       LAMIGO_OPT_TGT_FREE,
+       LAMIGO_OPT_TGT_FREE, /* == LAMIGO_OPT_SLOW_POOL_MAX_USED + math + warning */
        LAMIGO_OPT_VERSION,
 };
 
 static struct option options[] = {
-       { "ignore-reads", no_argument, NULL, LAMIGO_OPT_IGNORE_READS},
-       { "ignore-writes", no_argument, NULL, LAMIGO_OPT_IGNORE_WRITES},
-       { "large-io", required_argument, NULL, LAMIGO_OPT_LARGE_IO},
-       { "periods", required_argument, NULL, LAMIGO_OPT_PERIODS},
-       { "period-time", required_argument, NULL, LAMIGO_OPT_PERIOD_TIME},
-       { "ofd-interval", required_argument, NULL, LAMIGO_OPT_OFD_INTERVAL},
-       { "hot-fraction", required_argument, NULL, LAMIGO_OPT_HOT_FRACTION},
-       { "hot-after-idle", required_argument, NULL, LAMIGO_OPT_HOT_AFTER_IDLE},
-       { "alr-extra-args", required_argument, NULL, LAMIGO_OPT_ALR_EXTRA_ARGS},
        { "agent", required_argument, NULL, 'g'},
+       { "alr-extra-args", required_argument, NULL, LAMIGO_OPT_ALR_EXTRA_ARGS},
        { "config", required_argument, NULL, 'f' },
        { "debug", optional_argument, NULL, 'b' },
        { "dump", required_argument, NULL, 'w'},
-       { "help", no_argument, NULL, 'h' },
+       { "fast-pool", required_argument, NULL, LAMIGO_OPT_FAST_POOL },
+       { "fast-pool-max-used", required_argument, NULL, LAMIGO_OPT_FAST_POOL_MAX_USED },
+       { "heat-dump", required_argument, NULL, 'W'},
        { "heatfn", required_argument, NULL, 'H'},
+       { "help", no_argument, NULL, 'h' },
+       { "hot-after-idle", required_argument, NULL, LAMIGO_OPT_HOT_AFTER_IDLE},
+       { "hot-fraction", required_argument, NULL, LAMIGO_OPT_HOT_FRACTION},
        { "hot_after_idle", required_argument, NULL, 'I'},
-       { "heat-dump", required_argument, NULL, 'W'},
+       { "include-dom", no_argument, NULL, LAMIGO_OPT_INCLUDE_DOM },
+       { "ignore-reads", no_argument, NULL, LAMIGO_OPT_IGNORE_READS},
+       { "ignore-writes", no_argument, NULL, LAMIGO_OPT_IGNORE_WRITES},
+       { "large-io", required_argument, NULL, LAMIGO_OPT_LARGE_IO},
        { "max-cache", required_argument, NULL, 'c'},
+       { "mdt", required_argument, NULL, 'm' },
        { "min-age", required_argument, NULL, 'a'},
        { "mirror-cmd", required_argument, NULL, LAMIGO_OPT_MIRROR_CMD},
-       { "mdt", required_argument, NULL, 'm' },
-       { "mountpoint", required_argument, NULL, 'M' },
        { "mount", required_argument, NULL, 'M'},
+       { "mountpoint", required_argument, NULL, 'M' },
+       { "ofd-interval", required_argument, NULL, LAMIGO_OPT_OFD_INTERVAL},
        { "oss", required_argument, NULL, 'o'},
+       { "period-time", required_argument, NULL, LAMIGO_OPT_PERIOD_TIME},
+       { "periods", required_argument, NULL, LAMIGO_OPT_PERIODS},
        { "pool-refresh", required_argument, NULL, LAMIGO_OPT_POOL_REFRESH },
-       { "progress-interval", required_argument, NULL,
-               LAMIGO_OPT_PROGRESS_INTV },
+       { "progress-interval", required_argument, NULL, LAMIGO_OPT_PROGRESS_INTV },
        { "rescan", no_argument, NULL, 'r'},
-       { "ssh-log-verbosity", required_argument, NULL, LAMIGO_OPT_SSH_LOG_VERBOSITY },
+       { "slow-pool", required_argument, NULL, LAMIGO_OPT_SLOW_POOL },
+       { "slow-pool-max-used", required_argument, NULL, LAMIGO_OPT_SLOW_POOL_MAX_USED },
        { "src", required_argument, NULL, 's'},
        { "src-dom", no_argument, NULL, LAMIGO_OPT_SRC_DOM},
        { "src-free", required_argument, NULL, LAMIGO_OPT_SRC_FREE},
-       { "tgt-free", required_argument, NULL, LAMIGO_OPT_TGT_FREE},
+       { "ssh-log-verbosity", required_argument, NULL, LAMIGO_OPT_SSH_LOG_VERBOSITY },
        { "statfs-refresh", required_argument, NULL, LAMIGO_OPT_STATFS_REFRESH },
-       { "thread-number", required_argument, NULL, 'n' },
        { "tgt", required_argument, NULL, 't'},
+       { "tgt-free", required_argument, NULL, LAMIGO_OPT_TGT_FREE},
+       { "thread-number", required_argument, NULL, 'n' },
        { "user", required_argument, 0, 'u'},
        { "verbose", no_argument, NULL, 'v'},
        { "version", no_argument, NULL, LAMIGO_OPT_VERSION },
@@ -2034,21 +2050,36 @@ void lamigo_process_opt(int c, char *optarg)
                opt.o_alr_extra_args = optarg;
                break;
        case LAMIGO_OPT_SRC_FREE:
+               LX_WARN("option '--src-free' is deprecated, please use --fast-pool-max-used instead\n");
                opt.o_fast_pool_max_used = 100 - atoi(optarg);
                if (opt.o_fast_pool_max_used < 1 || opt.o_fast_pool_max_used > 99)
                        LX_FATAL("invalid argument ('%s') to --src-free\n", optarg);
                break;
+       case LAMIGO_OPT_FAST_POOL_MAX_USED:
+               opt.o_fast_pool_max_used = atoi(optarg);
+               if (opt.o_fast_pool_max_used < 1 || opt.o_fast_pool_max_used > 99)
+                       LX_FATAL("invalid argument ('%s') to '--fast-pool-max-used'\n", optarg);
+               break;
        case LAMIGO_OPT_SRC_DOM:
+               LX_WARN("option '--src-dom' is deprecated, please use --include-dom instead\n");
+               /* Fall through. */
+       case LAMIGO_OPT_INCLUDE_DOM:
                opt.o_include_dom = 1;
                break;
        case LAMIGO_OPT_SSH_LOG_VERBOSITY:
                lipe_ssh_log_verbosity = atoi(optarg);
                break;
        case LAMIGO_OPT_TGT_FREE:
+               LX_WARN("option '--tgt-free' is deprecated, please use --slow-pool-max-used instead\n");
                opt.o_slow_pool_max_used = 100 - atoi(optarg);
                if (opt.o_slow_pool_max_used < 1 || opt.o_slow_pool_max_used > 99)
                        LX_FATAL("invalid argument ('%s') to --tgt-free\n", optarg);
                break;
+       case LAMIGO_OPT_SLOW_POOL_MAX_USED:
+               opt.o_slow_pool_max_used = atoi(optarg);
+               if (opt.o_slow_pool_max_used < 1 || opt.o_slow_pool_max_used > 99)
+                       LX_FATAL("invalid argument ('%s') to '--slow-pool-max-used'\n", optarg);
+               break;
        case LAMIGO_OPT_VERSION:
                lipe_version();
                exit(EXIT_SUCCESS);
@@ -2125,9 +2156,15 @@ void lamigo_process_opt(int c, char *optarg)
                opt.o_rescan = true;
                break;
        case 's':
+               LX_WARN("options '-s' and '--src' are deprecated, please use --fast-pool instead\n");
+               /* Fall through. */
+       case LAMIGO_OPT_FAST_POOL:
                lamigo_parse_pool(optarg);
                break;
        case 't':
+               LX_WARN("options '-t' and '--tgt' are deprecated, please use --slow-pool instead\n");
+               /* Fall through. */
+       case LAMIGO_OPT_SLOW_POOL:
                opt.o_slow_pool = strdup(optarg);
                break;
        case 'u':

diff --git a/lustre/tests/hot-pools.sh b/lustre/tests/hot-pools.sh
index ec3e2c7..6ff413d 100755 (executable)
--- a/lustre/tests/hot-pools.sh
+++ b/lustre/tests/hot-pools.sh
@@ -254,19 +254,19 @@ start_one_lamigo_cmd() {
        done
 
        cmd+=${usr:+" -u $usr"}
-       cmd+=${LAMIGO_SRC:+" -s $LAMIGO_SRC"}
-       cmd+=${LAMIGO_TGT:+" -t $LAMIGO_TGT"}
-       cmd+=${LAMIGO_AGE:+" -a $LAMIGO_AGE"}
-       cmd+=${LAMIGO_THREAD_NUM:+" -n $LAMIGO_THREAD_NUM"}
-       cmd+=${LAMIGO_CACHE:+" -c $LAMIGO_CACHE"}
+       cmd+=${LAMIGO_SRC:+" --fast-pool=$LAMIGO_SRC"}
+       cmd+=${LAMIGO_TGT:+" --slow-pool=$LAMIGO_TGT"}
+       cmd+=${LAMIGO_AGE:+" --min-age=$LAMIGO_AGE"}
+       cmd+=${LAMIGO_THREAD_NUM:+" --thread-number=$LAMIGO_THREAD_NUM"}
+       cmd+=${LAMIGO_CACHE:+" --max-cache=$LAMIGO_CACHE"}
        cmd+=${LAMIGO_PROG_INTV:+" --progress-interval=$LAMIGO_PROG_INTV"}
        cmd+=${LAMIGO_MIRROR_CMD:+" --mirror-cmd=\"$LAMIGO_MIRROR_CMD\""}
 
        cmd+=${LAMIGO_DUMP:+" -w ${LAMIGO_DUMP}.$mdt"}
 
-       ! $LAMIGO_DEBUG || cmd+=" -b"
-       ! $LAMIGO_RESCAN || cmd+=" -r"
-       ! $LAMIGO_VERBOSE || cmd+=" -v"
+       ! $LAMIGO_DEBUG || cmd+=" --debug"
+       ! $LAMIGO_RESCAN || cmd+=" --rescan"
+       ! $LAMIGO_VERBOSE || cmd+=" --verbose"
 
        cmd+=" $LAMIGO_EXTRA_OPT"
 
@@ -1915,7 +1915,7 @@ test_58() {
        init_hot_pools_env
 
        # start lamigo
-       LAMIGO_EXTRA_OPT="--src-dom" start_lamigo_cmd
+       LAMIGO_EXTRA_OPT="--include-dom" start_lamigo_cmd
        check_lamigo_is_started || error "failed to start lamigo"
        stack_trap stop_lamigo_cmd
 
@@ -2027,6 +2027,42 @@ test_59() {
 }
 run_test 59 "lpurge: check layout before opening"
 
+test_70() {
+       init_hot_pools_env
+       local src=${LAMIGO_SRC}
+       local tgt=${LAMIGO_TGT}
+       local src_free=72 # "default" is 70
+       local tgt_free=12 # "default" is 10
+
+       LAMIGO_SRC= LAMIGO_TGT= LAMIGO_EXTRA_OPT="--src=${src} --src-dom --src-free=${src_free} --tgt=${tgt} --tgt-free=${tgt_free}" \
+               start_one_lamigo_cmd
+       check_one_lamigo_is_started || error "failed to start lamigo"
+       stack_trap stop_one_lamigo_cmd
+
+       verify_one_lamigo_param 0 fast_pool "$src"
+       verify_one_lamigo_param 0 fast_pool_max_used $((100 - src_free))
+       verify_one_lamigo_param 0 slow_pool "$tgt"
+       verify_one_lamigo_param 0 slow_pool_max_used $((100 - tgt_free))
+       verify_one_lamigo_param 0 include_dom 1
+}
+run_test 70 "lamigo: old options still work"
+
+test_71() {
+       init_hot_pools_env
+       local fast_pool_max_used=28 # default is 30
+       local slow_pool_max_used=88 # default is 90
+
+       LAMIGO_EXTRA_OPT="--fast-pool-max-used=${fast_pool_max_used} --slow-pool-max-used=${slow_pool_max_used} --include-dom" \
+               start_one_lamigo_cmd
+       check_one_lamigo_is_started || error "failed to start lamigo"
+       stack_trap stop_one_lamigo_cmd
+
+       verify_one_lamigo_param 0 fast_pool_max_used ${fast_pool_max_used}
+       verify_one_lamigo_param 0 slow_pool_max_used ${slow_pool_max_used}
+       verify_one_lamigo_param 0 include_dom 1
+}
+run_test 71 "lamigo: new options work"
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status