From ecdcaa398668291dcfad26c4e21745dbfee19bc4 Mon Sep 17 00:00:00 2001 From: Maximilian Dilger Date: Thu, 6 Jun 2024 01:27:05 -0400 Subject: [PATCH] LU-17512 utils: new ? operator for jobid_name Added new ? operator when setting the jobid_name. The intended use is: "jobid_name=%j?%H" This will use the jobid if it is available and otherwise uses the short hostname. Signed-off-by: Maximilian Dilger Change-Id: I418860fce5a81aa8a0a0a43c2d8bdb6d107779f9 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55332 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin Reviewed-by: Andreas Dilger Reviewed-by: Thomas Bertschinger --- lustre/obdclass/jobid.c | 7 ++ lustre/tests/sanity.sh | 287 +++++++++++++++++++++++++++--------------------- 2 files changed, 166 insertions(+), 128 deletions(-) diff --git a/lustre/obdclass/jobid.c b/lustre/obdclass/jobid.c index 0d7e8a6..3315bd4 100644 --- a/lustre/obdclass/jobid.c +++ b/lustre/obdclass/jobid.c @@ -692,6 +692,7 @@ static int jobid_print_current_comm(char *jobid, ssize_t joblen) * %h = hostname * %H = short hostname * %j = jobid from environment + * ? = for use between %j and %H. Uses jobid if set, otherwise use Hostname * %p = pid * %u = uid * @@ -746,6 +747,12 @@ static int jobid_interpret_string(const char *jobfmt, char *jobid, l = jobid_get_from_cache(jobid, joblen); if (l < 0) l = 0; + if (*jobfmt == '?') { + if (l == 0) + jobfmt++; + else + jobfmt += 3; + } break; case 'p': /* process ID */ l = snprintf(jobid, joblen, "%u", current->pid); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index d6276c6..9a5fdca 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -9167,6 +9167,135 @@ test_56ec() { } run_test 56ec "check lfs getstripe,setstripe --hex --yaml" +# LU-15565 +test_56ed() { + local new=$DIR/$tdir/$tfile-layout-new + local old=$DIR/$tdir/$tfile-layout-old + + test_mkdir $DIR/$tdir + + cat << PFL_LAYOUT_NEW > $new +lcm_layout_gen: 3 +lcm_mirror_count: 1 +lcm_entry_count: 3 +components: + - lcme_id: 1 + lcme_mirror_id: 0 + lcme_flags: init + lcme_extent.e_start: 0 + lcme_extent.e_end: 1073741824 + sub_layout: + lmm_stripe_count: 1 + lmm_stripe_size: 4194304 + lmm_pattern: raid0 + lmm_layout_gen: 0 + lmm_stripe_offset: 0 + lmm_objects: + - l_ost_idx: 0 + l_fid: 0x280000400:0x2:0x0 + - lcme_id: 2 + lcme_mirror_id: 0 + lcme_flags: 0 + lcme_extent.e_start: 1073741824 + lcme_extent.e_end: 4294967296 + sub_layout: + lmm_stripe_count: 4 + lmm_stripe_size: 4194304 + lmm_pattern: raid0 + lmm_layout_gen: 0 + lmm_stripe_offset: -1 + - lcme_id: 3 + lcme_mirror_id: 0 + lcme_flags: 0 + lcme_extent.e_start: 4294967296 + lcme_extent.e_end: EOF + sub_layout: + lmm_stripe_count: 4 + lmm_stripe_size: 4194304 + lmm_pattern: raid0 + lmm_layout_gen: 0 + lmm_stripe_offset: -1 +PFL_LAYOUT_NEW + +# Layout changed in 2.16.0 + cat << PFL_LAYOUT_OLD > $old + lcm_layout_gen: 3 + lcm_mirror_count: 1 + lcm_entry_count: 3 + component0: + lcme_id: 1 + lcme_mirror_id: 0 + lcme_flags: init + lcme_extent.e_start: 0 + lcme_extent.e_end: 1073741824 + sub_layout: + lmm_stripe_count: 1 + lmm_stripe_size: 4194304 + lmm_pattern: raid0 + lmm_layout_gen: 0 + lmm_stripe_offset: 0 + lmm_objects: + - l_ost_idx: 0 + l_fid: 0x280000400:0x2:0x0 + component1: + lcme_id: 2 + lcme_mirror_id: 0 + lcme_flags: 0 + lcme_extent.e_start: 1073741824 + lcme_extent.e_end: 4294967296 + sub_layout: + lmm_stripe_count: 4 + lmm_stripe_size: 4194304 + lmm_pattern: raid0 + lmm_layout_gen: 0 + lmm_stripe_offset: -1 + component2: + lcme_id: 3 + lcme_mirror_id: 0 + lcme_flags: 0 + lcme_extent.e_start: 4294967296 + lcme_extent.e_end: EOF + sub_layout: + lmm_stripe_count: 4 + lmm_stripe_size: 4194304 + lmm_pattern: raid0 + lmm_layout_gen: 0 + lmm_stripe_offset: -1 +PFL_LAYOUT_OLD + + local yaml_file=$DIR/$tdir/layout.yaml + local dir=$DIR/$tdir + local test=$dir/$tfile.test + + $LFS setstripe -E 4M -c 2 -E EOF -c 4 $test || + error "failed to setstripe" + $LFS getstripe --yaml $test > $yaml_file || + error "failed to getstripe --yaml" + + $LFS setstripe --yaml $old $dir/$tfile.new || + error "failed to setstripe from new YAML format" + # test that the old format can still be used + $LFS setstripe --yaml $old $dir/$tfile.old || + error "failed to setstripe from old YAML format" + + local orig=$(get_layout_param $test) + local rest=$(cat $yaml_file | parse_layout_param) + [[ "$orig" == "$rest" ]] || + error "failed to parse current YAML layout" + + orig=$(get_layout_param $dir/$tfile.old) + rest=$(cat $old | parse_layout_param) + [[ "$orig" == "$rest" ]] || + error "failed to parse old YAML layout" + + orig=$(get_layout_param $dir/$tfile.new) + rest=$(cat $new | parse_layout_param) + [[ "$orig" == "$rest" ]] || + error "failed to parse new YAML layout" + +} +run_test 56ed "verify new YAML format is valid and back-compatible" + test_56eda() { local dir=$DIR/$tdir local subdir=$dir/subdir @@ -21588,134 +21717,36 @@ test_205i() { } run_test 205i "check job_xattr parameter accepts and rejects values correctly" -# LU-15565 -test_205j() { - local new=$DIR/$tdir/$tfile-layout-new - local old=$DIR/$tdir/$tfile-layout-old - - test_mkdir $DIR/$tdir - - cat << PFL_LAYOUT_NEW > $new -lcm_layout_gen: 3 -lcm_mirror_count: 1 -lcm_entry_count: 3 -components: - - lcme_id: 1 - lcme_mirror_id: 0 - lcme_flags: init - lcme_extent.e_start: 0 - lcme_extent.e_end: 1073741824 - sub_layout: - lmm_stripe_count: 1 - lmm_stripe_size: 4194304 - lmm_pattern: raid0 - lmm_layout_gen: 0 - lmm_stripe_offset: 0 - lmm_objects: - - l_ost_idx: 0 - l_fid: 0x280000400:0x2:0x0 - - lcme_id: 2 - lcme_mirror_id: 0 - lcme_flags: 0 - lcme_extent.e_start: 1073741824 - lcme_extent.e_end: 4294967296 - sub_layout: - lmm_stripe_count: 4 - lmm_stripe_size: 4194304 - lmm_pattern: raid0 - lmm_layout_gen: 0 - lmm_stripe_offset: -1 - - lcme_id: 3 - lcme_mirror_id: 0 - lcme_flags: 0 - lcme_extent.e_start: 4294967296 - lcme_extent.e_end: EOF - sub_layout: - lmm_stripe_count: 4 - lmm_stripe_size: 4194304 - lmm_pattern: raid0 - lmm_layout_gen: 0 - lmm_stripe_offset: -1 -PFL_LAYOUT_NEW - -# Layout changed in 2.16.0 - cat << PFL_LAYOUT_OLD > $old - lcm_layout_gen: 3 - lcm_mirror_count: 1 - lcm_entry_count: 3 - component0: - lcme_id: 1 - lcme_mirror_id: 0 - lcme_flags: init - lcme_extent.e_start: 0 - lcme_extent.e_end: 1073741824 - sub_layout: - lmm_stripe_count: 1 - lmm_stripe_size: 4194304 - lmm_pattern: raid0 - lmm_layout_gen: 0 - lmm_stripe_offset: 0 - lmm_objects: - - l_ost_idx: 0 - l_fid: 0x280000400:0x2:0x0 - component1: - lcme_id: 2 - lcme_mirror_id: 0 - lcme_flags: 0 - lcme_extent.e_start: 1073741824 - lcme_extent.e_end: 4294967296 - sub_layout: - lmm_stripe_count: 4 - lmm_stripe_size: 4194304 - lmm_pattern: raid0 - lmm_layout_gen: 0 - lmm_stripe_offset: -1 - component2: - lcme_id: 3 - lcme_mirror_id: 0 - lcme_flags: 0 - lcme_extent.e_start: 4294967296 - lcme_extent.e_end: EOF - sub_layout: - lmm_stripe_count: 4 - lmm_stripe_size: 4194304 - lmm_pattern: raid0 - lmm_layout_gen: 0 - lmm_stripe_offset: -1 -PFL_LAYOUT_OLD - - local yaml_file=$DIR/$tdir/layout.yaml - local dir=$DIR/$tdir - local test=$dir/$tfile.test - - $LFS setstripe -E 4M -c 2 -E EOF -c 4 $test || - error "failed to setstripe" - $LFS getstripe --yaml $test > $yaml_file || - error "failed to getstripe --yaml" - - $LFS setstripe --yaml $old $dir/$tfile.new || - error "failed to setstripe from new YAML format" - # test that the old format can still be used - $LFS setstripe --yaml $old $dir/$tfile.old || - error "failed to setstripe from old YAML format" - - local orig=$(get_layout_param $test) - local rest=$(cat $yaml_file | parse_layout_param) - [[ "$orig" == "$rest" ]] || - error "failed to parse current YAML layout" - - orig=$(get_layout_param $dir/$tfile.old) - rest=$(cat $old | parse_layout_param) - [[ "$orig" == "$rest" ]] || - error "failed to parse old YAML layout" - - orig=$(get_layout_param $dir/$tfile.new) - rest=$(cat $new | parse_layout_param) - [[ "$orig" == "$rest" ]] || - error "failed to parse new YAML layout" - -} -run_test 205j "verify new YAML format is valid and back-compatible" + # LU-17512 + # test '?' operator +test_205k(){ + # copied from test_205a + [[ $PARALLEL == "yes" ]] && skip "skip parallel run" + remote_mds_nodsh && skip "remote MDS with nodsh" + [[ "$($LCTL get_param -n mdc.*.connect_flags)" =~ jobstats ]] || + skip "Server doesn't support jobstats" + [[ "$JOBID_VAR" == "disable" ]] && skip_env "jobstats is disabled" + + local jobid_save=$($LCTL get_param jobid_var jobid_name) + stack_trap "$LCTL set_param $jobid_save" + + do_facet mds1 $LCTL set_param mdt.*.job_stats=clear + $LCTL set_param jobid_var=FAKE jobid_name=%j?%H:%e:%u + cp -a /etc/hosts $DIR; + job_id_name=($(do_facet mds1 $LCTL get_param mdt.*.job_stats | + awk '/job_id:/ { print $3 }' | tr -d '"')) + [[ "$job_id_name" == "$(hostname -s):cp:0" ]] || + error "expected $(hostname -s):cp:0, got $job_id_name instead" + + do_facet mds1 $LCTL set_param mdt.*.job_stats=clear + $LCTL set_param jobid_var=USER jobid_name=%j?%H:%e:%u + cp -a /etc/hosts $DIR; + job_id_name=($(do_facet mds1 $LCTL get_param mdt.*.job_stats | + awk '/job_id:/ { print $3 }' | tr -d '"')) + [[ "$job_id_name" == "$USER:cp:0" ]] || + error "expected $USER:cp:0 but got $job_id_name instead" +} +run_test 205k "Verify '?' operator on job stats" # LU-1480, LU-1773 and LU-1657 test_206() { -- 1.8.3.1