From 3d7895bf08980530e1e5947a00fc9500f35a55de Mon Sep 17 00:00:00 2001 From: Patrick Farrell Date: Tue, 11 Dec 2018 13:34:27 -0600 Subject: [PATCH] LU-10070 lod: SEL: Add FLR support Add FLR support for self-extending layouts. The basic model is that when a layout intent would modify an FLR replica, we first run the extent_update code to perform any layout extent changes for self extending layouts. This treats the FLR operations (stale, resync, etc) similarly to i/o, creating initialized layout where those operations need it. This makes the interaction between SEL and FLR fairly simple. Add FLR tests for self-extending layouts Cray-bug-id: LUS-2528 Signed-off-by: Patrick Farrell Change-Id: Ia23df8e226955f64e9b19df993b66d2d4f820f33 Reviewed-on: https://review.whamcloud.com/33785 Reviewed-by: Patrick Farrell Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexey Lyashkov Reviewed-by: Oleg Drokin --- lustre/lod/lod_object.c | 59 ++++++++++++--- lustre/tests/sanity-flr.sh | 180 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 9 deletions(-) diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index e69ecf8..1b023b6 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -6786,15 +6786,21 @@ static inline int lod_comp_index(struct lod_object *lo, /** * Stale other mirrors by writing extent. */ -static void lod_stale_components(struct lod_object *lo, int primary, - struct lu_extent *extent) +static int lod_stale_components(const struct lu_env *env, struct lod_object *lo, + int primary, struct lu_extent *extent, + struct thandle *th) { struct lod_layout_component *pri_comp, *lod_comp; + struct lu_extent pri_extent; + int rc = 0; int i; + ENTRY; /* The writing extent decides which components in the primary * are affected... */ CDEBUG(D_LAYOUT, "primary mirror %d, "DEXT"\n", primary, PEXT(extent)); + +restart: lod_foreach_mirror_comp(pri_comp, lo, primary) { if (!lu_extent_is_overlapped(extent, &pri_comp->llc_extent)) continue; @@ -6803,15 +6809,27 @@ static void lod_stale_components(struct lod_object *lo, int primary, lod_comp_index(lo, pri_comp), PEXT(&pri_comp->llc_extent)); + pri_extent.e_start = pri_comp->llc_extent.e_start; + pri_extent.e_end = pri_comp->llc_extent.e_end; + for (i = 0; i < lo->ldo_mirror_count; i++) { if (i == primary) continue; + rc = lod_declare_update_extents(env, lo, &pri_extent, + th, i, 0); + /* if update_extents changed the layout, it may have + * reallocated the component array, so start over to + * avoid using stale pointers */ + if (rc == 1) + goto restart; + if (rc < 0) + RETURN(rc); /* ... and then stale other components that are * overlapping with primary components */ lod_foreach_mirror_comp(lod_comp, lo, i) { if (!lu_extent_is_overlapped( - &pri_comp->llc_extent, + &pri_extent, &lod_comp->llc_extent)) continue; @@ -6823,6 +6841,8 @@ static void lod_stale_components(struct lod_object *lo, int primary, } } } + + RETURN(rc); } /** @@ -7058,6 +7078,7 @@ static int lod_declare_update_rdonly(const struct lu_env *env, if (mlc->mlc_opc == MD_LAYOUT_WRITE) { struct layout_intent *layout = mlc->mlc_intent; + int write = layout->li_opc == LAYOUT_INTENT_WRITE; int picked; extent = layout->li_extent; @@ -7072,6 +7093,12 @@ static int lod_declare_update_rdonly(const struct lu_env *env, PFID(lod_object_fid(lo)), lo->ldo_mirrors[picked].lme_id); + /* Update extents of primary before staling */ + rc = lod_declare_update_extents(env, lo, &extent, th, picked, + write); + if (rc < 0) + GOTO(out, rc); + if (layout->li_opc == LAYOUT_INTENT_TRUNC) { /** * trunc transfers [0, size) in the intent extent, we'd @@ -7082,7 +7109,9 @@ static int lod_declare_update_rdonly(const struct lu_env *env, } /* stale overlapping components from other mirrors */ - lod_stale_components(lo, picked, &extent); + rc = lod_stale_components(env, lo, picked, &extent, th); + if (rc < 0) + GOTO(out, rc); /* restore truncate intent extent */ if (layout->li_opc == LAYOUT_INTENT_TRUNC) @@ -7222,6 +7251,9 @@ static int lod_declare_update_write_pending(const struct lu_env *env, * 2. transfer layout version to all objects to close write era. */ if (mlc->mlc_opc == MD_LAYOUT_WRITE) { + struct layout_intent *layout = mlc->mlc_intent; + int write = layout->li_opc == LAYOUT_INTENT_WRITE; + LASSERT(mlc->mlc_intent != NULL); extent = mlc->mlc_intent->li_extent; @@ -7229,6 +7261,12 @@ static int lod_declare_update_write_pending(const struct lu_env *env, CDEBUG(D_LAYOUT, DFID": intent to write: "DEXT"\n", PFID(lod_object_fid(lo)), PEXT(&extent)); + /* 1. Update extents of primary before staling */ + rc = lod_declare_update_extents(env, lo, &extent, th, primary, + write); + if (rc < 0) + GOTO(out, rc); + if (mlc->mlc_intent->li_opc == LAYOUT_INTENT_TRUNC) { /** * trunc transfers [0, size) in the intent extent, we'd @@ -7237,10 +7275,13 @@ static int lod_declare_update_write_pending(const struct lu_env *env, extent.e_start = extent.e_end; extent.e_end = OBD_OBJECT_EOF; } - /* 1. stale overlapping components */ - lod_stale_components(lo, primary, &extent); - /* 2. find out the components need instantiating. + /* 2. stale overlapping components */ + rc = lod_stale_components(env, lo, primary, &extent, th); + if (rc < 0) + GOTO(out, rc); + + /* 3. find the components which need instantiating. * instantiate [0, mlc->mlc_intent->e_end) */ /* restore truncate intent extent */ @@ -7380,9 +7421,9 @@ static int lod_declare_update_sync_pending(const struct lu_env *env, GOTO(out, rc = -EINVAL); } - CDEBUG(D_LAYOUT, DFID": resynced %u/%zu components\n", + CDEBUG(D_LAYOUT, DFID": synced %u resynced %u/%zu components\n", PFID(lod_object_fid(lo)), - resync_components, mlc->mlc_resync_count); + sync_components, resync_components, mlc->mlc_resync_count); lo->ldo_flr_state = LCM_FL_RDONLY; lod_obj_inc_layout_gen(lo); diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 9a1b82d..443778b 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -2286,6 +2286,186 @@ test_203() { } run_test 203 "mirror file preserve mirror ID" +# Simple test of FLR + self-extending layout, SEL in non-primary mirror +test_204a() { + [ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] && + skip "skipped for lustre < $SEL_VER" + + local comp_file=$DIR/$tdir/$tfile + local flg_opts="" + local found="" + + test_mkdir $DIR/$tdir + + # first mirror is 0-10M, then 10M-(-1), second mirror is 1M followed + # by extension space to -1 + $LFS setstripe -N -E 10M -E-1 -N -E 1M -E-1 -z64M $comp_file || + error "Create $comp_file failed" + + # Write to first component, extending & staling second mirror + dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc || + error "dd to extend + stale failed" + + $LFS getstripe $comp_file + + flg_opts="--component-flags init,stale" + found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "write: Second comp end incorrect" + + flg_opts="--component-flags extension" + found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "write: Third comp start incorrect" + + # mirror resync should not change the extents + $LFS mirror resync $comp_file + + flg_opts="--component-flags init" + found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: Second comp end incorrect" + + flg_opts="--component-flags extension" + found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: Third comp start incorrect" + + sel_layout_sanity $comp_file 5 + + rm -f $comp_file +} +run_test 204a "FLR write/stale/resync tests with self-extending mirror" + +# Simple test of FLR + self-extending layout, SEL in primary mirror +test_204b() { + [ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] && + skip "skipped for lustre < $SEL_VER" + + local comp_file=$DIR/$tdir/$tfile + local flg_opts="" + local found="" + + test_mkdir $DIR/$tdir + + # first mirror is 1M followed by extension space to -1, second mirror + # is 0-10M, then 10M-(-1), + $LFS setstripe -N -E 1M -E-1 -z64M -N -E 10M -E-1 $comp_file || + error "Create $comp_file failed" + + # Write to first component, extending first component & staling + # other mirror + dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc || + error "dd to extend + stale failed" + + $LFS getstripe $comp_file + + flg_opts="--component-flags init" + found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "write: First comp end incorrect" + + flg_opts="--component-flags extension" + found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "write: Second comp start incorrect" + + flg_opts="--component-flags init,stale" + found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "write: First mirror comp flags incorrect" + + # This component is staled because it overlaps the extended first + # component of the primary mirror, even though it doesn't overlap + # the actual write - thus not inited. + flg_opts="--component-flags stale" + found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "write: Second mirror comp flags incorrect" + + # mirror resync should not change the extents + $LFS mirror resync $comp_file + + $LFS getstripe $comp_file + + flg_opts="--component-flags init" + found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: First comp end incorrect" + + flg_opts="--component-flags extension" + found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: Second comp start incorrect" + + flg_opts="--component-flags init" + found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: First mirror comp flags incorrect" + + flg_opts="--component-flags init" + found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: Second mirror comp flags incorrect" + + sel_layout_sanity $comp_file 5 + + rm -f $comp_file +} +run_test 204b "FLR write/stale/resync tests with self-extending primary" + +# FLR + SEL failed extension & component removal +# extension space in second mirror +test_204c() { + [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return + [ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] && + skip "skipped for lustre < $SEL_VER" + + local comp_file=$DIR/$tdir/$tfile + local found="" + local ost_idx1=0 + local ost_name=$(ostname_from_index $ost_idx1) + + test_mkdir $DIR/$tdir + + # first mirror is is 0-10M, then 10M-(-1), second mirror is 0-1M, then + # extension space from 1M to 1G, then normal space to -1 + $LFS setstripe -N -E 10M -E-1 -N -E 1M -E 1G -i $ost_idx1 -z 64M \ + -E -1 $comp_file || error "Create $comp_file failed" + + do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=1 + sleep_maxage + + # write to first comp (0 - 10M) of mirror 1, extending + staling + # first + second comp of mirror 2 + dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc + RC=$? + + do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=0 + sleep_maxage + + [ $RC -eq 0 ] || error "dd to extend + stale failed" + + $LFS getstripe $comp_file + + found=$($LFS find --component-start 0m --component-end 1m \ + --comp-flags init,stale $comp_file | wc -l) + [ $found -eq 1 ] || error "write: First mirror comp incorrect" + + found=$($LFS find --component-start 1m --component-end EOF \ + --comp-flags stale,^init $comp_file | wc -l) + [ $found -eq 1 ] || error "write: Second mirror comp incorrect" + + local mirror_id=$($LFS getstripe --component-start=1m \ + --component-end=EOF $comp_file | \ + grep lcme_mirror_id | awk '{ print $2 }') + + [[ $mirror_id -eq 2 ]] || + error "component not in correct mirror? $mirror_id" + + $LFS mirror resync $comp_file + + $LFS getstripe $comp_file + + # component dimensions should not change from resync + found=$($LFS find --component-start 1m --component-end EOF \ + --component-flags init $comp_file | wc -l) + [ $found -eq 1 ] || error "resync: Second mirror comp incorrect" + + sel_layout_sanity $comp_file 4 + + rm -f $comp_file +} +run_test 204c "FLR write/stale/resync test with component removal" + complete $SECONDS check_and_cleanup_lustre exit_status -- 1.8.3.1