From: Alexander Boyko Date: Mon, 17 May 2021 13:29:01 +0000 (-0400) Subject: LU-14688 mdt: changelog purge deletes plain llog X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=2c498e83df8a231e7222a69e9f3dac981aa5dfc5;p=fs%2Flustre-release.git LU-14688 mdt: changelog purge deletes plain llog With a massive cancel records changelog could delete a plain llog file and skip one by one record cancelling. Also patch fixes the race between llog_destroy and llog_next_block. Lustre-change: https://review.whamcloud.com/43719 Lustre-commit: d813c75df6798efbf3228347628c0d671ca7269c HPE-bug-id: LUS-9950 Signed-off-by: Alexander Boyko Change-Id: I47c2ed97945e979745255381f83b6a417d7ba8b1 Reviewed-by: Andreas Dilger Reviewed-by: Mike Pershin Reviewed-on: https://review.whamcloud.com/44262 Tested-by: jenkins Tested-by: Maloo --- diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 2ee1a1e..e911616 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -245,6 +245,7 @@ extern char obd_jobid_var[]; #define OBD_FAIL_MDS_STATFS_SPOOF 0x168 #define OBD_FAIL_MDS_REINT_OPEN 0x169 #define OBD_FAIL_MDS_REINT_OPEN2 0x16a +#define OBD_FAIL_MDS_CHANGELOG_DEL 0x16c /* layout lock */ #define OBD_FAIL_MDS_NO_LL_GETATTR 0x170 diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c index fa3793b..3bf500f 100644 --- a/lustre/mdd/mdd_device.c +++ b/lustre/mdd/mdd_device.c @@ -345,6 +345,28 @@ static int llog_changelog_cancel_cb(const struct lu_env *env, OBD_RACE(OBD_FAIL_MDS_CHANGELOG_RACE); } + /* Records folow one by one, cr_index++. We could calculate the + * last cr_index at this plain llog. And if it less then cookie endrec + * cancel the whole file. + */ + if ((LLOG_HDR_BITMAP_SIZE(llh->lgh_hdr) - hdr->lrh_index + + rec->cr.cr_index) < cl_cookie->endrec) { + int rc; + + if (unlikely(OBD_FAIL_PRECHECK(OBD_FAIL_MDS_CHANGELOG_DEL))) { + if (cfs_fail_val == 0) { + cfs_fail_val = (unsigned long)llh & 0xFFFFFFFF; + OBD_RACE(OBD_FAIL_MDS_CHANGELOG_DEL); + } + } + rc = llog_destroy(env, llh); + if (!rc) { + CDEBUG(D_HA, "Changelog destroyed plain "DFID"\n", + PFID(&llh->lgh_id.lgl_oi.oi_fid)); + RETURN(LLOG_DEL_PLAIN); + } + } + /* cancel them one at a time. I suppose we could store up the cookies * and cancel them all at once; probably more efficient, but this is * done as a user call, so who cares... */ diff --git a/lustre/obdclass/llog.c b/lustre/obdclass/llog.c index 4eb7ce9..84ef000 100644 --- a/lustre/obdclass/llog.c +++ b/lustre/obdclass/llog.c @@ -535,6 +535,8 @@ repeat: CDEBUG(D_OTHER, "cur_offset %llu, chunk_offset %llu," " buf_offset %u, rc = %d\n", cur_offset, (__u64)chunk_offset, buf_offset, rc); + if (rc == -ESTALE) + GOTO(out, rc = 0); /* we`ve tried to reread the chunk, but there is no * new records */ if (rc == -EIO && repeated && (chunk_offset + buf_offset) == diff --git a/lustre/obdclass/llog_osd.c b/lustre/obdclass/llog_osd.c index 51829c8..a12f43e 100644 --- a/lustre/obdclass/llog_osd.c +++ b/lustre/obdclass/llog_osd.c @@ -901,9 +901,18 @@ static int llog_osd_next_block(const struct lu_env *env, LASSERT(loghandle); LASSERT(loghandle->lgh_ctxt); + if (OBD_FAIL_PRECHECK(OBD_FAIL_MDS_CHANGELOG_DEL) && + cfs_fail_val == ((unsigned long)loghandle & 0xFFFFFFFF)) { + OBD_RACE(OBD_FAIL_MDS_CHANGELOG_DEL); + msleep(MSEC_PER_SEC >> 2); + } + o = loghandle->lgh_obj; LASSERT(o); - LASSERT(llog_osd_exist(loghandle)); + dt_read_lock(env, o, 0); + if (!llog_osd_exist(loghandle)) + GOTO(out, rc = -ESTALE); //object was destroyed + dt = lu2dt_dev(o->do_lu.lo_dev); LASSERT(dt); @@ -1032,6 +1041,7 @@ retry: } GOTO(out, rc = -EIO); out: + dt_read_unlock(env, o); return rc; } @@ -1076,7 +1086,10 @@ static int llog_osd_prev_block(const struct lu_env *env, o = loghandle->lgh_obj; LASSERT(o); - LASSERT(llog_osd_exist(loghandle)); + dt_read_lock(env, o, 0); + if (!llog_osd_exist(loghandle)) + GOTO(out, rc = -ESTALE); + dt = lu2dt_dev(o->do_lu.lo_dev); LASSERT(dt); @@ -1159,6 +1172,7 @@ static int llog_osd_prev_block(const struct lu_env *env, } GOTO(out, rc = -EIO); out: + dt_read_unlock(env, o); return rc; } diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 94aaec8..4c3b9c4 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -15375,8 +15375,8 @@ run_test 160l "Verify that MTIME changelog records contain the parent FID" test_160m() { remote_mds_nodsh && skip "remote MDS with nodsh" && return - [[ $MDS1_VERSION -ge $(version_code 2.14.51) ]] || - skip "Need MDS version at least 2.14.51" + [[ $MDS1_VERSION -ge $(version_code 2.14.0.4) ]] || + skip "Need MDS version at least 2.14.0.4" local cl_users local cl_user1 local cl_user2 @@ -15414,6 +15414,56 @@ test_160m() { } run_test 160m "Changelog clear race" +test_160n() { + remote_mds_nodsh && skip "remote MDS with nodsh" && return + [[ $MDS1_VERSION -ge $(version_code 2.14.0.4) ]] || + skip "Need MDS version at least 2.14.0.4" + local cl_users + local cl_user1 + local cl_user2 + local pid1 + local first_rec + local last_rec=0 + + # Create a user + changelog_register || error "first changelog_register failed" + + cl_users=(${CL_USERS[mds1]}) + cl_user1="${cl_users[0]}" + + # generate some changelog records to accumulate on MDT0 + test_mkdir -i0 -c1 $DIR/$tdir || error "test_mkdir $tdir failed" + first_rec=$(changelog_users $SINGLEMDS | + awk '/^current.index:/ { print $NF }') + while (( last_rec < (( first_rec + 65000)) )); do + createmany -m $DIR/$tdir/$tfile 10000 || + error "create $DIR/$tdir/$tfile failed" + + for i in $(seq 0 10000); do + mrename $DIR/$tdir/$tfile$i $DIR/$tdir/$tfile-new$i \ + > /dev/null + done + + unlinkmany $DIR/$tdir/$tfile-new 10000 || + error "unlinkmany failed unlink" + last_rec=$(changelog_users $SINGLEMDS | + awk '/^current.index:/ { print $NF }') + echo last record $last_rec + (( last_rec == 0 )) && error "no changelog found" + done + +#define OBD_FAIL_MDS_CHANGELOG_DEL 0x16c + do_facet mds1 $LCTL set_param fail_loc=0x8000016c fail_val=0 + + __changelog_clear mds1 $cl_user1 0 & + pid1=$! + sleep 2 + __changelog_clear mds1 $cl_user1 0 || + error "fail to cancel record for $cl_user1" + wait $pid1 + [[ $? -eq 0 ]] || error "fail to cancel record for $cl_user2" +} +run_test 160n "Changelog destroy race" test_161a() { [ $PARALLEL == "yes" ] && skip "skip parallel run"