LU-14606 llog: hide ENOENT for cancelling record

Llog allows parallel records processing. A record could be cancelled
at callback. If two threads processing and cancelling the same record,
one thread would get ENOENT.
The error was observed during purging changlog records.The patch
adds reproducer test sanity 160m.

This is a valid case, let's hide ENOENT error from a caller.

Lustre-change: https://review.whamcloud.com/43264
Lustre-commit: 0b60647c0382426e3b4105d82d04862d2e4831cb

HPE-bug-id: LUS-9826
Signed-off-by: Alexander Boyko <alexander.boyko@hpe.com>
Change-Id: Id00b959e6f329c2ad34966f8a17a52f71680f24c
Reviewed-by: Alexander Zarochentsev <alexander.zarochentsev@hpe.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/43572
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index d6d2f48..12b673a 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -245,6 +245,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_MDS_LLOG_CREATE_FAILED2 0x15b
 #define OBD_FAIL_MDS_FLD_LOOKUP                        0x15c
 #define OBD_FAIL_MDS_CHANGELOG_REORDER 0x15d
+#define OBD_FAIL_MDS_CHANGELOG_RACE     0x15f
 #define OBD_FAIL_MDS_INTENT_DELAY              0x160
 #define OBD_FAIL_MDS_XATTR_REP                 0x161
 #define OBD_FAIL_MDS_TRACK_OVERFLOW     0x162

diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c
index 53cc2fa..a105cdc 100644 (file)
--- a/lustre/mdd/mdd_device.c
+++ b/lustre/mdd/mdd_device.c
@@ -341,10 +341,8 @@ static int llog_changelog_cancel_cb(const struct lu_env *env,
                                    struct llog_rec_hdr *hdr, void *data)
 {
        struct llog_changelog_rec *rec = (struct llog_changelog_rec *)hdr;
-       struct llog_cookie       cookie;
        struct changelog_cancel_cookie *cl_cookie =
                (struct changelog_cancel_cookie *)data;
-       int                      rc;
 
        ENTRY;
 
@@ -379,15 +377,18 @@ static int llog_changelog_cancel_cb(const struct lu_env *env,
                /* records are in order, so we're done */
                RETURN(LLOG_PROC_BREAK);
 
-       cookie.lgc_lgl = llh->lgh_id;
-       cookie.lgc_index = hdr->lrh_index;
+       if (unlikely(OBD_FAIL_PRECHECK(OBD_FAIL_MDS_CHANGELOG_RACE))) {
+               if (cfs_fail_val == 0)
+                       cfs_fail_val = hdr->lrh_index;
+               if (cfs_fail_val == hdr->lrh_index)
+                       OBD_RACE(OBD_FAIL_MDS_CHANGELOG_RACE);
+       }
 
        /* cancel them one at a time.  I suppose we could store up the cookies
         * and cancel them all at once; probably more efficient, but this is
         * done as a user call, so who cares... */
-       rc = llog_cat_cancel_records(env, llh->u.phd.phd_cat_handle, 1,
-                                    &cookie);
-       RETURN(rc < 0 ? rc : 0);
+
+       RETURN(LLOG_DEL_RECORD);
 }
 
 static int llog_changelog_cancel(const struct lu_env *env,

diff --git a/lustre/obdclass/llog.c b/lustre/obdclass/llog.c
index e9228b3..b9ce2b7 100644 (file)
--- a/lustre/obdclass/llog.c
+++ b/lustre/obdclass/llog.c
@@ -312,13 +312,19 @@ int llog_cancel_rec(const struct lu_env *env, struct llog_handle *loghandle,
        }
 
 out_unlock:
+       if (rc < 0 && subtract_count) {
+               /* restore bitmap while holding a mutex */
+               loghandle->lgh_hdr->llh_count++;
+               subtract_count = false;
+               ext2_set_bit(index, LLOG_HDR_BITMAP(llh));
+       }
        mutex_unlock(&loghandle->lgh_hdr_mutex);
        up_write(&loghandle->lgh_lock);
 out_trans:
        rc1 = dt_trans_stop(env, dt, th);
        if (rc == 0)
                rc = rc1;
-       if (rc < 0 && subtract_count) {
+       if (rc1 < 0 && subtract_count) {
                mutex_lock(&loghandle->lgh_hdr_mutex);
                loghandle->lgh_hdr->llh_count++;
                ext2_set_bit(index, LLOG_HDR_BITMAP(llh));
@@ -723,6 +729,12 @@ repeat:
                                        rc = llog_cancel_rec(lpi->lpi_env,
                                                             loghandle,
                                                             rec->lrh_index);
+                                       /* Allow parallel cancelling, ENOENT
+                                        * means record was canceled at another
+                                        * processing thread or callback
+                                        */
+                                       if (rc == -ENOENT)
+                                               rc = 0;
                                }
                                if (rc)
                                        GOTO(out, rc);

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 3bf134a..589f882 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -14021,6 +14021,47 @@ test_160j() {
 }
 run_test 160j "client can be umounted  while its chanangelog is being used"
 
+test_160m() {
+       remote_mds_nodsh && skip "remote MDS with nodsh" && return
+       [[ $MDS1_VERSION -ge $(version_code 2.12.7) ]] ||
+               skip "Need MDS version at least 2.12.7"
+       local cl_users
+       local cl_user1
+       local cl_user2
+       local pid1
+
+       # Create a user
+       changelog_register || error "first changelog_register failed"
+       changelog_register || error "second changelog_register failed"
+
+       cl_users=(${CL_USERS[mds1]})
+       cl_user1="${cl_users[0]}"
+       cl_user2="${cl_users[1]}"
+       # generate some changelog records to accumulate on MDT0
+       test_mkdir -p -i0 -c1 $DIR/$tdir || error "test_mkdir $tdir failed"
+       createmany -m $DIR/$tdir/$tfile 50 ||
+               error "create $DIR/$tdir/$tfile failed"
+       unlinkmany $DIR/$tdir/$tfile 50 || error "unlinkmany failed"
+       rm -f $DIR/$tdir
+
+       # check changelogs have been generated
+       local nbcl=$(changelog_dump | wc -l)
+       [[ $nbcl -eq 0 ]] && error "no changelogs found"
+
+#define OBD_FAIL_MDS_CHANGELOG_RACE     0x15f
+       do_facet mds1 $LCTL set_param fail_loc=0x8000015f fail_val=0
+
+       __changelog_clear mds1 $cl_user1 +10
+       __changelog_clear mds1 $cl_user2 0 &
+       pid1=$!
+       sleep 2
+       __changelog_clear mds1 $cl_user1 0 ||
+               error "fail to cancel record for $cl_user1"
+       wait $pid1
+       [[ $? -eq 0 ]] || error "fail to cancel record for $cl_user2"
+}
+run_test 160m "Changelog clear race"
+
 test_161a() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"