From c2b63995eee53088165d81ecd9245a230112b4e8 Mon Sep 17 00:00:00 2001 From: Vladimir Saveliev Date: Mon, 1 Mar 2021 11:52:51 +0300 Subject: [PATCH] LU-12752 mdt: commitrw_write() - check dying object under lock If process writes to unlinked file the following race between mdt_commitrw_write() and mdd_close() may occur because mdt_commitrw_write() checks whether an object is dying without lock: mdt_commitrw_write() checks lu_object_is_dying(&mo->mot_header) and it not yet mdd_close() interposes and destroys the object via mdo_destroy() lod_destroy() lod_sub_destroy() osd_destroy() obj->oo_destroyed = 1; mdt_commitrw_write() continues, locks the object and returns ENOENT from dt_attr_get() osd_attr_get() if (unlikely(obj->oo_destroyed)) return -ENOENT; If the file is built of DoM and raid component ll_delete_inode() calls cl_sync_file_range() which is to iterate over both mdt and raid components via mdc_io_fsync_start() and osc_io_fsync_start(). As mdc_io_fsync_start() fails with -ENOENT due to failed write rpc, osc_io_fsync_start() does not get called. Then truncate_inode_pages_final() finds not-discarded pages and fails with: (osc_page.c:183:osc_page_delete()) Trying to teardown failed: -16 (osc_page.c:184:osc_page_delete()) ASSERTION( 0 ) failed: (osc_page.c:184:osc_page_delete()) LBUG Test to illustrate the issue is added. The fix is to call lu_object_is_dying() under object lock. Lustre-change: https://review.whamcloud.com/41797 Lustre-commit: d48a0ebb5a8d5d49684325434b503e8aab085397 Change-Id: I463c8a6f85d4f5fd934b167c6194f50ae9d4b7d4 Signed-off-by: Vladimir Saveliev Reviewed-by: Andreas Dilger Reviewed-by: Mike Pershin Reviewed-by: Oleg Drokin Reviewed-on: https://review.whamcloud.com/46612 Tested-by: jenkins Tested-by: Maloo --- lustre/include/obd_support.h | 1 + lustre/mdt/mdt_io.c | 14 ++++++++------ lustre/tests/sanity.sh | 11 +++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 07725e0..1378b09 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -245,6 +245,7 @@ extern char obd_jobid_var[]; #define OBD_FAIL_MDS_STATFS_SPOOF 0x168 #define OBD_FAIL_MDS_REINT_OPEN 0x169 #define OBD_FAIL_MDS_REINT_OPEN2 0x16a +#define OBD_FAIL_MDS_COMMITRW_DELAY 0x16b #define OBD_FAIL_MDS_CHANGELOG_DEL 0x16c #define OBD_FAIL_MDS_DIR_PAGE_WALK 0x16f diff --git a/lustre/mdt/mdt_io.c b/lustre/mdt/mdt_io.c index 0e1dbb4..9a92114 100644 --- a/lustre/mdt/mdt_io.c +++ b/lustre/mdt/mdt_io.c @@ -601,12 +601,6 @@ static int mdt_commitrw_write(const struct lu_env *env, struct obd_export *exp, retry: if (!dt_object_exists(dob)) GOTO(out, rc = -ENOENT); - if (lu_object_is_dying(&mo->mot_header)) { - /* Commit to stale object can be just skipped silently. */ - CDEBUG(D_INODE, "skip commit to stale object "DFID"\n", - PFID(mdt_object_fid(mo))); - GOTO(out, rc = 0); - } if (niocount == 0) { rc = -EPROTO; @@ -616,6 +610,8 @@ retry: GOTO(out, rc); } + CFS_FAIL_TIMEOUT(OBD_FAIL_MDS_COMMITRW_DELAY, cfs_fail_val); + th = dt_trans_create(env, dt); if (IS_ERR(th)) GOTO(out, rc = PTR_ERR(th)); @@ -647,6 +643,12 @@ retry: GOTO(out_stop, rc); dt_write_lock(env, dob, 0); + if (lu_object_is_dying(&mo->mot_header)) { + /* Commit to stale object can be just skipped silently. */ + CDEBUG(D_INODE, "skip commit to stale object "DFID"\n", + PFID(mdt_object_fid(mo))); + GOTO(unlock, rc = 0); + } rc = dt_write_commit(env, dob, lnb, niocount, th, oa->o_size); if (rc) { restart = th->th_restart_tran; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 25d0026..2b67996 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -22144,6 +22144,17 @@ test_273a() { } run_test 273a "DoM: layout swapping should fail with DOM" +test_273b() { + mkdir -p $DIR/$tdir + $LFS setstripe -E 1M -L mdt -E -1 -c -1 $DIR/$tdir + +#define OBD_FAIL_MDS_COMMITRW_DELAY 0x16b + do_facet mds1 $LCTL set_param fail_loc=0x8000016b fail_val=2 + + $MULTIOP $DIR/$tdir/$tfile Ouw2097152c +} +run_test 273b "DoM: race writeback and object destroy" + test_275() { remote_ost_nodsh && skip "remote OST with nodsh" [ $OST1_VERSION -lt $(version_code 2.10.57) ] && -- 1.8.3.1