LU-14177 pcc: clear PCC-RO cache from old client access

For the purpose of the compatibility and interoperability, we have
added a PCC-RO connection flags.

To avoid inconsistent data access, MDT does not (try to) grant
layout lock to the client at the time of getattr() and open().
When an old client without PCC-RO support requests a layout lock
via a intent lock request on the file in LCM_FL_PCC_RDONLY state,
MDT needs to clear the LCM_FL_PCC_RDONLY flag on the layout first
which will invalidate all PCC-RO caches on the clients, and then
return the layout to the old client.

Lustre-change: https://review.whamcloud.com/40850
Lustre-commit: TBD (from 0c76ae7f3cb6fc3a9f70d1398f773d8afffa50f1)

Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: I69707d1ac53decaddd32bcf231b15d3565fb200f
Reviewed-on: https://review.whamcloud.com/45269
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Lai Siyao <lai.siyao@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h
index baad226..3348b78 100644 (file)
--- a/lustre/include/lustre_export.h
+++ b/lustre/include/lustre_export.h
@@ -463,6 +463,11 @@ static inline int exp_connect_lock_convert(struct obd_export *exp)
        return !!(exp_connect_flags2(exp) & OBD_CONNECT2_LOCK_CONVERT);
 }
 
+static inline int exp_connect_pccro(struct obd_export *exp)
+{
+       return !!(exp_connect_flags2(exp) & OBD_CONNECT2_PCCRO);
+}
+
 extern struct obd_export *class_conn2export(struct lustre_handle *conn);
 
 static inline int exp_connect_archive_id_array(struct obd_export *exp)

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index b4bcb43..2d5d555 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -2130,8 +2130,17 @@ static int mdt_getattr_name_lock(struct mdt_thread_info *info,
                LASSERT(!(child_bits & MDS_INODELOCK_LAYOUT));
                if (S_ISREG(lu_object_attr(&child->mot_obj)) &&
                    !mdt_object_remote(child) && ldlm_rep != NULL) {
+                       /*
+                        * For the old client without PCC-RO support, it needs
+                        * to clear the LCM_FL_PCC_RDONLY layout flag to avoid
+                        * the inconsistent data access for the purpose of
+                        * compatibility and interoperability.
+                        * Thus it does not grant layout lock to the client
+                        * here.
+                        */
                        if (!OBD_FAIL_CHECK(OBD_FAIL_MDS_NO_LL_GETATTR) &&
-                           exp_connect_layout(info->mti_exp)) {
+                           exp_connect_layout(info->mti_exp) &&
+                           exp_connect_pccro(info->mti_exp)) {
                                /* try to grant layout lock for regular file. */
                                try_bits = MDS_INODELOCK_LAYOUT;
                        }
@@ -2154,6 +2163,7 @@ static int mdt_getattr_name_lock(struct mdt_thread_info *info,
                                child_bits &= ~MDS_INODELOCK_UPDATE;
                        rc = mdt_object_lock(info, child, lhc, child_bits);
                }
+
                if (unlikely(rc != 0))
                        GOTO(out_child, rc);
        }
@@ -4384,6 +4394,37 @@ static int mdt_layout_change_pccro(struct mdt_thread_info *info,
 
        mdt_object_unlock(info, obj, lhc, 1);
        rc = mdt_layout_change(info, obj, lhc, layout);
+
+       RETURN(rc);
+}
+
+static int mdt_layout_clear_pccro(struct mdt_thread_info *info,
+                                 struct mdt_object *obj)
+{
+       struct mdt_lock_handle *lhc = &info->mti_lh[MDT_LH_RMT];
+       struct md_layout_change layout = { .mlc_opc = MD_LAYOUT_WRITE };
+       struct layout_intent intent = {
+               .li_opc = LAYOUT_INTENT_PCCRO_CLEAR,
+               .li_extent.e_start = 0,
+               .li_extent.e_end = LUSTRE_EOF,
+       };
+       int rc;
+
+       ENTRY;
+
+       layout.mlc_intent = &intent;
+       mdt_lock_handle_init(lhc);
+       mdt_lock_reg_init(lhc, LCK_EX);
+       rc = mdt_reint_object_lock(info, obj, lhc,
+                                  MDS_INODELOCK_LAYOUT, false);
+       if (rc)
+               RETURN(rc);
+
+       mutex_lock(&obj->mot_som_mutex);
+       rc = mo_layout_change(info->mti_env, mdt_object_child(obj), &layout);
+       mutex_unlock(&obj->mot_som_mutex);
+       mdt_object_unlock(info, obj, lhc, rc);
+
        RETURN(rc);
 }
 
@@ -4456,6 +4497,15 @@ static int mdt_intent_layout(enum ldlm_intent_flags it_opc,
 
                        if (layout_size > info->mti_mdt->mdt_max_mdsize)
                                info->mti_mdt->mdt_max_mdsize = layout_size;
+
+                       /*
+                        * For the old client without PCC-RO support, it needs
+                        * to clear LCM_FL_PCC_RDONLY flag first to avoid
+                        * inconsistent data access.
+                        */
+                       if (!exp_connect_pccro(info->mti_exp))
+                               rc = mdt_layout_clear_pccro(info, obj);
+
                }
                CDEBUG(D_INFO, "%s: layout_size %d\n",
                       mdt_obd_name(info->mti_mdt), layout_size);
@@ -4479,7 +4529,7 @@ static int mdt_intent_layout(enum ldlm_intent_flags it_opc,
 
        /* take lock in ldlm_lock_enqueue() for LAYOUT_INTENT_ACCESS */
        if (layout.mlc_opc == MD_LAYOUT_NOP)
-               GOTO(out, rc = 0);
+               GOTO(out, rc);
 
        rc = mdt_check_resent(info, mdt_reconstruct_generic, lhc);
        if (rc < 0)

diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c
index d5953ce..e0667f8 100644 (file)
--- a/lustre/mdt/mdt_open.c
+++ b/lustre/mdt/mdt_open.c
@@ -922,6 +922,14 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
                trybits |= MDS_INODELOCK_LAYOUT;
        }
 
+       /*
+        * Don't grant layout lock to the client which does not support PCC-RO
+        * to avoid inconsistent data access for the purpose of compatibility
+        * and interoperability.
+        */
+       if (!exp_connect_pccro(info->mti_exp))
+               trybits &= ~MDS_INODELOCK_LAYOUT;
+
        if (*ibits | trybits)
                rc = mdt_object_lock_try(info, obj, lhc, ibits, trybits, false);