LU-14177 pcc: clear PCC-RO cache from old client access

For the purpose of the compatibility and interoperability, we have
added a PCC-RO connection flags.

To avoid inconsistent data access, MDT does not (try to) grant
layout lock to the client at the time of getattr() and open().
When an old client without PCC-RO support requests a layout lock
via a intent lock request on the file in LCM_FL_PCC_RDONLY state,
MDT needs to clear the LCM_FL_PCC_RDONLY flag on the layout first
which will invalidate all PCC-RO caches on the clients, and then
return the layout to the old client.

Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: I69707d1ac53decaddd32bcf231b15d3565fb200f

diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h
index f7ee8ec..6f1e60a 100644 (file)
--- a/lustre/include/lustre_export.h
+++ b/lustre/include/lustre_export.h
@@ -463,6 +463,11 @@ static inline int exp_connect_lock_convert(struct obd_export *exp)
        return !!(exp_connect_flags2(exp) & OBD_CONNECT2_LOCK_CONVERT);
 }
 
+static inline int exp_connect_pccro(struct obd_export *exp)
+{
+       return !!(exp_connect_flags2(exp) & OBD_CONNECT2_PCCRO);
+}
+
 extern struct obd_export *class_conn2export(struct lustre_handle *conn);
 
 static inline int exp_connect_archive_id_array(struct obd_export *exp)

diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index 217d851..ada8fb8 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -910,7 +910,8 @@ struct ptlrpc_body_v2 {
                                OBD_CONNECT2_CRUSH | \
                                OBD_CONNECT2_ENCRYPT | \
                                OBD_CONNECT2_GETATTR_PFID |\
-                               OBD_CONNECT2_LSEEK)
+                               OBD_CONNECT2_LSEEK |\
+                               OBD_CONNECT2_PCCRO)
 
 #define OST_CONNECT_SUPPORTED  (OBD_CONNECT_SRVLOCK | OBD_CONNECT_GRANT | \
                                OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | \

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index 3744aa2..916126d 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -279,7 +279,8 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
                                   OBD_CONNECT2_ASYNC_DISCARD |
                                   OBD_CONNECT2_PCC |
                                   OBD_CONNECT2_CRUSH | OBD_CONNECT2_LSEEK |
-                                  OBD_CONNECT2_GETATTR_PFID;
+                                  OBD_CONNECT2_GETATTR_PFID |
+                                  OBD_CONNECT2_PCCRO;
 
 #ifdef HAVE_LRU_RESIZE_SUPPORT
         if (sbi->ll_flags & LL_SBI_LRU_RESIZE)

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index eaadfd0..449e09a 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -2079,8 +2079,17 @@ static int mdt_getattr_name_lock(struct mdt_thread_info *info,
                LASSERT(!(child_bits & MDS_INODELOCK_LAYOUT));
                if (S_ISREG(lu_object_attr(&child->mot_obj)) &&
                    !mdt_object_remote(child) && ldlm_rep != NULL) {
+                       /*
+                        * For the old client without PCC-RO support, it needs
+                        * to clear the LCM_FL_PCC_RDONLY layout flag to avoid
+                        * the inconsistent data access for the purpose of
+                        * compatibility and interoperability.
+                        * Thus it does not grant layout lock to the client
+                        * here.
+                        */
                        if (!OBD_FAIL_CHECK(OBD_FAIL_MDS_NO_LL_GETATTR) &&
-                           exp_connect_layout(info->mti_exp)) {
+                           exp_connect_layout(info->mti_exp) &&
+                           !exp_connect_pccro(info->mti_exp)) {
                                /* try to grant layout lock for regular file. */
                                try_bits = MDS_INODELOCK_LAYOUT;
                        }
@@ -2103,6 +2112,7 @@ static int mdt_getattr_name_lock(struct mdt_thread_info *info,
                                child_bits &= ~MDS_INODELOCK_UPDATE;
                        rc = mdt_object_lock(info, child, lhc, child_bits);
                }
+
                if (unlikely(rc != 0))
                        GOTO(out_child, rc);
        }
@@ -4301,6 +4311,36 @@ out_shrink:
         return rc;
 }
 
+static int mdt_layout_clear_pccro(struct mdt_thread_info *info,
+                                 struct mdt_object *obj)
+{
+       struct mdt_lock_handle *lhc = &info->mti_lh[MDT_LH_RMT];
+       struct md_layout_change layout = { .mlc_opc = MD_LAYOUT_WRITE };
+       struct layout_intent intent = {
+               .li_opc = LAYOUT_INTENT_PCCRO_CLEAR,
+               .li_extent.e_start = 0,
+               .li_extent.e_end = LUSTRE_EOF,
+       };
+       int rc;
+
+       ENTRY;
+
+       layout.mlc_intent = &intent;
+       mdt_lock_handle_init(lhc);
+       mdt_lock_reg_init(lhc, LCK_EX);
+       rc = mdt_reint_object_lock(info, obj, lhc,
+                                  MDS_INODELOCK_LAYOUT, false);
+       if (rc)
+               RETURN(rc);
+
+       mutex_lock(&obj->mot_som_mutex);
+       rc = mo_layout_change(info->mti_env, mdt_object_child(obj), &layout);
+       mutex_unlock(&obj->mot_som_mutex);
+       mdt_object_unlock(info, obj, lhc, rc);
+
+       RETURN(rc);
+}
+
 static int mdt_intent_layout(enum ldlm_intent_flags it_opc,
                             struct mdt_thread_info *info,
                             struct ldlm_lock **lockp,
@@ -4370,6 +4410,15 @@ static int mdt_intent_layout(enum ldlm_intent_flags it_opc,
 
                        if (layout_size > info->mti_mdt->mdt_max_mdsize)
                                info->mti_mdt->mdt_max_mdsize = layout_size;
+
+                       /*
+                        * For the old client without PCC-RO support, it needs
+                        * to clear LCM_FL_PCC_RDONLY flag first to avoid
+                        * inconsistent data access.
+                        */
+                       if (!exp_connect_pccro(info->mti_exp))
+                               rc = mdt_layout_clear_pccro(info, obj);
+
                }
                CDEBUG(D_INFO, "%s: layout_size %d\n",
                       mdt_obd_name(info->mti_mdt), layout_size);
@@ -4393,7 +4442,7 @@ static int mdt_intent_layout(enum ldlm_intent_flags it_opc,
 
        /* take lock in ldlm_lock_enqueue() for LAYOUT_INTENT_ACCESS */
        if (layout.mlc_opc == MD_LAYOUT_NOP)
-               GOTO(out, rc = 0);
+               GOTO(out, rc);
 
        rc = mdt_check_resent(info, mdt_reconstruct_generic, lhc);
        if (rc < 0)

diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c
index e9831b3..2cacb38 100644 (file)
--- a/lustre/mdt/mdt_open.c
+++ b/lustre/mdt/mdt_open.c
@@ -922,6 +922,14 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
                trybits |= MDS_INODELOCK_LAYOUT;
        }
 
+       /*
+        * Dont grant layout lock to the client which does not support PCC-RO
+        * to avoid inconsistent data access for the purpose of compatibility
+        * and interoperability.
+        */
+       if (!exp_connect_pccro(info->mti_exp))
+               trybits &= ~MDS_INODELOCK_LAYOUT;
+
        if (*ibits | trybits)
                rc = mdt_object_lock_try(info, obj, lhc, ibits, trybits, false);