LU-5409 obd: add CFS_FAULT_CHECK()

Add the macro CFS_FAULT_CHECK() which behaves like CFS_FAIL_CHECK()
except that any site may be matched by setting CFS_FAULT (0x02000000)
in cfs_fail_loc. Add cfs_fail_err for use as a return value with
CFS_FAULT_CHECK(). Add fault injection points to the DT object
operation wrappers using CFS_FAULT_CHECK().

Signed-off-by: John L. Hammond <john.hammond@intel.com>
Change-Id: I9841dbe0c62a13107c62382c8d7a73af207b1ef4
Reviewed-on: http://review.whamcloud.com/11263
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Robert Read <robert.read@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/libcfs/include/libcfs/libcfs_fail.h b/libcfs/include/libcfs/libcfs_fail.h
index 51f07bc..79c6795 100644 (file)
--- a/libcfs/include/libcfs/libcfs_fail.h
+++ b/libcfs/include/libcfs/libcfs_fail.h
@@ -38,6 +38,7 @@
 
 extern unsigned long cfs_fail_loc;
 extern unsigned int cfs_fail_val;
+extern int cfs_fail_err;
 
 extern wait_queue_head_t cfs_race_waitq;
 extern int cfs_race_state;
@@ -70,9 +71,15 @@ enum {
 #define CFS_FAIL_RAND        0x08000000 /* fail 1/N of the times */
 #define CFS_FAIL_USR1        0x04000000 /* user flag */
 
-#define CFS_FAIL_PRECHECK(id) (cfs_fail_loc &&                                \
-                              (cfs_fail_loc & CFS_FAIL_MASK_LOC) ==           \
-                              ((id) & CFS_FAIL_MASK_LOC))
+/* CFS_FAULT may be combined with any one of the above flags. */
+#define CFS_FAULT           0x02000000 /* match any CFS_FAULT_CHECK */
+
+static inline bool CFS_FAIL_PRECHECK(__u32 id)
+{
+       return cfs_fail_loc != 0 &&
+             ((cfs_fail_loc & CFS_FAIL_MASK_LOC) == (id & CFS_FAIL_MASK_LOC) ||
+              (cfs_fail_loc & id & CFS_FAULT));
+}
 
 static inline int cfs_fail_check_set(__u32 id, __u32 value,
                                     int set, int quiet)
@@ -146,6 +153,9 @@ static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
 #define CFS_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
         cfs_fail_timeout_set(id, value, ms, CFS_FAIL_LOC_ORSET)
 
+#define CFS_FAULT_CHECK(id)                    \
+       CFS_FAIL_CHECK(CFS_FAULT | (id))
+
 #ifdef __KERNEL__
 /* The idea here is to synchronise two threads to force a race. The
  * first thread that calls this with a matching fail_loc is put to

diff --git a/libcfs/libcfs/fail.c b/libcfs/libcfs/fail.c
index 4315bb3..13a7539 100644 (file)
--- a/libcfs/libcfs/fail.c
+++ b/libcfs/libcfs/fail.c
@@ -37,11 +37,13 @@
 
 unsigned long cfs_fail_loc = 0;
 unsigned int cfs_fail_val = 0;
+int cfs_fail_err;
 wait_queue_head_t cfs_race_waitq;
 int cfs_race_state;
 
 EXPORT_SYMBOL(cfs_fail_loc);
 EXPORT_SYMBOL(cfs_fail_val);
+EXPORT_SYMBOL(cfs_fail_err);
 EXPORT_SYMBOL(cfs_race_waitq);
 EXPORT_SYMBOL(cfs_race_state);
 

diff --git a/libcfs/libcfs/linux/linux-proc.c b/libcfs/libcfs/linux/linux-proc.c
index 822bd1c..113dc18 100644 (file)
--- a/libcfs/libcfs/linux/linux-proc.c
+++ b/libcfs/libcfs/linux/linux-proc.c
@@ -511,6 +511,14 @@ static struct ctl_table lnet_table[] = {
                .mode           = 0644,
                .proc_handler   = &proc_dointvec
        },
+       {
+               INIT_CTL_NAME
+               .procname       = "fail_err",
+               .data           = &cfs_fail_err,
+               .maxlen         = sizeof(cfs_fail_err),
+               .mode           = 0644,
+               .proc_handler   = &proc_dointvec,
+       },
        { 0 }
 };
 

diff --git a/lustre/include/dt_object.h b/lustre/include/dt_object.h
index acf22f5..254134b 100644 (file)
--- a/lustre/include/dt_object.h
+++ b/lustre/include/dt_object.h
@@ -1067,6 +1067,10 @@ static inline int dt_declare_create(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_create);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_CREATE))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_create(env, dt, attr, hint, dof, th);
 }
 
@@ -1080,6 +1084,10 @@ static inline int dt_create(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_create);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_CREATE))
+               return cfs_fail_err;
+
         return dt->do_ops->do_create(env, dt, attr, hint, dof, th);
 }
 
@@ -1090,6 +1098,10 @@ static inline int dt_declare_destroy(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_destroy);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_DESTROY))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_destroy(env, dt, th);
 }
 
@@ -1100,6 +1112,10 @@ static inline int dt_destroy(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_destroy);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DESTROY))
+               return cfs_fail_err;
+
         return dt->do_ops->do_destroy(env, dt, th);
 }
 
@@ -1157,6 +1173,10 @@ static inline int dt_declare_attr_get(const struct lu_env *env,
        LASSERT(dt);
        LASSERT(dt->do_ops);
        LASSERT(dt->do_ops->do_declare_attr_get);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_ATTR_GET))
+               return cfs_fail_err;
+
        return dt->do_ops->do_declare_attr_get(env, dt, capa);
 }
 
@@ -1166,6 +1186,10 @@ static inline int dt_attr_get(const struct lu_env *env, struct dt_object *dt,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_attr_get);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_ATTR_GET))
+               return cfs_fail_err;
+
         return dt->do_ops->do_attr_get(env, dt, la, arg);
 }
 
@@ -1177,6 +1201,10 @@ static inline int dt_declare_attr_set(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_attr_set);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_ATTR_SET))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_attr_set(env, dt, la, th);
 }
 
@@ -1187,6 +1215,10 @@ static inline int dt_attr_set(const struct lu_env *env, struct dt_object *dt,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_attr_set);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_ATTR_SET))
+               return cfs_fail_err;
+
         return dt->do_ops->do_attr_set(env, dt, la, th, capa);
 }
 
@@ -1196,6 +1228,10 @@ static inline int dt_declare_ref_add(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_ref_add);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_REF_ADD))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_ref_add(env, dt, th);
 }
 
@@ -1205,6 +1241,10 @@ static inline int dt_ref_add(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_ref_add);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_REF_ADD))
+               return cfs_fail_err;
+
         return dt->do_ops->do_ref_add(env, dt, th);
 }
 
@@ -1214,6 +1254,10 @@ static inline int dt_declare_ref_del(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_ref_del);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_REF_DEL))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_ref_del(env, dt, th);
 }
 
@@ -1223,6 +1267,10 @@ static inline int dt_ref_del(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_ref_del);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_REF_DEL))
+               return cfs_fail_err;
+
         return dt->do_ops->do_ref_del(env, dt, th);
 }
 
@@ -1380,6 +1428,10 @@ static inline int dt_declare_insert(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_index_ops);
         LASSERT(dt->do_index_ops->dio_declare_insert);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_INSERT))
+               return cfs_fail_err;
+
         return dt->do_index_ops->dio_declare_insert(env, dt, rec, key, th);
 }
 
@@ -1394,6 +1446,10 @@ static inline int dt_insert(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_index_ops);
         LASSERT(dt->do_index_ops->dio_insert);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_INSERT))
+               return cfs_fail_err;
+
         return dt->do_index_ops->dio_insert(env, dt, rec, key, th,
                                             capa, noquota);
 }
@@ -1406,6 +1462,10 @@ static inline int dt_declare_xattr_del(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_xattr_del);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_XATTR_DEL))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_xattr_del(env, dt, name, th);
 }
 
@@ -1417,6 +1477,10 @@ static inline int dt_xattr_del(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_xattr_del);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_XATTR_DEL))
+               return cfs_fail_err;
+
         return dt->do_ops->do_xattr_del(env, dt, name, th, capa);
 }
 
@@ -1429,6 +1493,10 @@ static inline int dt_declare_xattr_set(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_declare_xattr_set);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_XATTR_SET))
+               return cfs_fail_err;
+
         return dt->do_ops->do_declare_xattr_set(env, dt, buf, name, fl, th);
 }
 
@@ -1440,6 +1508,10 @@ static inline int dt_xattr_set(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_xattr_set);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_XATTR_SET))
+               return cfs_fail_err;
+
         return dt->do_ops->do_xattr_set(env, dt, buf, name, fl, th, capa);
 }
 
@@ -1452,6 +1524,10 @@ static inline int dt_declare_xattr_get(const struct lu_env *env,
        LASSERT(dt);
        LASSERT(dt->do_ops);
        LASSERT(dt->do_ops->do_declare_xattr_get);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_XATTR_GET))
+               return cfs_fail_err;
+
        return dt->do_ops->do_declare_xattr_get(env, dt, buf, name, capa);
 }
 
@@ -1462,6 +1538,10 @@ static inline int dt_xattr_get(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_xattr_get);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_XATTR_GET))
+               return cfs_fail_err;
+
         return dt->do_ops->do_xattr_get(env, dt, buf, name, capa);
 }
 
@@ -1472,6 +1552,10 @@ static inline int dt_xattr_list(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_ops);
         LASSERT(dt->do_ops->do_xattr_list);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_XATTR_LIST))
+               return cfs_fail_err;
+
         return dt->do_ops->do_xattr_list(env, dt, buf, capa);
 }
 
@@ -1483,6 +1567,10 @@ static inline int dt_declare_delete(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_index_ops);
         LASSERT(dt->do_index_ops->dio_declare_delete);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DECLARE_DELETE))
+               return cfs_fail_err;
+
         return dt->do_index_ops->dio_declare_delete(env, dt, key, th);
 }
 
@@ -1495,6 +1583,10 @@ static inline int dt_delete(const struct lu_env *env,
         LASSERT(dt);
         LASSERT(dt->do_index_ops);
         LASSERT(dt->do_index_ops->dio_delete);
+
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_DELETE))
+               return cfs_fail_err;
+
         return dt->do_index_ops->dio_delete(env, dt, key, th, capa);
 }
 
@@ -1531,6 +1623,9 @@ static inline int dt_lookup(const struct lu_env *env,
         LASSERT(dt->do_index_ops);
         LASSERT(dt->do_index_ops->dio_lookup);
 
+       if (CFS_FAULT_CHECK(OBD_FAIL_DT_LOOKUP))
+               return cfs_fail_err;
+
         ret = dt->do_index_ops->dio_lookup(env, dt, rec, key, capa);
         if (ret > 0)
                 ret = 0;

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 7801966..05860bb 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -538,6 +538,33 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type,
 /* LMV */
 #define OBD_FAIL_UNKNOWN_LMV_STRIPE            0x1901
 
+/* DT */
+#define OBD_FAIL_DT_DECLARE_ATTR_GET           0x2000
+#define OBD_FAIL_DT_ATTR_GET                   0x2001
+#define OBD_FAIL_DT_DECLARE_ATTR_SET           0x2002
+#define OBD_FAIL_DT_ATTR_SET                   0x2003
+#define OBD_FAIL_DT_DECLARE_XATTR_GET          0x2004
+#define OBD_FAIL_DT_XATTR_GET                  0x2005
+#define OBD_FAIL_DT_DECLARE_XATTR_SET          0x2006
+#define OBD_FAIL_DT_XATTR_SET                  0x2007
+#define OBD_FAIL_DT_DECLARE_XATTR_DEL          0x2008
+#define OBD_FAIL_DT_XATTR_DEL                  0x2009
+#define OBD_FAIL_DT_XATTR_LIST                 0x200a
+#define OBD_FAIL_DT_DECLARE_CREATE             0x200b
+#define OBD_FAIL_DT_CREATE                     0x200c
+#define OBD_FAIL_DT_DECLARE_DESTROY            0x200d
+#define OBD_FAIL_DT_DESTROY                    0x200e
+#define OBD_FAIL_DT_INDEX_TRY                  0x200f
+#define OBD_FAIL_DT_DECLARE_REF_ADD            0x2010
+#define OBD_FAIL_DT_REF_ADD                    0x2011
+#define OBD_FAIL_DT_DECLARE_REF_DEL            0x2012
+#define OBD_FAIL_DT_REF_DEL                    0x2013
+#define OBD_FAIL_DT_DECLARE_INSERT             0x2014
+#define OBD_FAIL_DT_INSERT                     0x2015
+#define OBD_FAIL_DT_DECLARE_DELETE             0x2016
+#define OBD_FAIL_DT_DELETE                     0x2017
+#define OBD_FAIL_DT_LOOKUP                     0x2018
+
 /* Assign references to moved code to reduce code changes */
 #define OBD_FAIL_PRECHECK(id)                   CFS_FAIL_PRECHECK(id)
 #define OBD_FAIL_CHECK(id)                      CFS_FAIL_CHECK(id)