LU-14119 osd: add mount option "resetoi"

OI files on zfs are special, and they can't be deleted by user space
tools like rm. Sometimes the OI files may contain stale OI mappings,
and they needed to be removed for namespace consistency. Add a mount
option 'resetoi' to recreate OI files on mount time, and it will
support both ldiskfs and zfs. This should be the standard way to
recreate OI files, other than mount as backend filesystem and unlink
them manually.

Add sanity-scrub 17.

Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: Idc0e4c2f3b81675c49c6c005bc30b61d8fd04503
Reviewed-on: https://review.whamcloud.com/41402
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index 00e3ebd..acf36f5 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -7791,6 +7791,7 @@ static int osd_mount(const struct lu_env *env,
                        "force_over_256tb",
                        "force_over_512tb",
                        "force_over_1024tb",
+                       "resetoi",
                        NULL
                };
                strncat(options, opts, PAGE_SIZE);
@@ -7922,8 +7923,10 @@ static int osd_device_init0(const struct lu_env *env,
 {
        struct lu_device *l = osd2lu_dev(o);
        struct osd_thread_info *info;
-       int rc;
        int cplen = 0;
+       char *opts = NULL;
+       bool restored = false;
+       int rc;
 
        /* if the module was re-loaded, env can loose its keys */
        rc = lu_env_refill((struct lu_env *)env);
@@ -7990,10 +7993,14 @@ static int osd_device_init0(const struct lu_env *env,
        if (rc != 0)
                GOTO(out_site, rc);
 
+       opts = lustre_cfg_string(cfg, 3);
+       if (opts && strstr(opts, "resetoi"))
+               restored = true;
+
        INIT_LIST_HEAD(&o->od_ios_list);
        /* setup scrub, including OI files initialization */
        o->od_in_init = 1;
-       rc = osd_scrub_setup(env, o);
+       rc = osd_scrub_setup(env, o, restored);
        o->od_in_init = 0;
        if (rc < 0)
                GOTO(out_site, rc);

diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h
index f9db66f..ad53ec6 100644 (file)
--- a/lustre/osd-ldiskfs/osd_internal.h
+++ b/lustre/osd-ldiskfs/osd_internal.h
@@ -798,7 +798,8 @@ char *osd_lf_fid2name(const struct lu_fid *fid);
 int osd_scrub_start(const struct lu_env *env, struct osd_device *dev,
                    __u32 flags);
 void osd_scrub_stop(struct osd_device *dev);
-int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev);
+int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev,
+                   bool restored);
 void osd_scrub_cleanup(const struct lu_env *env, struct osd_device *dev);
 int osd_oii_insert(struct osd_device *dev, const struct lu_fid *fid,
                   struct osd_inode_id *id, int insert);

diff --git a/lustre/osd-ldiskfs/osd_oi.c b/lustre/osd-ldiskfs/osd_oi.c
index bc4882a..b44af04 100644 (file)
--- a/lustre/osd-ldiskfs/osd_oi.c
+++ b/lustre/osd-ldiskfs/osd_oi.c
@@ -405,6 +405,8 @@ int osd_oi_init(struct osd_thread_info *info, struct osd_device *osd,
        }
 
        if (restored) {
+               LCONSOLE_WARN("%s: reset Object Index mappings\n",
+                             osd_dev2name(osd));
                rc = osd_remove_ois(info, osd);
                if (rc)
                        RETURN(rc);

diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c
index ecc7631..b541f5c 100644 (file)
--- a/lustre/osd-ldiskfs/osd_scrub.c
+++ b/lustre/osd-ldiskfs/osd_scrub.c
@@ -2559,7 +2559,8 @@ void osd_scrub_stop(struct osd_device *dev)
 
 static const char osd_scrub_name[] = "OI_scrub";
 
-int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev)
+int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev,
+                   bool restored)
 {
        struct osd_thread_info *info = osd_oti_get(env);
        struct lustre_scrub *scrub = &dev->od_scrub.os_scrub;
@@ -2573,7 +2574,6 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev)
        struct osd_inode_id *id = &info->oti_id;
        struct dt_object *obj;
        bool dirty = false;
-       bool restored = false;
        int rc = 0;
        ENTRY;
 

diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c
index 30ec13e..cde52ff 100644 (file)
--- a/lustre/osd-zfs/osd_handler.c
+++ b/lustre/osd-zfs/osd_handler.c
@@ -1070,12 +1070,14 @@ osd_unlinked_drain(const struct lu_env *env, struct osd_device *osd)
 static int osd_mount(const struct lu_env *env,
                     struct osd_device *o, struct lustre_cfg *cfg)
 {
-       char                    *mntdev = lustre_cfg_string(cfg, 1);
-       char                    *str    = lustre_cfg_string(cfg, 2);
-       char                    *svname = lustre_cfg_string(cfg, 4);
+       char *mntdev = lustre_cfg_string(cfg, 1);
+       char *str = lustre_cfg_string(cfg, 2);
+       char *svname = lustre_cfg_string(cfg, 4);
        dnode_t *rootdn;
-       const char              *opts;
-       int                      rc;
+       const char *opts;
+       bool resetoi = false;
+       int rc;
+
        ENTRY;
 
        if (o->od_os != NULL)
@@ -1092,6 +1094,8 @@ static int osd_mount(const struct lu_env *env,
        if (rc >= sizeof(o->od_svname))
                RETURN(-E2BIG);
 
+       opts = lustre_cfg_string(cfg, 3);
+
        o->od_index_backup_stop = 0;
        o->od_index = -1; /* -1 means index is invalid */
        rc = server_name2index(o->od_svname, &o->od_index, NULL);
@@ -1162,8 +1166,11 @@ static int osd_mount(const struct lu_env *env,
        if (rc)
                GOTO(err, rc);
 
+       if (opts && strstr(opts, "resetoi"))
+               resetoi = true;
+
        o->od_in_init = 1;
-       rc = osd_scrub_setup(env, o);
+       rc = osd_scrub_setup(env, o, resetoi);
        o->od_in_init = 0;
        if (rc)
                GOTO(err, rc);
@@ -1206,7 +1213,6 @@ static int osd_mount(const struct lu_env *env,
 #endif
 
        /* parse mount option "noacl", and enable ACL by default */
-       opts = lustre_cfg_string(cfg, 3);
        if (opts == NULL || strstr(opts, "noacl") == NULL)
                o->od_posix_acl = 1;
 

diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h
index f5754c1..498756a 100644 (file)
--- a/lustre/osd-zfs/osd_internal.h
+++ b/lustre/osd-zfs/osd_internal.h
@@ -633,7 +633,7 @@ int osd_find_new_dnode(const struct lu_env *env, dmu_tx_t *tx,
                       uint64_t oid, dnode_t **dnp);
 
 /* osd_oi.c */
-int osd_oi_init(const struct lu_env *env, struct osd_device *o);
+int osd_oi_init(const struct lu_env *env, struct osd_device *o, bool reset);
 void osd_oi_fini(const struct lu_env *env, struct osd_device *o);
 int osd_fid_lookup(const struct lu_env *env,
                   struct osd_device *, const struct lu_fid *, uint64_t *);
@@ -690,7 +690,8 @@ int __osd_xattr_load_by_oid(struct osd_device *osd, uint64_t oid,
                            nvlist_t **sa);
 
 /* osd_scrub.c */
-int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev);
+int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev,
+                   bool resetoi);
 void osd_scrub_cleanup(const struct lu_env *env, struct osd_device *dev);
 int osd_scrub_start(const struct lu_env *env, struct osd_device *dev,
                    __u32 flags);

diff --git a/lustre/osd-zfs/osd_oi.c b/lustre/osd-zfs/osd_oi.c
index af618f0..38ff708 100644 (file)
--- a/lustre/osd-zfs/osd_oi.c
+++ b/lustre/osd-zfs/osd_oi.c
@@ -228,6 +228,65 @@ out:
        return rc;
 }
 
+static int osd_oi_destroy(const struct lu_env *env, struct osd_device *o,
+                         const char *name)
+{
+       struct osd_oi oi;
+       dmu_tx_t *tx;
+       dnode_t *rootdn;
+       uint64_t oid;
+       int rc;
+
+       ENTRY;
+
+       if (o->od_dt_dev.dd_rdonly)
+               RETURN(-EROFS);
+
+       rc = osd_oi_lookup(env, o, o->od_rootid, name, &oi);
+       if (rc == -ENOENT)
+               RETURN(0);
+       if (rc)
+               RETURN(rc);
+
+       oid = oi.oi_zapid;
+
+       rc = __osd_obj2dnode(o->od_os, o->od_rootid, &rootdn);
+       if (rc)
+               RETURN(rc);
+
+       tx = dmu_tx_create(o->od_os);
+       dmu_tx_mark_netfree(tx);
+       dmu_tx_hold_free(tx, oid, 0, DMU_OBJECT_END);
+       osd_tx_hold_zap(tx, oid, rootdn, FALSE, NULL);
+       rc = -dmu_tx_assign(tx, TXG_WAIT);
+       if (rc) {
+               dmu_tx_abort(tx);
+               GOTO(out, rc);
+       }
+
+       rc = -dmu_object_free(o->od_os, oid, tx);
+       if (rc) {
+               CERROR("%s: failed to free %s %llu: rc = %d\n",
+                      o->od_svname, name, oid, rc);
+               GOTO(commit, rc);
+       }
+
+       rc = osd_zap_remove(o, o->od_rootid, rootdn, name, tx);
+       if (rc) {
+               CERROR("%s: zap_remove %s failed: rc = %d\n",
+                      o->od_svname, name, rc);
+               GOTO(commit, rc);
+       }
+
+       EXIT;
+commit:
+       dmu_tx_commit(tx);
+out:
+       osd_dnode_rele(rootdn);
+
+       return rc;
+}
+
 static int
 osd_oi_find_or_create(const struct lu_env *env, struct osd_device *o,
                      uint64_t parent, const char *name, uint64_t *child)
@@ -844,7 +903,7 @@ osd_oi_init_remote_parent(const struct lu_env *env, struct osd_device *o)
 /**
  * Initialize the OIs by either opening or creating them as needed.
  */
-int osd_oi_init(const struct lu_env *env, struct osd_device *o)
+int osd_oi_init(const struct lu_env *env, struct osd_device *o, bool reset)
 {
        struct lustre_scrub *scrub = &o->od_scrub;
        struct scrub_file *sf = &scrub->os_file;
@@ -871,7 +930,7 @@ int osd_oi_init(const struct lu_env *env, struct osd_device *o)
                GOTO(out, rc = count);
 
        if (count > 0) {
-               if (count == sf->sf_oi_count)
+               if (count == sf->sf_oi_count && !reset)
                        goto open;
 
                if (sf->sf_oi_count == 0) {
@@ -908,11 +967,20 @@ int osd_oi_init(const struct lu_env *env, struct osd_device *o)
        if (rc)
                GOTO(out, rc);
 
+       if (reset)
+               LCONSOLE_WARN("%s: reset Object Index mappings\n",
+                             osd_name(o));
+
        for (i = 0; i < count; i++) {
                LASSERT(sizeof(osd_oti_get(env)->oti_buf) >= 32);
 
                snprintf(key, sizeof(osd_oti_get(env)->oti_buf) - 1,
                         "%s.%d", DMU_OSD_OI_NAME_BASE, i);
+               if (reset) {
+                       rc = osd_oi_destroy(env, o, key);
+                       if (rc)
+                               GOTO(out, rc);
+               }
                rc = osd_oi_find_or_create(env, o, o->od_root, key, &sdb);
                if (rc)
                        GOTO(out, rc);

diff --git a/lustre/osd-zfs/osd_scrub.c b/lustre/osd-zfs/osd_scrub.c
index 8579ee0..f8fb843 100644 (file)
--- a/lustre/osd-zfs/osd_scrub.c
+++ b/lustre/osd-zfs/osd_scrub.c
@@ -1402,7 +1402,8 @@ void osd_scrub_stop(struct osd_device *dev)
 
 static const char osd_scrub_name[] = "OI_scrub";
 
-int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev)
+int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev,
+                   bool resetoi)
 {
        struct osd_thread_info *info = osd_oti_get(env);
        struct lustre_scrub *scrub = &dev->od_scrub;
@@ -1485,7 +1486,7 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev)
        }
 
        /* Initialize OI files. */
-       rc = osd_oi_init(env, dev);
+       rc = osd_oi_init(env, dev, resetoi);
        if (rc < 0)
                GOTO(cleanup_obj, rc);
 

diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh
index 3a90dc0..fa6e043 100644 (file)
--- a/lustre/tests/sanity-scrub.sh
+++ b/lustre/tests/sanity-scrub.sh
@@ -1304,6 +1304,39 @@ test_17b() {
 }
 run_test 17b "ENOSPC on .. insertion shouldn't leak inodes"
 
+test_18() {
+       local n
+       local fids=()
+       local opts=$(csa_add "$MOUNT_OPTS_SCRUB" -o resetoi)
+
+       scrub_prep 10
+       scrub_start_mds 1 "$MOUNT_OPTS_SCRUB"
+       mount_client $MOUNT || error "(2) Fail to start client!"
+       for n in $(seq $MDSCOUNT); do
+               fids+=($($LFS path2fid $DIR/$tdir/mds$n/test-framework.sh))
+       done
+       cleanup_mount $MOUNT > /dev/null || error "(3) Fail to stop client!"
+       for n in $(seq $MDSCOUNT); do
+               stop mds$n > /dev/null || error "(4) Fail to stop MDS$n!"
+       done
+       scrub_start_mds 5 "$opts"
+       do_facet mds1 dmesg | grep "reset Object Index" ||
+               error "(6) reset log not found"
+       mount_client $MOUNT || error "(7) Fail to start client!"
+       scrub_check_data 7
+
+       local fid
+       local path
+       for n in $(seq $MDSCOUNT); do
+               path=$($LFS fid2path $DIR ${fids[$((n - 1))]})
+               [ "$path" == "$DIR/$tdir/mds$n/test-framework.sh" ] ||
+                       error "path mismatch $path != $DIR/$tdir/mds$n/test-framework.sh"
+               fid=$($LFS path2fid $DIR/$tdir/mds$n/test-framework.sh)
+               [ "${fids[$((n - 1))]}" == "$fid" ] ||
+                       error "$DIR/$tdir/mds$n/test-framework.sh FID mismatch ${fids[$((n - 1))]} != $fid"
+       done
+}
+run_test 18 "test mount -o resetoi to recreate OI files"
 
 # restore MDS/OST size
 MDSSIZE=${SAVED_MDSSIZE}