From 882a9f784de27bb6043345316ed3f5f4835a9bbf Mon Sep 17 00:00:00 2001 From: James Simmons Date: Thu, 30 Sep 2021 12:26:19 -0400 Subject: [PATCH] LU-14927 scrub: create shared scrub_needs_check() function. The current functions osd_consistency_check() in both ldiskfs and zfs use ktime_* functions which are exported for pure GPL modules. This is not the case for ZFS. We can refactor the code to create a new common function scrub_needs_check() that can be used along side osd_consistency_check(). Fix a few cases where the error code is not checked for ZFS. Change-Id: I0cc6cd84a35ecc10b511096f4e749a2961da3bbf Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/44689 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Aurelien Degremont Reviewed-by: Oleg Drokin --- lustre/include/lustre_scrub.h | 5 +++++ lustre/obdclass/scrub.c | 20 ++++++++++++++++++ lustre/osd-ldiskfs/osd_handler.c | 29 ++++++++++---------------- lustre/osd-ldiskfs/osd_internal.h | 1 - lustre/osd-ldiskfs/osd_lproc.c | 5 +++-- lustre/osd-ldiskfs/osd_scrub.c | 4 +++- lustre/osd-zfs/osd_handler.c | 6 +++--- lustre/osd-zfs/osd_index.c | 44 +++++++++++++-------------------------- lustre/osd-zfs/osd_internal.h | 3 +-- lustre/osd-zfs/osd_lproc.c | 5 +++-- lustre/osd-zfs/osd_object.c | 2 +- lustre/osd-zfs/osd_scrub.c | 5 +++-- 12 files changed, 68 insertions(+), 61 deletions(-) diff --git a/lustre/include/lustre_scrub.h b/lustre/include/lustre_scrub.h index b5d06c3..1911830 100644 --- a/lustre/include/lustre_scrub.h +++ b/lustre/include/lustre_scrub.h @@ -284,6 +284,9 @@ struct lustre_scrub { /* The time for next checkpoint, seconds */ time64_t os_time_next_checkpoint; + /* How long to wait to start scrubbing */ + time64_t os_auto_scrub_interval; + /* How many objects have been checked since last checkpoint. */ __u64 os_new_checked; __u64 os_pos_current; @@ -345,6 +348,8 @@ void scrub_file_init(struct lustre_scrub *scrub, uuid_t uuid); void scrub_file_reset(struct lustre_scrub *scrub, uuid_t uuid, u64 flags); int scrub_file_load(const struct lu_env *env, struct lustre_scrub *scrub); int scrub_file_store(const struct lu_env *env, struct lustre_scrub *scrub); +bool scrub_needs_check(struct lustre_scrub *scrub, const struct lu_fid *fid, + u64 index); int scrub_checkpoint(const struct lu_env *env, struct lustre_scrub *scrub); int scrub_thread_prep(const struct lu_env *env, struct lustre_scrub *scrub, uuid_t uuid, u64 start); diff --git a/lustre/obdclass/scrub.c b/lustre/obdclass/scrub.c index 94a334d..d3d46b8 100644 --- a/lustre/obdclass/scrub.c +++ b/lustre/obdclass/scrub.c @@ -246,6 +246,26 @@ log: } EXPORT_SYMBOL(scrub_file_store); +bool scrub_needs_check(struct lustre_scrub *scrub, const struct lu_fid *fid, + u64 index) +{ + bool check = true; + + if (!fid_is_norm(fid) && !fid_is_igif(fid)) + check = false; + else if (scrub->os_running && scrub->os_pos_current > index) + check = false; + else if (scrub->os_auto_scrub_interval == AS_NEVER) + check = false; + else if (ktime_get_real_seconds() < + scrub->os_file.sf_time_last_complete + + scrub->os_auto_scrub_interval) + check = false; + + return check; +} +EXPORT_SYMBOL(scrub_needs_check); + int scrub_checkpoint(const struct lu_env *env, struct lustre_scrub *scrub) { struct scrub_file *sf = &scrub->os_file; diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 1b9a566..2bfd3b5 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -1260,7 +1260,7 @@ trigger: goto found; } - if (dev->od_auto_scrub_interval == AS_NEVER) { + if (dev->od_scrub.os_scrub.os_auto_scrub_interval == AS_NEVER) { if (!remote) GOTO(out, result = -EREMCHG); @@ -5854,8 +5854,10 @@ static int osd_ea_add_rec(const struct lu_env *env, struct osd_object *pobj, } static int -osd_consistency_check(struct osd_thread_info *oti, struct osd_device *dev, - const struct lu_fid *fid, struct osd_inode_id *id) +osd_ldiskfs_consistency_check(struct osd_thread_info *oti, + struct osd_device *dev, + const struct lu_fid *fid, + struct osd_inode_id *id) { struct lustre_scrub *scrub = &dev->od_scrub.os_scrub; struct inode *inode = NULL; @@ -5864,18 +5866,8 @@ osd_consistency_check(struct osd_thread_info *oti, struct osd_device *dev, int rc; ENTRY; - - if (!fid_is_norm(fid) && !fid_is_igif(fid)) - RETURN(0); - - if (scrub->os_running && scrub->os_pos_current > id->oii_ino) + if (!scrub_needs_check(scrub, fid, id->oii_ino)) RETURN(0); - - if (dev->od_auto_scrub_interval == AS_NEVER || - ktime_get_real_seconds() < - scrub->os_file.sf_time_last_complete + dev->od_auto_scrub_interval) - RETURN(0); - again: rc = osd_oi_lookup(oti, dev, fid, &oti->oti_id, 0); if (rc == -ENOENT) { @@ -5938,7 +5930,8 @@ trigger: GOTO(out, rc); } - if (dev->od_auto_scrub_interval != AS_NEVER && ++once == 1) { + if (dev->od_scrub.os_scrub.os_auto_scrub_interval != AS_NEVER && + ++once == 1) { rc = osd_scrub_start(oti->oti_env, dev, SS_AUTO_PARTIAL | SS_CLEAR_DRYRUN | SS_CLEAR_FAILOUT); CDEBUG_LIMIT(D_LFSCK | D_CONSOLE | D_WARNING, @@ -6192,7 +6185,7 @@ static int osd_ea_lookup_rec(const struct lu_env *env, struct osd_object *obj, if (rc != 0 || osd_remote_fid(env, dev, fid)) GOTO(out, rc); - rc = osd_consistency_check(oti, dev, fid, id); + rc = osd_ldiskfs_consistency_check(oti, dev, fid, id); if (rc != -ENOENT) { /* Other error should not affect lookup result. */ rc = 0; @@ -8049,7 +8042,7 @@ static int osd_mount(const struct lu_env *env, } if (lmd_flags & LMD_FLG_NOSCRUB) - o->od_auto_scrub_interval = AS_NEVER; + o->od_scrub.os_scrub.os_auto_scrub_interval = AS_NEVER; if (blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev))) { /* do not use pagecache with flash-backed storage */ @@ -8126,7 +8119,7 @@ static int osd_device_init0(const struct lu_env *env, o->od_readcache_max_filesize = OSD_MAX_CACHE_SIZE; o->od_readcache_max_iosize = OSD_READCACHE_MAX_IO_MB << 20; o->od_writethrough_max_iosize = OSD_WRITECACHE_MAX_IO_MB << 20; - o->od_auto_scrub_interval = AS_DEFAULT; + o->od_scrub.os_scrub.os_auto_scrub_interval = AS_DEFAULT; /* default fallocate to unwritten extents: LU-14326/LU-14333 */ o->od_fallocate_zero_blocks = 0; diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index 7d8a5c0..3c47beb 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -280,7 +280,6 @@ struct osd_device { od_nonrotational:1; - __s64 od_auto_scrub_interval; __u32 od_dirent_journal; int od_index; struct proc_dir_entry *od_proc_entry; diff --git a/lustre/osd-ldiskfs/osd_lproc.c b/lustre/osd-ldiskfs/osd_lproc.c index 906e53a..0fbe1c0 100644 --- a/lustre/osd-ldiskfs/osd_lproc.c +++ b/lustre/osd-ldiskfs/osd_lproc.c @@ -469,7 +469,8 @@ static ssize_t auto_scrub_show(struct kobject *kobj, struct attribute *attr, if (unlikely(!dev->od_mnt)) return -EINPROGRESS; - return sprintf(buf, "%lld\n", dev->od_auto_scrub_interval); + return scnprintf(buf, PAGE_SIZE, "%lld\n", + dev->od_scrub.os_scrub.os_auto_scrub_interval); } static ssize_t auto_scrub_store(struct kobject *kobj, struct attribute *attr, @@ -489,7 +490,7 @@ static ssize_t auto_scrub_store(struct kobject *kobj, struct attribute *attr, if (rc) return rc; - dev->od_auto_scrub_interval = val; + dev->od_scrub.os_scrub.os_auto_scrub_interval = val; return count; } LUSTRE_RW_ATTR(auto_scrub); diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c index cefaaaa..cdd2b6d 100644 --- a/lustre/osd-ldiskfs/osd_scrub.c +++ b/lustre/osd-ldiskfs/osd_scrub.c @@ -2457,6 +2457,7 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, struct osd_thread_info *info = osd_oti_get(env); struct lustre_scrub *scrub = &dev->od_scrub.os_scrub; struct lvfs_run_ctxt *ctxt = &dev->od_scrub.os_ctxt; + time64_t interval = scrub->os_auto_scrub_interval; struct scrub_file *sf = &scrub->os_file; struct super_block *sb = osd_sb(dev); struct lvfs_run_ctxt saved; @@ -2478,6 +2479,7 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, spin_lock_init(&scrub->os_lock); INIT_LIST_HEAD(&scrub->os_inconsistent_items); scrub->os_name = osd_name(dev); + scrub->os_auto_scrub_interval = interval; push_ctxt(&saved, ctxt); filp = filp_open(osd_scrub_name, O_RDWR | @@ -2599,7 +2601,7 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, dev->od_igif_inoi = 1; if (!dev->od_dt_dev.dd_rdonly && - dev->od_auto_scrub_interval != AS_NEVER && + dev->od_scrub.os_scrub.os_auto_scrub_interval != AS_NEVER && ((sf->sf_status == SS_PAUSED) || (sf->sf_status == SS_CRASHED && sf->sf_flags & (SF_RECREATED | SF_INCONSISTENT | diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index 92146d6..bd6d8df 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -1094,6 +1094,7 @@ static int osd_mount(const struct lu_env *env, char *mntdev = lustre_cfg_string(cfg, 1); char *str = lustre_cfg_string(cfg, 2); char *svname = lustre_cfg_string(cfg, 4); + time64_t interval = AS_DEFAULT; dnode_t *rootdn; const char *opts; bool resetoi = false; @@ -1135,7 +1136,7 @@ static int osd_mount(const struct lu_env *env, } if (flags & LMD_FLG_NOSCRUB) - o->od_auto_scrub_interval = AS_NEVER; + interval = AS_NEVER; } if (server_name_is_ost(o->od_svname)) @@ -1191,7 +1192,7 @@ static int osd_mount(const struct lu_env *env, resetoi = true; o->od_in_init = 1; - rc = osd_scrub_setup(env, o, resetoi); + rc = osd_scrub_setup(env, o, interval, resetoi); o->od_in_init = 0; if (rc) GOTO(err, rc); @@ -1310,7 +1311,6 @@ static int osd_device_init0(const struct lu_env *env, o->od_dt_dev.dd_ops = &osd_dt_ops; sema_init(&o->od_otable_sem, 1); INIT_LIST_HEAD(&o->od_ios_list); - o->od_auto_scrub_interval = AS_DEFAULT; o->od_sync_on_lseek = B_TRUE; /* ZFS does not support reporting nonrotional status yet, so this flag diff --git a/lustre/osd-zfs/osd_index.c b/lustre/osd-zfs/osd_index.c index 6dedc24..8b32422 100644 --- a/lustre/osd-zfs/osd_index.c +++ b/lustre/osd-zfs/osd_index.c @@ -536,10 +536,10 @@ out_nvbuf: return rc; } -static int -osd_consistency_check(const struct lu_env *env, struct osd_device *osd, - struct osd_object *obj, const struct lu_fid *fid, - uint64_t oid, bool is_dir) +static void +osd_zfs_consistency_check(const struct lu_env *env, struct osd_device *osd, + struct osd_object *obj, const struct lu_fid *fid, + u64 oid, bool is_dir) { struct lustre_scrub *scrub = &osd->od_scrub; dnode_t *dn = NULL; @@ -547,34 +547,21 @@ osd_consistency_check(const struct lu_env *env, struct osd_device *osd, int once = 0; bool insert; int rc; - ENTRY; - - if (!fid_is_norm(fid) && !fid_is_igif(fid)) - RETURN(0); + ENTRY; /* oid == ZFS_NO_OBJECT must be for lookup ".." case */ if (oid == ZFS_NO_OBJECT) { rc = osd_sa_handle_get(obj); if (rc) - RETURN(rc); + return; rc = -sa_lookup(obj->oo_sa_hdl, SA_ZPL_PARENT(osd), &oid, 8); if (rc) - RETURN(rc); - } - - if (scrub->os_running) { - if (scrub->os_pos_current > oid) - RETURN(0); - } else if (osd->od_auto_scrub_interval == AS_NEVER) { - RETURN(0); - } else { - if (ktime_get_real_seconds() < - scrub->os_file.sf_time_last_complete + - osd->od_auto_scrub_interval) - RETURN(0); + return; } + if (!scrub_needs_check(&osd->od_scrub, fid, oid)) + return; again: rc = osd_fid_lookup(env, osd, fid, &oid2); if (rc == -ENOENT) { @@ -585,7 +572,7 @@ again: rc = __osd_obj2dnode(osd->od_os, oid, &dn); /* The object has been removed (by race maybe). */ if (rc) - RETURN(rc = (rc == -EEXIST ? -ENOENT : rc)); + return; goto trigger; } else if (rc || oid == oid2) { @@ -600,7 +587,7 @@ trigger: rc = __osd_obj2dnode(osd->od_os, oid, &dn); /* The object has been removed (by race maybe). */ if (rc) - RETURN(rc = (rc == -EEXIST ? -ENOENT : rc)); + return; } rc = osd_oii_insert(env, osd, fid, oid, insert); @@ -619,7 +606,7 @@ trigger: GOTO(out, rc); } - if (osd->od_auto_scrub_interval != AS_NEVER && ++once == 1) { + if (osd->od_scrub.os_auto_scrub_interval != AS_NEVER && ++once == 1) { rc = osd_scrub_start(env, osd, SS_AUTO_FULL | SS_CLEAR_DRYRUN | SS_CLEAR_FAILOUT); CDEBUG_LIMIT(D_LFSCK | D_CONSOLE | D_WARNING, @@ -634,8 +621,6 @@ trigger: out: if (dn) osd_dnode_rele(dn); - - return rc; } static int osd_dir_lookup(const struct lu_env *env, struct dt_object *dt, @@ -687,6 +672,8 @@ out_unlock: up_read(&obj->oo_guard); out: if (!rc && !osd_remote_fid(env, osd, fid)) { + bool is_dir = S_ISDIR(DTTOIF(oti->oti_zde.lzd_reg.zde_type)); + /* * this should ask the scrubber to check OI given * the mapping we just found in the dir entry. @@ -696,8 +683,7 @@ out: * from the layers above, including LFSCK which * is supposed to fix dangling entries. */ - osd_consistency_check(env, osd, obj, fid, oid, - S_ISDIR(DTTOIF(oti->oti_zde.lzd_reg.zde_type))); + osd_zfs_consistency_check(env, osd, obj, fid, oid, is_dir); } return rc == 0 ? 1 : (rc == -ENOENT ? -ENODATA : rc); diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index 57bfbbd..e3fcd3d 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -362,7 +362,6 @@ struct osd_device { int od_connects; int od_index; - __s64 od_auto_scrub_interval; struct lu_site od_site; dnode_t *od_groupused_dn; @@ -695,7 +694,7 @@ int __osd_xattr_load_by_oid(struct osd_device *osd, uint64_t oid, /* osd_scrub.c */ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, - bool resetoi); + time64_t interval, bool resetoi); void osd_scrub_cleanup(const struct lu_env *env, struct osd_device *dev); int osd_scrub_start(const struct lu_env *env, struct osd_device *dev, __u32 flags); diff --git a/lustre/osd-zfs/osd_lproc.c b/lustre/osd-zfs/osd_lproc.c index b92f61f..75cd28a 100644 --- a/lustre/osd-zfs/osd_lproc.c +++ b/lustre/osd-zfs/osd_lproc.c @@ -230,7 +230,8 @@ static ssize_t auto_scrub_show(struct kobject *kobj, struct attribute *attr, if (!dev->od_os) return -EINPROGRESS; - return sprintf(buf, "%lld\n", dev->od_auto_scrub_interval); + return scnprintf(buf, PAGE_SIZE, "%lld\n", + dev->od_scrub.os_auto_scrub_interval); } static ssize_t auto_scrub_store(struct kobject *kobj, struct attribute *attr, @@ -250,7 +251,7 @@ static ssize_t auto_scrub_store(struct kobject *kobj, struct attribute *attr, if (rc) return rc; - dev->od_auto_scrub_interval = val; + dev->od_scrub.os_auto_scrub_interval = val; return count; } LUSTRE_RW_ATTR(auto_scrub); diff --git a/lustre/osd-zfs/osd_object.c b/lustre/osd-zfs/osd_object.c index 3ba6522..85bb856 100644 --- a/lustre/osd-zfs/osd_object.c +++ b/lustre/osd-zfs/osd_object.c @@ -659,7 +659,7 @@ trigger: } /* The case NOT allow to trigger OI scrub automatically. */ - if (osd->od_auto_scrub_interval == AS_NEVER) + if (osd->od_scrub.os_auto_scrub_interval == AS_NEVER) GOTO(out, rc); /* It is me to trigger the OI scrub. */ diff --git a/lustre/osd-zfs/osd_scrub.c b/lustre/osd-zfs/osd_scrub.c index c581303..979e377 100644 --- a/lustre/osd-zfs/osd_scrub.c +++ b/lustre/osd-zfs/osd_scrub.c @@ -1317,7 +1317,7 @@ void osd_scrub_stop(struct osd_device *dev) static const char osd_scrub_name[] = "OI_scrub"; int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, - bool resetoi) + time64_t interval, bool resetoi) { struct osd_thread_info *info = osd_oti_get(env); struct lustre_scrub *scrub = &dev->od_scrub; @@ -1337,6 +1337,7 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, spin_lock_init(&scrub->os_lock); INIT_LIST_HEAD(&scrub->os_inconsistent_items); scrub->os_name = osd_name(dev); + scrub->os_auto_scrub_interval = interval; /* 'What the @fid is' is not imporatant, because the object * has no OI mapping, and only is visible inside the OSD.*/ @@ -1408,7 +1409,7 @@ int osd_scrub_setup(const struct lu_env *env, struct osd_device *dev, osd_initial_OI_scrub(env, dev); if (!dev->od_dt_dev.dd_rdonly && - dev->od_auto_scrub_interval != AS_NEVER && + scrub->os_auto_scrub_interval != AS_NEVER && ((sf->sf_status == SS_PAUSED) || (sf->sf_status == SS_CRASHED && sf->sf_flags & (SF_RECREATED | SF_INCONSISTENT | -- 1.8.3.1