From 2f1b06d21cd41f21ace3d80b4f7044ed74222663 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Fri, 3 Dec 2021 22:14:09 -0700 Subject: [PATCH] EX-3637 osd-ldiskfs: revalidate nonrotational state Until the nonrotational state of the device is correctly exported by the underlying storage, periodically recheck the nonrotational state in case it is changed by the udev/tune_devices.sh script after the OST is mounted. It would be possible to check less often (e.g. after a time limit or some number of operations), but directly checking the rotational state each time is not more expensive and is only checked on statfs RPCs (sent about once per 5s from the MDS). If the nonrotational state is changed and the flash-related cache parameters have not been explicitly set, then tune them appropriately. Rename the parameter functions and variables for read_cache_enable, writethrough_cache_enable, and read_cache_max_filesize to match the parameter names to make them easier to find in the code. Lustre-change: https://review.whamcloud.com/45745 Lustre-commit: TBD (from cbc74314532632fbcab88531d43deed7555e125e) Signed-off-by: Andreas Dilger Change-Id: Iec78a5d5c22c0474eda84a5a793fbb00623ebbe5 Reviewed-by: Li Dongyang Reviewed-by: Gaurang Tapase Tested-by: Gaurang Tapase Reviewed-on: https://review.whamcloud.com/46514 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Shuichi Ihara --- lustre/osd-ldiskfs/osd_handler.c | 31 ++++++++++++++++++++++--------- lustre/osd-ldiskfs/osd_internal.h | 10 ++++++---- lustre/osd-ldiskfs/osd_io.c | 6 +++--- lustre/osd-ldiskfs/osd_lproc.c | 29 ++++++++++++++++++----------- lustre/osd-zfs/osd_internal.h | 3 ++- lustre/osd-zfs/osd_lproc.c | 16 ++++++++++------ 6 files changed, 61 insertions(+), 34 deletions(-) diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index b4a7dd9..9c6b126 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -2247,6 +2247,15 @@ static int osd_object_print(const struct lu_env *env, void *cookie, d ? d->id_ops->id_name : "plain"); } +static void osd_tune_nonrot(struct osd_device *osd, bool nonrotational) +{ + /* do not use pagecache with flash-backed storage */ + if (!osd->od_read_cache_enable_set) + osd->od_read_cache_enable = !nonrotational; + if (!osd->od_writethrough_cache_enable_set) + osd->od_writethrough_cache_enable = !nonrotational; +} + /* * Concurrency: shouldn't matter. */ @@ -2258,6 +2267,7 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, struct kstatfs *ksfs; __u64 reserved; int result = 0; + int nonrot; if (unlikely(osd->od_mnt == NULL)) return -EINPROGRESS; @@ -2279,6 +2289,13 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, if (unlikely(sb->s_flags & SB_RDONLY)) sfs->os_state |= OS_STATFS_READONLY; + /* periodically re-check status from the kernel, if not manually set */ + nonrot = !!blk_queue_nonrot(bdev_get_queue(osd_sb(osd)->s_bdev)); + if (unlikely(nonrot != osd->od_nonrotational && + !osd->od_nonrotational_set)) { + osd->od_nonrotational = nonrot; + osd_tune_nonrot(osd, nonrot); + } sfs->os_state |= osd->od_nonrotational ? OS_STATFS_NONROT : 0; if (ldiskfs_has_feature_extents(sb)) @@ -7948,10 +7965,10 @@ static int osd_mount(const struct lu_env *env, if (lmd_flags & LMD_FLG_NOSCRUB) o->od_auto_scrub_interval = AS_NEVER; + /* Can only check block device after mount */ if (blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev))) { - /* do not use pagecache with flash-backed storage */ - o->od_writethrough_cache = 0; - o->od_read_cache = 0; + o->od_nonrotational = 1; + osd_tune_nonrot(o, true); } GOTO(out, rc = 0); @@ -8018,8 +8035,8 @@ static int osd_device_init0(const struct lu_env *env, o->od_t10_type = 0; init_waitqueue_head(&o->od_commit_cb_done); - o->od_read_cache = 1; - o->od_writethrough_cache = 1; + o->od_read_cache_enable = 1; + o->od_writethrough_cache_enable = 1; o->od_readcache_max_filesize = OSD_MAX_CACHE_SIZE; o->od_readcache_max_iosize = OSD_READCACHE_MAX_IO_MB << 20; o->od_writethrough_max_iosize = OSD_WRITECACHE_MAX_IO_MB << 20; @@ -8046,10 +8063,6 @@ static int osd_device_init0(const struct lu_env *env, if (rc != 0) GOTO(out, rc); - /* Can only check block device after mount */ - o->od_nonrotational = - blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev)); - rc = osd_obj_map_init(env, o); if (rc != 0) GOTO(out_mnt, rc); diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index 77afcda..3131ee2 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -277,10 +277,12 @@ struct osd_device { od_in_init:1, od_index_in_idif:1, /* Other flags */ - od_read_cache:1, - od_writethrough_cache:1, - od_nonrotational:1; - + od_read_cache_enable:1, + od_read_cache_enable_set:1, + od_writethrough_cache_enable:1, + od_writethrough_cache_enable_set:1, + od_nonrotational:1, + od_nonrotational_set:1; __s64 od_auto_scrub_interval; __u32 od_dirent_journal; diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index 636b852..bd50d34 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -63,7 +63,7 @@ static inline bool osd_use_page_cache(struct osd_device *d) { /* do not use pagecache if write and read caching are disabled */ - if (d->od_writethrough_cache + d->od_read_cache == 0) + if (d->od_writethrough_cache_enable + d->od_read_cache_enable == 0) return false; /* use pagecache by default */ return true; @@ -874,7 +874,7 @@ static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, cache = osd_use_page_cache(osd); while (cache) { if (write) { - if (!osd->od_writethrough_cache) { + if (!osd->od_writethrough_cache_enable) { cache = false; break; } @@ -883,7 +883,7 @@ static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, break; } } else { - if (!osd->od_read_cache) { + if (!osd->od_read_cache_enable) { cache = false; break; } diff --git a/lustre/osd-ldiskfs/osd_lproc.c b/lustre/osd-ldiskfs/osd_lproc.c index 42fa6c3..d600856 100644 --- a/lustre/osd-ldiskfs/osd_lproc.c +++ b/lustre/osd-ldiskfs/osd_lproc.c @@ -264,7 +264,7 @@ static ssize_t read_cache_enable_show(struct kobject *kobj, if (unlikely(!osd->od_mnt)) return -EINPROGRESS; - return sprintf(buf, "%u\n", osd->od_read_cache); + return sprintf(buf, "%u\n", osd->od_read_cache_enable); } static ssize_t read_cache_enable_store(struct kobject *kobj, @@ -285,7 +285,9 @@ static ssize_t read_cache_enable_store(struct kobject *kobj, if (rc) return rc; - osd->od_read_cache = !!val; + osd->od_read_cache_enable = !!val; + osd->od_read_cache_enable_set = 1; + return count; } LUSTRE_RW_ATTR(read_cache_enable); @@ -302,7 +304,7 @@ static ssize_t writethrough_cache_enable_show(struct kobject *kobj, if (unlikely(!osd->od_mnt)) return -EINPROGRESS; - return sprintf(buf, "%u\n", osd->od_writethrough_cache); + return sprintf(buf, "%u\n", osd->od_writethrough_cache_enable); } static ssize_t writethrough_cache_enable_store(struct kobject *kobj, @@ -324,7 +326,9 @@ static ssize_t writethrough_cache_enable_store(struct kobject *kobj, if (rc) return rc; - osd->od_writethrough_cache = !!val; + osd->od_writethrough_cache_enable = !!val; + osd->od_writethrough_cache_enable_set = 1; + return count; } LUSTRE_RW_ATTR(writethrough_cache_enable); @@ -432,7 +436,9 @@ static ssize_t nonrotational_store(struct kobject *kobj, if (rc) return rc; - osd->od_nonrotational = val; + osd->od_nonrotational = !!val; + osd->od_nonrotational_set = 1; + return count; } LUSTRE_RW_ATTR(nonrotational); @@ -607,7 +613,8 @@ static int ldiskfs_osd_oi_scrub_seq_show(struct seq_file *m, void *data) LDEBUGFS_SEQ_FOPS_RO(ldiskfs_osd_oi_scrub); -static int ldiskfs_osd_readcache_seq_show(struct seq_file *m, void *data) +static int ldiskfs_osd_readcache_max_filesize_seq_show(struct seq_file *m, + void *data) { struct osd_device *osd = osd_dt_dev((struct dt_device *)m->private); @@ -620,8 +627,9 @@ static int ldiskfs_osd_readcache_seq_show(struct seq_file *m, void *data) } static ssize_t -ldiskfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, - size_t count, loff_t *off) +ldiskfs_osd_readcache_max_filesize_seq_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *off) { struct seq_file *m = file->private_data; struct dt_device *dt = m->private; @@ -649,8 +657,7 @@ ldiskfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, OSD_MAX_CACHE_SIZE : val; return count; } - -LDEBUGFS_SEQ_FOPS(ldiskfs_osd_readcache); +LDEBUGFS_SEQ_FOPS(ldiskfs_osd_readcache_max_filesize); static int ldiskfs_osd_readcache_max_io_seq_show(struct seq_file *m, void *data) { @@ -861,7 +868,7 @@ struct ldebugfs_vars ldebugfs_osd_obd_vars[] = { { .name = "oi_scrub", .fops = &ldiskfs_osd_oi_scrub_fops }, { .name = "readcache_max_filesize", - .fops = &ldiskfs_osd_readcache_fops }, + .fops = &ldiskfs_osd_readcache_max_filesize_fops}, { .name = "readcache_max_io_mb", .fops = &ldiskfs_osd_readcache_max_io_fops }, { .name = "writethrough_max_io_mb", diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index e97f9ff..8588101 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -351,7 +351,8 @@ struct osd_device { od_is_ost:1, od_in_init:1, od_posix_acl:1, - od_nonrotational:1; + od_nonrotational:1, + od_nonrotational_set:1; unsigned int od_dnsize; int od_index_backup_stop; diff --git a/lustre/osd-zfs/osd_lproc.c b/lustre/osd-zfs/osd_lproc.c index c91ccf4..9727e72 100644 --- a/lustre/osd-zfs/osd_lproc.c +++ b/lustre/osd-zfs/osd_lproc.c @@ -327,7 +327,9 @@ static ssize_t nonrotational_store(struct kobject *kobj, if (rc) return rc; - osd->od_nonrotational = val; + osd->od_nonrotational = !!val; + osd->od_nonrotational_set = 1; + return count; } LUSTRE_RW_ATTR(nonrotational); @@ -368,7 +370,8 @@ ssize_t index_backup_store(struct kobject *kobj, struct attribute *attr, } LUSTRE_RW_ATTR(index_backup); -static int zfs_osd_readcache_seq_show(struct seq_file *m, void *data) +static int zfs_osd_readcache_max_filesize_seq_show(struct seq_file *m, + void *data) { struct osd_device *osd = osd_dt_dev((struct dt_device *)m->private); @@ -381,8 +384,9 @@ static int zfs_osd_readcache_seq_show(struct seq_file *m, void *data) } static ssize_t -zfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, - size_t count, loff_t *off) +zfs_osd_readcache_max_filesize_seq_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *off) { struct seq_file *m = file->private_data; struct dt_device *dt = m->private; @@ -410,7 +414,7 @@ zfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, OSD_MAX_CACHE_SIZE : val; return count; } -LDEBUGFS_SEQ_FOPS(zfs_osd_readcache); +LDEBUGFS_SEQ_FOPS(zfs_osd_readcache_max_filesize); static struct attribute *zfs_attrs[] = { &lustre_attr_fstype.attr, @@ -426,7 +430,7 @@ struct ldebugfs_vars ldebugfs_osd_obd_vars[] = { { .name = "oi_scrub", .fops = &zfs_osd_oi_scrub_fops }, { .name = "readcache_max_filesize", - .fops = &zfs_osd_readcache_fops }, + .fops = &zfs_osd_readcache_max_filesize_fops }, { 0 } }; -- 1.8.3.1