From 5a1da772cf1721967d9dae55fae3d19aebba70f8 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Fri, 3 Dec 2021 22:14:09 -0700 Subject: [PATCH] EX-3637 osd-ldiskfs: revalidate nonrotational state Until the nonrotational state of the device is correctly exported by the underlying storage, periodically recheck the nonrotational state in case it is changed by the udev/tune_devices.sh script after the OST is mounted. It would be possible to check less often (e.g. after a time limit or some number of operations), but directly checking the rotational state each time is not more expensive and is only checked on statfs RPCs (sent about once per 5s from the MDS). If the nonrotational state is changed and the flash-related cache parameters have not been explicitly set, then tune them appropriately. Rename the parameter functions and variables for read_cache_enable, writethrough_cache_enable, and read_cache_max_filesize to match the parameter names to make them easier to find in the code. Lustre-change: https://review.whamcloud.com/45745 Lustre-commit: TBD (from cbc74314532632fbcab88531d43deed7555e125e) Signed-off-by: Andreas Dilger Change-Id: Iec78a5d5c22c0474eda84a5a793fbb006f3ebbe5 Reviewed-on: https://review.whamcloud.com/45829 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Li Dongyang Reviewed-by: Gaurang Tapase Tested-by: Gaurang Tapase --- lustre/osd-ldiskfs/osd_handler.c | 31 ++++++++++++++++++++++--------- lustre/osd-ldiskfs/osd_internal.h | 10 ++++++---- lustre/osd-ldiskfs/osd_io.c | 6 +++--- lustre/osd-ldiskfs/osd_lproc.c | 29 ++++++++++++++++++----------- lustre/osd-zfs/osd_internal.h | 3 ++- lustre/osd-zfs/osd_lproc.c | 16 ++++++++++------ 6 files changed, 61 insertions(+), 34 deletions(-) diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index f9643b0..97bbf86 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -2235,6 +2235,15 @@ static int osd_object_print(const struct lu_env *env, void *cookie, d ? d->id_ops->id_name : "plain"); } +static void osd_tune_nonrot(struct osd_device *osd, bool nonrotational) +{ + /* do not use pagecache with flash-backed storage */ + if (!osd->od_read_cache_enable_set) + osd->od_read_cache_enable = !nonrotational; + if (!osd->od_writethrough_cache_enable_set) + osd->od_writethrough_cache_enable = !nonrotational; +} + /* * Concurrency: shouldn't matter. */ @@ -2246,6 +2255,7 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, struct kstatfs *ksfs; __u64 reserved; int result = 0; + int nonrot; if (unlikely(osd->od_mnt == NULL)) return -EINPROGRESS; @@ -2267,6 +2277,13 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, if (unlikely(sb->s_flags & SB_RDONLY)) sfs->os_state |= OS_STATFS_READONLY; + /* periodically re-check status from the kernel, if not manually set */ + nonrot = !!blk_queue_nonrot(bdev_get_queue(osd_sb(osd)->s_bdev)); + if (unlikely(nonrot != osd->od_nonrotational && + !osd->od_nonrotational_set)) { + osd->od_nonrotational = nonrot; + osd_tune_nonrot(osd, nonrot); + } sfs->os_state |= osd->od_nonrotational ? OS_STATFS_NONROT : 0; if (ldiskfs_has_feature_extents(sb)) @@ -7941,10 +7958,10 @@ static int osd_mount(const struct lu_env *env, if (lmd_flags & LMD_FLG_NOSCRUB) o->od_auto_scrub_interval = AS_NEVER; + /* Can only check block device after mount */ if (blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev))) { - /* do not use pagecache with flash-backed storage */ - o->od_writethrough_cache = 0; - o->od_read_cache = 0; + o->od_nonrotational = 1; + osd_tune_nonrot(o, true); } GOTO(out, rc = 0); @@ -8011,8 +8028,8 @@ static int osd_device_init0(const struct lu_env *env, o->od_t10_type = 0; init_waitqueue_head(&o->od_commit_cb_done); - o->od_read_cache = 1; - o->od_writethrough_cache = 1; + o->od_read_cache_enable = 1; + o->od_writethrough_cache_enable = 1; o->od_readcache_max_filesize = OSD_MAX_CACHE_SIZE; o->od_readcache_max_iosize = OSD_READCACHE_MAX_IO_MB << 20; o->od_writethrough_max_iosize = OSD_WRITECACHE_MAX_IO_MB << 20; @@ -8039,10 +8056,6 @@ static int osd_device_init0(const struct lu_env *env, if (rc != 0) GOTO(out, rc); - /* Can only check block device after mount */ - o->od_nonrotational = - blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev)); - rc = osd_obj_map_init(env, o); if (rc != 0) GOTO(out_mnt, rc); diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index f618617..18ebdfc 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -276,10 +276,12 @@ struct osd_device { od_in_init:1, od_index_in_idif:1, /* Other flags */ - od_read_cache:1, - od_writethrough_cache:1, - od_nonrotational:1; - + od_read_cache_enable:1, + od_read_cache_enable_set:1, + od_writethrough_cache_enable:1, + od_writethrough_cache_enable_set:1, + od_nonrotational:1, + od_nonrotational_set:1; __s64 od_auto_scrub_interval; __u32 od_dirent_journal; diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index c3ec3a4..ea6e996 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -63,7 +63,7 @@ static inline bool osd_use_page_cache(struct osd_device *d) { /* do not use pagecache if write and read caching are disabled */ - if (d->od_writethrough_cache + d->od_read_cache == 0) + if (d->od_writethrough_cache_enable + d->od_read_cache_enable == 0) return false; /* use pagecache by default */ return true; @@ -874,7 +874,7 @@ static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, cache = osd_use_page_cache(osd); while (cache) { if (write) { - if (!osd->od_writethrough_cache) { + if (!osd->od_writethrough_cache_enable) { cache = false; break; } @@ -883,7 +883,7 @@ static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, break; } } else { - if (!osd->od_read_cache) { + if (!osd->od_read_cache_enable) { cache = false; break; } diff --git a/lustre/osd-ldiskfs/osd_lproc.c b/lustre/osd-ldiskfs/osd_lproc.c index 42fa6c3..5edc61e 100644 --- a/lustre/osd-ldiskfs/osd_lproc.c +++ b/lustre/osd-ldiskfs/osd_lproc.c @@ -264,7 +264,7 @@ static ssize_t read_cache_enable_show(struct kobject *kobj, if (unlikely(!osd->od_mnt)) return -EINPROGRESS; - return sprintf(buf, "%u\n", osd->od_read_cache); + return sprintf(buf, "%u\n", osd->od_read_cache_enable); } static ssize_t read_cache_enable_store(struct kobject *kobj, @@ -285,7 +285,9 @@ static ssize_t read_cache_enable_store(struct kobject *kobj, if (rc) return rc; - osd->od_read_cache = !!val; + osd->od_read_cache_enable = !!val; + osd->od_read_cache_enable_set = 1; + return count; } LUSTRE_RW_ATTR(read_cache_enable); @@ -302,7 +304,7 @@ static ssize_t writethrough_cache_enable_show(struct kobject *kobj, if (unlikely(!osd->od_mnt)) return -EINPROGRESS; - return sprintf(buf, "%u\n", osd->od_writethrough_cache); + return sprintf(buf, "%u\n", osd->od_writethrough_cache_enable); } static ssize_t writethrough_cache_enable_store(struct kobject *kobj, @@ -324,7 +326,9 @@ static ssize_t writethrough_cache_enable_store(struct kobject *kobj, if (rc) return rc; - osd->od_writethrough_cache = !!val; + osd->od_writethrough_cache_enable = !!val; + osd->od_writethrough_cache_enable = 1; + return count; } LUSTRE_RW_ATTR(writethrough_cache_enable); @@ -432,7 +436,9 @@ static ssize_t nonrotational_store(struct kobject *kobj, if (rc) return rc; - osd->od_nonrotational = val; + osd->od_nonrotational = !!val; + osd->od_nonrotational_set = 1; + return count; } LUSTRE_RW_ATTR(nonrotational); @@ -607,7 +613,8 @@ static int ldiskfs_osd_oi_scrub_seq_show(struct seq_file *m, void *data) LDEBUGFS_SEQ_FOPS_RO(ldiskfs_osd_oi_scrub); -static int ldiskfs_osd_readcache_seq_show(struct seq_file *m, void *data) +static int ldiskfs_osd_readcache_max_filesize_seq_show(struct seq_file *m, + void *data) { struct osd_device *osd = osd_dt_dev((struct dt_device *)m->private); @@ -620,8 +627,9 @@ static int ldiskfs_osd_readcache_seq_show(struct seq_file *m, void *data) } static ssize_t -ldiskfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, - size_t count, loff_t *off) +ldiskfs_osd_readcache_max_filesize_seq_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *off) { struct seq_file *m = file->private_data; struct dt_device *dt = m->private; @@ -649,8 +657,7 @@ ldiskfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, OSD_MAX_CACHE_SIZE : val; return count; } - -LDEBUGFS_SEQ_FOPS(ldiskfs_osd_readcache); +LDEBUGFS_SEQ_FOPS(ldiskfs_osd_readcache_max_filesize); static int ldiskfs_osd_readcache_max_io_seq_show(struct seq_file *m, void *data) { @@ -861,7 +868,7 @@ struct ldebugfs_vars ldebugfs_osd_obd_vars[] = { { .name = "oi_scrub", .fops = &ldiskfs_osd_oi_scrub_fops }, { .name = "readcache_max_filesize", - .fops = &ldiskfs_osd_readcache_fops }, + .fops = &ldiskfs_osd_readcache_max_filesize_fops}, { .name = "readcache_max_io_mb", .fops = &ldiskfs_osd_readcache_max_io_fops }, { .name = "writethrough_max_io_mb", diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index e97f9ff..8588101 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -351,7 +351,8 @@ struct osd_device { od_is_ost:1, od_in_init:1, od_posix_acl:1, - od_nonrotational:1; + od_nonrotational:1, + od_nonrotational_set:1; unsigned int od_dnsize; int od_index_backup_stop; diff --git a/lustre/osd-zfs/osd_lproc.c b/lustre/osd-zfs/osd_lproc.c index c91ccf4..9727e72 100644 --- a/lustre/osd-zfs/osd_lproc.c +++ b/lustre/osd-zfs/osd_lproc.c @@ -327,7 +327,9 @@ static ssize_t nonrotational_store(struct kobject *kobj, if (rc) return rc; - osd->od_nonrotational = val; + osd->od_nonrotational = !!val; + osd->od_nonrotational_set = 1; + return count; } LUSTRE_RW_ATTR(nonrotational); @@ -368,7 +370,8 @@ ssize_t index_backup_store(struct kobject *kobj, struct attribute *attr, } LUSTRE_RW_ATTR(index_backup); -static int zfs_osd_readcache_seq_show(struct seq_file *m, void *data) +static int zfs_osd_readcache_max_filesize_seq_show(struct seq_file *m, + void *data) { struct osd_device *osd = osd_dt_dev((struct dt_device *)m->private); @@ -381,8 +384,9 @@ static int zfs_osd_readcache_seq_show(struct seq_file *m, void *data) } static ssize_t -zfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, - size_t count, loff_t *off) +zfs_osd_readcache_max_filesize_seq_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *off) { struct seq_file *m = file->private_data; struct dt_device *dt = m->private; @@ -410,7 +414,7 @@ zfs_osd_readcache_seq_write(struct file *file, const char __user *buffer, OSD_MAX_CACHE_SIZE : val; return count; } -LDEBUGFS_SEQ_FOPS(zfs_osd_readcache); +LDEBUGFS_SEQ_FOPS(zfs_osd_readcache_max_filesize); static struct attribute *zfs_attrs[] = { &lustre_attr_fstype.attr, @@ -426,7 +430,7 @@ struct ldebugfs_vars ldebugfs_osd_obd_vars[] = { { .name = "oi_scrub", .fops = &zfs_osd_oi_scrub_fops }, { .name = "readcache_max_filesize", - .fops = &zfs_osd_readcache_fops }, + .fops = &zfs_osd_readcache_max_filesize_fops }, { 0 } }; -- 1.8.3.1