From 01f465e731914ff3348f5efec44585850b4df2b5 Mon Sep 17 00:00:00 2001 From: Patrick Farrell Date: Thu, 13 Jun 2019 16:51:44 -0400 Subject: [PATCH] LU-11963 osd: Add nonrotational flag to statfs It is potentially useful for the MDS and userspace to know whether or not an OST is using non-rotational media. Add a flag to obd_statfs that reflects this. Users can override this parameter in proc. ZFS does not currently make this information available to Lustre, so default to rotational and allow users to override. Lustre-Change: https://review.whamcloud.com/34235 Lustre-Commit: 68635c3d9b3113621b93fd989f1a3f8f064385b9 LU-12396 utils: lfs should not output 'nul' char If lfs prints a nul char, it breaks parsing of the output. Fixes: 68635c3d9b31 ("LU-11963 osd: Add nonrotational flag to statfs") Lustre-Change: https://review.whamcloud.com/35137 Lustre-Commit: fd3958b61c5f1c7ed520f07553b999af5522d8e0 Signed-off-by: Patrick Farrell Change-Id: Iac2b54c5d8cc1eb79cdace764e93578c7b058661 Reviewed-by: Andreas Dilger Reviewed-by: Li Dongyang Signed-off-by: Minh Diep Reviewed-on: https://review.whamcloud.com/35226 Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons --- lustre/include/uapi/linux/lustre/lustre_user.h | 1 + lustre/osd-ldiskfs/osd_handler.c | 8 ++++++ lustre/osd-ldiskfs/osd_internal.h | 4 ++- lustre/osd-ldiskfs/osd_lproc.c | 39 +++++++++++++++++++++++++ lustre/osd-zfs/osd_handler.c | 16 +++++++++-- lustre/osd-zfs/osd_internal.h | 3 +- lustre/osd-zfs/osd_lproc.c | 38 ++++++++++++++++++++++++ lustre/ptlrpc/wiretest.c | 14 +++++++++ lustre/utils/lfs.c | 40 ++++++++++++++++++-------- 9 files changed, 147 insertions(+), 16 deletions(-) diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index b75f0c6..4ddb902 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -131,6 +131,7 @@ enum obd_statfs_state { OS_STATE_ENOSPC = 0x00000020, /**< not enough free space */ OS_STATE_ENOINO = 0x00000040, /**< not enough inodes */ OS_STATE_SUM = 0x00000100, /**< aggregated for all tagrets */ + OS_STATE_NONROT = 0x00000200, /**< non-rotational device */ }; /** filesystem statistics/attributes for target device */ diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 2189a75..0891270 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -2197,6 +2197,9 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, statfs_pack(sfs, ksfs); if (unlikely(sb->s_flags & MS_RDONLY)) sfs->os_state |= OS_STATE_READONLY; + + sfs->os_state |= osd->od_nonrotational ? OS_STATE_NONROT : 0; + if (ldiskfs_has_feature_extents(sb)) sfs->os_maxbytes = sb->s_maxbytes; else @@ -7765,6 +7768,7 @@ static int osd_device_init0(const struct lu_env *env, o->od_read_cache = 1; o->od_writethrough_cache = 1; o->od_readcache_max_filesize = OSD_MAX_CACHE_SIZE; + o->od_auto_scrub_interval = AS_DEFAULT; cplen = strlcpy(o->od_svname, lustre_cfg_string(cfg, 4), @@ -7786,6 +7790,10 @@ static int osd_device_init0(const struct lu_env *env, if (rc != 0) GOTO(out, rc); + /* Can only check block device after mount */ + o->od_nonrotational = + blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev)); + rc = osd_obj_map_init(env, o); if (rc != 0) GOTO(out_mnt, rc); diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index aa8a293..811542f 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -267,7 +267,9 @@ struct osd_device { od_check_ff:1, od_is_ost:1, od_in_init:1, - od_index_in_idif:1; + od_index_in_idif:1, + /* Other flags */ + od_nonrotational:1; __s64 od_auto_scrub_interval; __u32 od_dirent_journal; diff --git a/lustre/osd-ldiskfs/osd_lproc.c b/lustre/osd-ldiskfs/osd_lproc.c index 85b2055..8a484a0 100644 --- a/lustre/osd-ldiskfs/osd_lproc.c +++ b/lustre/osd-ldiskfs/osd_lproc.c @@ -348,6 +348,43 @@ lprocfs_osd_force_sync_seq_write(struct file *file, const char __user *buffer, } LPROC_SEQ_FOPS_WR_ONLY(ldiskfs, osd_force_sync); +static int ldiskfs_osd_nonrotational_seq_show(struct seq_file *m, void *data) +{ + struct dt_device *dt = m->private; + struct osd_device *osd = osd_dt_dev(dt); + + LASSERT(osd); + if (unlikely(!osd->od_mnt)) + return -EINPROGRESS; + + seq_printf(m, "%u\n", osd->od_nonrotational); + return 0; +} + +static ssize_t +ldiskfs_osd_nonrotational_seq_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *off) +{ + struct seq_file *m = file->private_data; + struct dt_device *dt = m->private; + struct osd_device *osd = osd_dt_dev(dt); + bool val; + int rc; + + LASSERT(osd); + if (unlikely(!osd->od_mnt)) + return -EINPROGRESS; + + rc = kstrtobool_from_user(buffer, count, &val); + if (rc) + return rc; + + osd->od_nonrotational = val; + return count; +} +LPROC_SEQ_FOPS(ldiskfs_osd_nonrotational); + static int ldiskfs_osd_pdo_seq_show(struct seq_file *m, void *data) { seq_printf(m, "%s\n", ldiskfs_pdo ? "ON" : "OFF"); @@ -713,6 +750,8 @@ struct lprocfs_vars lprocfs_osd_obd_vars[] = { .fops = &ldiskfs_osd_cache_fops }, { .name = "writethrough_cache_enable", .fops = &ldiskfs_osd_wcache_fops }, + { .name = "nonrotational", + .fops = &ldiskfs_osd_nonrotational_fops }, { .name = "readcache_max_filesize", .fops = &ldiskfs_osd_readcache_fops }, { .name = "index_backup", diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index 07d86b1..a60959a 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -554,16 +554,23 @@ static int osd_objset_statfs(struct osd_device *osd, struct obd_statfs *osfs) int osd_statfs(const struct lu_env *env, struct dt_device *d, struct obd_statfs *osfs) { - int rc; + struct osd_device *osd = osd_dt_dev(d); + int rc; ENTRY; - rc = osd_objset_statfs(osd_dt_dev(d), osfs); + rc = osd_objset_statfs(osd, osfs); if (unlikely(rc != 0)) RETURN(rc); osfs->os_bavail -= min_t(u64, OSD_GRANT_FOR_LOCAL_OIDS / osfs->os_bsize, osfs->os_bavail); + + /* ZFS does not support reporting nonrotional status yet, so return + * flag only if user has set nonrotational. + */ + osfs->os_state |= osd->od_nonrotational ? OS_STATE_NONROT : 0; + RETURN(0); } @@ -1274,6 +1281,11 @@ static int osd_device_init0(const struct lu_env *env, INIT_LIST_HEAD(&o->od_ios_list); o->od_auto_scrub_interval = AS_DEFAULT; + /* ZFS does not support reporting nonrotional status yet, so this flag + * is only set if explicitly set by the user. + */ + o->od_nonrotational = 0; + out: RETURN(rc); } diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index 1cd85c7..e5a51cc 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -347,7 +347,8 @@ struct osd_device { od_xattr_in_sa:1, od_is_ost:1, od_in_init:1, - od_posix_acl:1; + od_posix_acl:1, + od_nonrotational:1; unsigned int od_dnsize; int od_index_backup_stop; diff --git a/lustre/osd-zfs/osd_lproc.c b/lustre/osd-zfs/osd_lproc.c index bedc478..0b7377f 100644 --- a/lustre/osd-zfs/osd_lproc.c +++ b/lustre/osd-zfs/osd_lproc.c @@ -291,6 +291,42 @@ lprocfs_osd_force_sync_seq_write(struct file *file, const char __user *buffer, } LPROC_SEQ_FOPS_WR_ONLY(zfs, osd_force_sync); +static int zfs_osd_nonrotational_seq_show(struct seq_file *m, void *data) +{ + struct dt_device *dt = m->private; + struct osd_device *osd = osd_dt_dev(dt); + + LASSERT(osd); + if (!osd->od_os) + return -EINPROGRESS; + + seq_printf(m, "%u\n", osd->od_nonrotational); + return 0; +} + +static ssize_t +zfs_osd_nonrotational_seq_write(struct file *file, const char __user *buffer, + size_t count, loff_t *off) +{ + struct seq_file *m = file->private_data; + struct dt_device *dt = m->private; + struct osd_device *osd = osd_dt_dev(dt); + int rc; + bool val; + + LASSERT(osd); + if (!osd->od_os) + return -EINPROGRESS; + + rc = kstrtobool_from_user(buffer, count, &val); + if (rc) + return rc; + + osd->od_nonrotational = val; + return count; +} +LPROC_SEQ_FOPS(zfs_osd_nonrotational); + static int zfs_osd_index_backup_seq_show(struct seq_file *m, void *data) { struct osd_device *dev = osd_dt_dev((struct dt_device *)m->private); @@ -394,6 +430,8 @@ struct lprocfs_vars lprocfs_osd_obd_vars[] = { .fops = &zfs_osd_mntdev_fops }, { .name = "force_sync", .fops = &zfs_osd_force_sync_fops }, + { .name = "nonrotational", + .fops = &zfs_osd_nonrotational_fops }, { .name = "index_backup", .fops = &zfs_osd_index_backup_fops }, { .name = "readcache_max_filesize", diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index 21c849b..63cba8b 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -1937,6 +1937,20 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct obd_statfs, os_spare9)); LASSERTF((int)sizeof(((struct obd_statfs *)0)->os_spare9) == 4, "found %lld\n", (long long)(int)sizeof(((struct obd_statfs *)0)->os_spare9)); + LASSERTF(OS_STATE_DEGRADED == 0x1, "found %lld\n", + (long long)OS_STATE_DEGRADED); + LASSERTF(OS_STATE_READONLY == 0x2, "found %lld\n", + (long long)OS_STATE_READONLY); + LASSERTF(OS_STATE_NOPRECREATE == 0x4, "found %lld\n", + (long long)OS_STATE_NOPRECREATE); + LASSERTF(OS_STATE_ENOSPC == 0x20, "found %lld\n", + (long long)OS_STATE_ENOSPC); + LASSERTF(OS_STATE_ENOINO == 0x40, "found %lld\n", + (long long)OS_STATE_ENOINO); + LASSERTF(OS_STATE_SUM == 0x100, "found %lld\n", + (long long)OS_STATE_SUM); + LASSERTF(OS_STATE_NONROT == 0x200, "found %lld\n", + (long long)OS_STATE_NONROT); /* Checks for struct obd_ioobj */ LASSERTF((int)sizeof(struct obd_ioobj) == 24, "found %lld\n", diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 8ada8f8..7f53af5 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -4628,6 +4628,27 @@ static inline int obd_statfs_ratio(const struct obd_statfs *st, bool inodes) return (ratio - (int)ratio) > 0 ? (int)(ratio + 1) : (int)ratio; } +/* This is only used to reflect various problem states for lfs df, so we only + * translate the flags reflecting those states. + */ +static char obd_statfs_state_names[] = { + [OS_STATE_DEGRADED] = 'D', + [OS_STATE_READONLY] = 'R', + [OS_STATE_NOPRECREATE] = 'N', + [OS_STATE_ENOSPC] = 'S', + [OS_STATE_ENOINO] = 'I', +}; + +static char obd_statfs_state2char(int s) +{ + /* Unknown name */ + if (s > ARRAY_SIZE(obd_statfs_state_names)/sizeof(char) || + obd_statfs_state_names[s] == 0) + return '?'; + + return obd_statfs_state_names[s]; +} + static int showdf(char *mntdir, struct obd_statfs *stat, char *uuid, enum mntdf_flags flags, char *type, int index, int rc) @@ -4701,21 +4722,16 @@ static int showdf(char *mntdir, struct obd_statfs *stat, printf("[%s:%d]", type, index); if (stat->os_state) { - /* - * Each character represents the matching - * OS_STATE_* bit. - */ - const char state_names[] = "DRSI"; - __u32 state; - __u32 i; + uint32_t state; + uint32_t i; printf(" "); - for (i = 0, state = stat->os_state; - state && i < sizeof(state_names); i++) { - if (!(state & (1 << i))) + for (i = 0, state = stat->os_state; state != 0; i++) { + uint32_t mask = 1 << i; + if (!(state & mask) || mask == OS_STATE_NONROT) continue; - printf("%c", state_names[i]); - state ^= 1 << i; + printf("%c", obd_statfs_state2char(mask)); + state &= ~mask; } } -- 1.8.3.1