From 68635c3d9b3113621b93fd989f1a3f8f064385b9 Mon Sep 17 00:00:00 2001 From: Patrick Farrell Date: Wed, 27 Feb 2019 16:29:59 -0500 Subject: [PATCH] LU-11963 osd: Add nonrotational flag to statfs It is potentially useful for the MDS and userspace to know whether or not an OST is using non-rotational media. Add a flag to obd_statfs that reflects this. Users can override this parameter in proc. ZFS does not currently make this information available to Lustre, so default to rotational and allow users to override. Signed-off-by: Patrick Farrell Change-Id: Iac2b54c5d8cc1eb79cdace764e93578c7b058661 Reviewed-on: https://review.whamcloud.com/34235 Reviewed-by: Andreas Dilger Reviewed-by: Li Dongyang Tested-by: Jenkins Tested-by: Maloo --- lustre/include/uapi/linux/lustre/lustre_user.h | 1 + lustre/osd-ldiskfs/osd_handler.c | 8 +++++ lustre/osd-ldiskfs/osd_internal.h | 4 ++- lustre/osd-ldiskfs/osd_lproc.c | 38 +++++++++++++++++++++++ lustre/osd-zfs/osd_handler.c | 16 ++++++++-- lustre/osd-zfs/osd_internal.h | 3 +- lustre/osd-zfs/osd_lproc.c | 38 +++++++++++++++++++++++ lustre/ptlrpc/wiretest.c | 14 +++++++++ lustre/utils/lfs.c | 43 +++++++++++++++++++------- 9 files changed, 149 insertions(+), 16 deletions(-) diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 9ead0fc..0bad5c7 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -132,6 +132,7 @@ enum obd_statfs_state { OS_STATE_ENOSPC = 0x00000020, /**< not enough free space */ OS_STATE_ENOINO = 0x00000040, /**< not enough inodes */ OS_STATE_SUM = 0x00000100, /**< aggregated for all tagrets */ + OS_STATE_NONROT = 0x00000200, /**< non-rotational device */ }; /** filesystem statistics/attributes for target device */ diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 6714436..cf8b64d 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -2198,6 +2198,9 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, statfs_pack(sfs, ksfs); if (unlikely(sb->s_flags & MS_RDONLY)) sfs->os_state |= OS_STATE_READONLY; + + sfs->os_state |= osd->od_nonrotational ? OS_STATE_NONROT : 0; + if (ldiskfs_has_feature_extents(sb)) sfs->os_maxbytes = sb->s_maxbytes; else @@ -7757,6 +7760,7 @@ static int osd_device_init0(const struct lu_env *env, o->od_read_cache = 1; o->od_writethrough_cache = 1; o->od_readcache_max_filesize = OSD_MAX_CACHE_SIZE; + o->od_auto_scrub_interval = AS_DEFAULT; cplen = strlcpy(o->od_svname, lustre_cfg_string(cfg, 4), @@ -7778,6 +7782,10 @@ static int osd_device_init0(const struct lu_env *env, if (rc != 0) GOTO(out, rc); + /* Can only check block device after mount */ + o->od_nonrotational = + blk_queue_nonrot(bdev_get_queue(osd_sb(o)->s_bdev)); + rc = osd_obj_map_init(env, o); if (rc != 0) GOTO(out_mnt, rc); diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index accc6bf..bb3bec1 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -267,7 +267,9 @@ struct osd_device { od_check_ff:1, od_is_ost:1, od_in_init:1, - od_index_in_idif:1; + od_index_in_idif:1, + /* Other flags */ + od_nonrotational:1; __s64 od_auto_scrub_interval; __u32 od_dirent_journal; diff --git a/lustre/osd-ldiskfs/osd_lproc.c b/lustre/osd-ldiskfs/osd_lproc.c index 9a2e12a..02cf972 100644 --- a/lustre/osd-ldiskfs/osd_lproc.c +++ b/lustre/osd-ldiskfs/osd_lproc.c @@ -353,6 +353,43 @@ ssize_t force_sync_store(struct kobject *kobj, struct attribute *attr, } LUSTRE_WO_ATTR(force_sync); +static ssize_t nonrotational_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osd_device *osd = osd_dt_dev(dt); + + LASSERT(osd); + if (unlikely(!osd->od_mnt)) + return -EINPROGRESS; + + return sprintf(buf, "%u\n", osd->od_nonrotational); +} + +static ssize_t nonrotational_store(struct kobject *kobj, + struct attribute *attr, const char *buffer, + size_t count) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osd_device *osd = osd_dt_dev(dt); + bool val; + int rc; + + LASSERT(osd); + if (unlikely(!osd->od_mnt)) + return -EINPROGRESS; + + rc = kstrtobool(buffer, &val); + if (rc) + return rc; + + osd->od_nonrotational = val; + return count; +} +LUSTRE_RW_ATTR(nonrotational); + static ssize_t pdo_show(struct kobject *kobj, struct attribute *attr, char *buf) { @@ -668,6 +705,7 @@ static struct attribute *ldiskfs_attrs[] = { &lustre_attr_fstype.attr, &lustre_attr_mntdev.attr, &lustre_attr_force_sync.attr, + &lustre_attr_nonrotational.attr, &lustre_attr_index_backup.attr, &lustre_attr_auto_scrub.attr, &lustre_attr_pdo.attr, diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index 9a84648..edf0a16 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -554,16 +554,23 @@ static int osd_objset_statfs(struct osd_device *osd, struct obd_statfs *osfs) int osd_statfs(const struct lu_env *env, struct dt_device *d, struct obd_statfs *osfs) { - int rc; + struct osd_device *osd = osd_dt_dev(d); + int rc; ENTRY; - rc = osd_objset_statfs(osd_dt_dev(d), osfs); + rc = osd_objset_statfs(osd, osfs); if (unlikely(rc != 0)) RETURN(rc); osfs->os_bavail -= min_t(u64, OSD_GRANT_FOR_LOCAL_OIDS / osfs->os_bsize, osfs->os_bavail); + + /* ZFS does not support reporting nonrotional status yet, so return + * flag only if user has set nonrotational. + */ + osfs->os_state |= osd->od_nonrotational ? OS_STATE_NONROT : 0; + RETURN(0); } @@ -1275,6 +1282,11 @@ static int osd_device_init0(const struct lu_env *env, INIT_LIST_HEAD(&o->od_ios_list); o->od_auto_scrub_interval = AS_DEFAULT; + /* ZFS does not support reporting nonrotional status yet, so this flag + * is only set if explicitly set by the user. + */ + o->od_nonrotational = 0; + out: RETURN(rc); } diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index 1fa1f76..e63e6d6 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -347,7 +347,8 @@ struct osd_device { od_xattr_in_sa:1, od_is_ost:1, od_in_init:1, - od_posix_acl:1; + od_posix_acl:1, + od_nonrotational:1; unsigned int od_dnsize; int od_index_backup_stop; diff --git a/lustre/osd-zfs/osd_lproc.c b/lustre/osd-zfs/osd_lproc.c index b3360ad..e84ed46 100644 --- a/lustre/osd-zfs/osd_lproc.c +++ b/lustre/osd-zfs/osd_lproc.c @@ -295,6 +295,43 @@ ssize_t force_sync_store(struct kobject *kobj, struct attribute *attr, } LUSTRE_WO_ATTR(force_sync); +static ssize_t nonrotational_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osd_device *osd = osd_dt_dev(dt); + + LASSERT(osd); + if (!osd->od_os) + return -EINPROGRESS; + + return sprintf(buf, "%u\n", osd->od_nonrotational); +} + +static ssize_t nonrotational_store(struct kobject *kobj, + struct attribute *attr, const char *buffer, + size_t count) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osd_device *osd = osd_dt_dev(dt); + bool val; + int rc; + + LASSERT(osd); + if (!osd->od_os) + return -EINPROGRESS; + + rc = kstrtobool(buffer, &val); + if (rc) + return rc; + + osd->od_nonrotational = val; + return count; +} +LUSTRE_RW_ATTR(nonrotational); + static ssize_t index_backup_show(struct kobject *kobj, struct attribute *attr, char *buf) { @@ -373,6 +410,7 @@ static struct attribute *zfs_attrs[] = { &lustre_attr_fstype.attr, &lustre_attr_mntdev.attr, &lustre_attr_force_sync.attr, + &lustre_attr_nonrotational.attr, &lustre_attr_index_backup.attr, &lustre_attr_auto_scrub.attr, NULL, diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index d4ef2fe..7c949f3 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -1945,6 +1945,20 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct obd_statfs, os_spare9)); LASSERTF((int)sizeof(((struct obd_statfs *)0)->os_spare9) == 4, "found %lld\n", (long long)(int)sizeof(((struct obd_statfs *)0)->os_spare9)); + LASSERTF(OS_STATE_DEGRADED == 0x1, "found %lld\n", + (long long)OS_STATE_DEGRADED); + LASSERTF(OS_STATE_READONLY == 0x2, "found %lld\n", + (long long)OS_STATE_READONLY); + LASSERTF(OS_STATE_NOPRECREATE == 0x4, "found %lld\n", + (long long)OS_STATE_NOPRECREATE); + LASSERTF(OS_STATE_ENOSPC == 0x20, "found %lld\n", + (long long)OS_STATE_ENOSPC); + LASSERTF(OS_STATE_ENOINO == 0x40, "found %lld\n", + (long long)OS_STATE_ENOINO); + LASSERTF(OS_STATE_SUM == 0x100, "found %lld\n", + (long long)OS_STATE_SUM); + LASSERTF(OS_STATE_NONROT == 0x200, "found %lld\n", + (long long)OS_STATE_NONROT); /* Checks for struct obd_ioobj */ LASSERTF((int)sizeof(struct obd_ioobj) == 24, "found %lld\n", diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 8b68b44..701868a 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -4884,6 +4884,30 @@ static inline int obd_statfs_ratio(const struct obd_statfs *st, bool inodes) return (ratio - (int)ratio) > 0 ? (int)(ratio + 1) : (int)ratio; } +/* This is only used to reflect various problem states for lfs df, so we only + * translate the flags reflecting those states. + */ +static char obd_statfs_state_names[] = { + [OS_STATE_DEGRADED] = 'D', + [OS_STATE_READONLY] = 'R', + [OS_STATE_NOPRECREATE] = 'N', + [OS_STATE_ENOSPC] = 'S', + [OS_STATE_ENOINO] = 'I', +}; + +static char obd_statfs_state2char(int s) +{ + /* Not an error state, do not print here */ + if (s == OS_STATE_NONROT) + return '\0'; + /* Unknown name */ + if (s > ARRAY_SIZE(obd_statfs_state_names)/sizeof(char) || + obd_statfs_state_names[s] == 0) + return '?'; + + return obd_statfs_state_names[s]; +} + static int showdf(char *mntdir, struct obd_statfs *stat, char *uuid, enum mntdf_flags flags, char *type, int index, int rc) @@ -4957,21 +4981,16 @@ static int showdf(char *mntdir, struct obd_statfs *stat, printf("[%s:%d]", type, index); if (stat->os_state) { - /* - * Each character represents the matching - * OS_STATE_* bit. - */ - const char state_names[] = "DRSI"; - __u32 state; - __u32 i; + uint32_t state; + uint32_t i; printf(" "); - for (i = 0, state = stat->os_state; - state && i < sizeof(state_names); i++) { - if (!(state & (1 << i))) + for (i = 0, state = stat->os_state; state != 0; i++) { + uint32_t mask = 1 << i; + if (!(state & mask)) continue; - printf("%c", state_names[i]); - state ^= 1 << i; + printf("%c", obd_statfs_state2char(mask)); + state &= ~mask; } } -- 1.8.3.1