From a046e879fcadd601c9a19fd906f82ecbd2d4efd5 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Fri, 3 Nov 2017 09:22:12 +0800 Subject: [PATCH 1/1] LU-7991 quota: project quota against ZFS backend This patch enables the project quota feature against ZFS backend. The project quota patch for ZFS itself will be landed via https://github.com/zfsonlinux/zfs/pull/6290. Such patch may be released via zfs-0.7.x (x >= 5). This patch also cleanup the quota logic for ZFS backend and sanity-quota.sh Signed-off-by: Fan Yong Change-Id: Ic488efe19a6e34a3615811f79fb9ea7954ad9263 Reviewed-on: https://review.whamcloud.com/27093 Reviewed-by: Andreas Dilger Reviewed-by: Wang Shilong Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- config/lustre-build-zfs.m4 | 4 +- lustre/include/lustre_quota.h | 7 ++ lustre/include/uapi/linux/lustre/lustre_idl.h | 1 + lustre/osd-ldiskfs/osd_internal.h | 7 -- lustre/osd-ldiskfs/osd_quota.c | 20 ++-- lustre/osd-zfs/osd_handler.c | 16 ++++ lustre/osd-zfs/osd_internal.h | 31 +++++-- lustre/osd-zfs/osd_io.c | 23 +++-- lustre/osd-zfs/osd_object.c | 129 +++++++++++++++++++++++--- lustre/osd-zfs/osd_oi.c | 4 +- lustre/osd-zfs/osd_quota.c | 51 ++++++---- lustre/osd-zfs/osd_xattr.c | 19 +++- lustre/tests/sanity-quota.sh | 41 ++++++-- 13 files changed, 272 insertions(+), 81 deletions(-) diff --git a/config/lustre-build-zfs.m4 b/config/lustre-build-zfs.m4 index 9e39c80..297e790 100644 --- a/config/lustre-build-zfs.m4 +++ b/config/lustre-build-zfs.m4 @@ -523,10 +523,10 @@ your distribution. dnl # ZFS 0.7.0 feature: SPA_FEATURE_USEROBJ_ACCOUNTING dnl # LB_CHECK_COMPILE([if zfs has native dnode accounting supported], - dmu_objset_userobjspace_upgrade, [ + dmu_objset_id_quota_upgrade, [ #include ],[ - dmu_objset_userobjspace_upgrade(NULL); + dmu_objset_id_quota_upgrade(NULL); ],[ AC_DEFINE(HAVE_DMU_USEROBJ_ACCOUNTING, 1, [Have native dnode accounting in ZFS]) diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h index 0b41879..8cb25d2 100644 --- a/lustre/include/lustre_quota.h +++ b/lustre/include/lustre_quota.h @@ -59,6 +59,13 @@ union lquota_rec { struct lquota_acct_rec lqr_acct_rec; }; +/* flags for inode/block quota accounting */ +enum osd_qid_declare_flags { + OSD_QID_INODE = 1 << 0, + OSD_QID_BLK = 1 << 1, + OSD_QID_FORCE = 1 << 2, +}; + /* Index features supported by the global index objects * Only used for migration purpose and should be removed once on-disk migration * is no longer needed */ diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index f497574..a845ecf 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -1366,6 +1366,7 @@ union lquota_id { struct lu_fid qid_fid; /* FID for per-directory quota */ __u64 qid_uid; /* user identifier */ __u64 qid_gid; /* group identifier */ + __u64 qid_projid; /* project identifier */ }; /* quotactl management */ diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index 6eb0909..c0bf3e8 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -628,13 +628,6 @@ struct osd_thread_info { unsigned int oti_declare_ops_used[OSD_OT_MAX]; }; -/* flags for inode/block quota accounting */ -enum osd_qid_declare_flags { - OSD_QID_INODE = 1 << 0, - OSD_QID_BLK = 1 << 1, - OSD_QID_FORCE = 1 << 2, -}; - extern int ldiskfs_pdo; static inline int __osd_xattr_get(struct inode *inode, struct dentry *dentry, diff --git a/lustre/osd-ldiskfs/osd_quota.c b/lustre/osd-ldiskfs/osd_quota.c index 2345501..2ff5770 100644 --- a/lustre/osd-ldiskfs/osd_quota.c +++ b/lustre/osd-ldiskfs/osd_quota.c @@ -625,8 +625,8 @@ int osd_declare_inode_qid(const struct lu_env *env, qid_t uid, qid_t gid, { struct osd_thread_info *info = osd_oti_get(env); struct lquota_id_info *qi = &info->oti_qi; - int rcu, rcg, rcp; /* user & group & project rc */ - int force = osd_qid_declare_flags & OSD_QID_FORCE; + int rcu, rcg, rcp = 0; /* user & group & project rc */ + bool force = !!(osd_qid_declare_flags & OSD_QID_FORCE); ENTRY; /* let's start with user quota */ @@ -655,24 +655,20 @@ int osd_declare_inode_qid(const struct lu_env *env, qid_t uid, qid_t gid, if (force && (rcg == -EDQUOT || rcg == -EINPROGRESS)) /* as before, ignore EDQUOT & EINPROGRESS for root */ rcg = 0; + +#ifdef HAVE_PROJECT_QUOTA if (rcg && (rcg != -EDQUOT || flags == NULL)) RETURN(rcg); /* and now project quota */ - qi->lqi_id.qid_gid = projid; - qi->lqi_type = PRJQUOTA; + qi->lqi_id.qid_projid = projid; + qi->lqi_type = PRJQUOTA; rcp = osd_declare_qid(env, oh, qi, obj, true, flags); if (force && (rcp == -EDQUOT || rcp == -EINPROGRESS)) /* as before, ignore EDQUOT & EINPROGRESS for root */ rcp = 0; +#endif - if (rcu) - RETURN(rcu); - if (rcg) - RETURN(rcg); - if (rcp) - RETURN(rcp); - - RETURN(0); + RETURN(rcu ? rcu : (rcg ? rcg : rcp)); } diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index fc6c701..5adcd1a 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -1078,6 +1078,15 @@ static int osd_mount(const struct lu_env *env, if (rc) GOTO(err, rc); +#ifdef ZFS_PROJINHERIT + if (dmu_objset_projectquota_enabled(o->od_os)) { + rc = __osd_obj2dnode(o->od_os, DMU_PROJECTUSED_OBJECT, + &o->od_projectused_dn); + if (rc && rc != -ENOENT) + GOTO(err, rc); + } +#endif + /* 1. initialize oi before any file create or file open */ rc = osd_oi_init(env, o); if (rc) @@ -1158,6 +1167,13 @@ static void osd_umount(const struct lu_env *env, struct osd_device *o) o->od_groupused_dn = NULL; } +#ifdef ZFS_PROJINHERIT + if (o->od_projectused_dn) { + osd_dnode_rele(o->od_projectused_dn); + o->od_projectused_dn = NULL; + } +#endif + if (o->od_os != NULL) { if (!o->od_dt_dev.dd_rdonly) /* force a txg sync to get all commit callbacks */ diff --git a/lustre/osd-zfs/osd_internal.h b/lustre/osd-zfs/osd_internal.h index 503249e..4e19b20 100644 --- a/lustre/osd-zfs/osd_internal.h +++ b/lustre/osd-zfs/osd_internal.h @@ -155,6 +155,9 @@ struct osa_attr { uint64_t mode; uint64_t gid; uint64_t uid; +#ifdef ZFS_PROJINHERIT + uint64_t projid; +#endif uint64_t nlink; uint64_t rdev; uint64_t flags; @@ -176,8 +179,8 @@ struct osd_idmap_cache { oic_remote:1; /* FID isn't local */ }; -/* max.number of regular attrubites the callers may ask for */ -#define OSD_MAX_IN_BULK 13 +/* max.number of regular attributes the callers may ask for */ +# define OSD_MAX_IN_BULK (sizeof(struct osa_attr)/sizeof(uint64_t)) struct osd_thread_info { const struct lu_env *oti_env; @@ -298,8 +301,11 @@ struct osd_device { int od_connects; struct lu_site od_site; - dnode_t *od_groupused_dn; - dnode_t *od_userused_dn; + dnode_t *od_groupused_dn; + dnode_t *od_userused_dn; +#ifdef ZFS_PROJINHERIT + dnode_t *od_projectused_dn; +#endif /* quota slave instance */ struct qsd_instance *od_quota_slave; @@ -361,6 +367,9 @@ struct osd_object { __u32 oo_destroyed:1, oo_late_xattr:1, +#ifdef ZFS_PROJINHERIT + oo_with_projid:1, +#endif oo_late_attr_set:1; /* the i_flags in LMA */ @@ -382,9 +391,9 @@ extern const struct dt_index_operations osd_acct_index_ops; extern struct lu_device_operations osd_lu_ops; extern struct dt_index_operations osd_dir_ops; int osd_declare_quota(const struct lu_env *env, struct osd_device *osd, - qid_t uid, qid_t gid, long long space, - struct osd_thandle *oh, bool is_blk, int *flags, - bool force); + qid_t uid, qid_t gid, qid_t projid, long long space, + struct osd_thandle *oh, int *flags, + enum osd_qid_declare_flags osd_qid_declare_flags); uint64_t osd_objs_count_estimate(uint64_t refdbytes, uint64_t usedobjs, uint64_t nrblocks, uint64_t est_maxblockshift); int osd_unlinked_object_free(const struct lu_env *env, struct osd_device *osd, @@ -507,7 +516,7 @@ int __osd_zap_create(const struct lu_env *env, struct osd_device *osd, int __osd_object_create(const struct lu_env *env, struct osd_object *obj, dnode_t **dnp, dmu_tx_t *tx, struct lu_attr *la); int __osd_attr_init(const struct lu_env *env, struct osd_device *osd, - sa_handle_t *sa_hdl, dmu_tx_t *tx, + struct osd_object *obj, sa_handle_t *sa_hdl, dmu_tx_t *tx, struct lu_attr *la, uint64_t parent, nvlist_t *); /* osd_oi.c */ @@ -612,6 +621,9 @@ static inline uint64_t attrs_fs2zfs(const uint32_t flags) { return (flags & LUSTRE_APPEND_FL ? ZFS_APPENDONLY : 0) | (flags & LUSTRE_NODUMP_FL ? ZFS_NODUMP : 0) | +#ifdef ZFS_PROJINHERIT + (flags & LUSTRE_PROJINHERIT_FL ? ZFS_PROJINHERIT : 0) | +#endif (flags & LUSTRE_IMMUTABLE_FL ? ZFS_IMMUTABLE : 0); } @@ -619,6 +631,9 @@ static inline uint32_t attrs_zfs2fs(const uint64_t flags) { return (flags & ZFS_APPENDONLY ? LUSTRE_APPEND_FL : 0) | (flags & ZFS_NODUMP ? LUSTRE_NODUMP_FL : 0) | +#ifdef ZFS_PROJINHERIT + (flags & ZFS_PROJINHERIT ? LUSTRE_PROJINHERIT_FL : 0) | +#endif (flags & ZFS_IMMUTABLE ? LUSTRE_IMMUTABLE_FL : 0); } diff --git a/lustre/osd-zfs/osd_io.c b/lustre/osd-zfs/osd_io.c index 856b2e1..afe987c 100644 --- a/lustre/osd-zfs/osd_io.c +++ b/lustre/osd-zfs/osd_io.c @@ -44,6 +44,7 @@ #include #include #include +#include #include "osd_internal.h" @@ -180,8 +181,8 @@ static ssize_t osd_declare_write(const struct lu_env *env, struct dt_object *dt, * as llog or last_rcvd files. We needn't enforce quota on those * objects, so always set the lqi_space as 0. */ RETURN(osd_declare_quota(env, osd, obj->oo_attr.la_uid, - obj->oo_attr.la_gid, 0, oh, true, NULL, - false)); + obj->oo_attr.la_gid, obj->oo_attr.la_projid, + 0, oh, NULL, OSD_QID_BLK)); } static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, @@ -578,10 +579,11 @@ static int osd_declare_write_commit(const struct lu_env *env, uint32_t size = 0; uint32_t blksz = obj->oo_dn->dn_datablksz; int i, rc, flags = 0; - bool ignore_quota = false, synced = false; + bool synced = false; long long space = 0; struct page *last_page = NULL; unsigned long discont_pages = 0; + enum osd_qid_declare_flags declare_flags = OSD_QID_BLK; ENTRY; LASSERT(dt_object_exists(dt)); @@ -614,7 +616,8 @@ static int osd_declare_write_commit(const struct lu_env *env, if ((lnb[i].lnb_flags & OBD_BRW_NOQUOTA) || (lnb[i].lnb_flags & (OBD_BRW_FROM_GRANT | OBD_BRW_SYNC)) == OBD_BRW_FROM_GRANT) - ignore_quota = true; + declare_flags |= OSD_QID_FORCE; + if (size == 0) { /* first valid lnb */ offset = lnb[i].lnb_file_offset; @@ -660,8 +663,8 @@ static int osd_declare_write_commit(const struct lu_env *env, retry: /* acquire quota space if needed */ rc = osd_declare_quota(env, osd, obj->oo_attr.la_uid, - obj->oo_attr.la_gid, space, oh, true, &flags, - ignore_quota); + obj->oo_attr.la_gid, obj->oo_attr.la_projid, + space, oh, &flags, declare_flags); if (!synced && rc == -EDQUOT && (flags & QUOTA_FL_SYNC) != 0) { dt_sync(env, th->th_dev); @@ -678,6 +681,10 @@ retry: lnb[0].lnb_flags |= OBD_BRW_OVER_USRQUOTA; if (flags & QUOTA_FL_OVER_GRPQUOTA) lnb[0].lnb_flags |= OBD_BRW_OVER_GRPQUOTA; +#ifdef ZFS_PROJINHERIT + if (flags & QUOTA_FL_OVER_PRJQUOTA) + lnb[0].lnb_flags |= OBD_BRW_OVER_PRJQUOTA; +#endif RETURN(rc); } @@ -1002,8 +1009,8 @@ static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, } RETURN(osd_declare_quota(env, osd, obj->oo_attr.la_uid, - obj->oo_attr.la_gid, 0, oh, true, NULL, - false)); + obj->oo_attr.la_gid, obj->oo_attr.la_projid, + 0, oh, NULL, OSD_QID_BLK)); } static int osd_ladvise(const struct lu_env *env, struct dt_object *dt, diff --git a/lustre/osd-zfs/osd_object.c b/lustre/osd-zfs/osd_object.c index 6eb175e..e9b127d 100644 --- a/lustre/osd-zfs/osd_object.c +++ b/lustre/osd-zfs/osd_object.c @@ -214,6 +214,25 @@ int __osd_object_attr_get(const struct lu_env *env, struct osd_device *o, if (rc) GOTO(out_sa, rc); +#ifdef ZFS_PROJINHERIT + if (o->od_projectused_dn && osa->flags & ZFS_PROJID) { + rc = -sa_lookup(obj->oo_sa_hdl, SA_ZPL_PROJID(o), + &osa->projid, 8); + if (rc) + GOTO(out_sa, rc); + + la->la_projid = osa->projid; + la->la_valid |= LA_PROJID; + obj->oo_with_projid = 1; + } else { + la->la_projid = ZFS_DEFAULT_PROJID; + la->la_valid &= ~LA_PROJID; + } +#else + la->la_projid = 0; + la->la_valid &= ~LA_PROJID; +#endif + la->la_atime = osa->atime[0]; la->la_mtime = osa->mtime[0]; la->la_ctime = osa->ctime[0]; @@ -395,6 +414,11 @@ static dnode_t *osd_quota_fid2dmu(const struct osd_device *osd, case ACCT_GROUP_OID: dn = osd->od_groupused_dn; break; +#ifdef ZFS_PROJINHERIT + case ACCT_PROJECT_OID: + dn = osd->od_projectused_dn; + break; +#endif default: break; } @@ -548,13 +572,15 @@ static int osd_declare_destroy(const struct lu_env *env, struct dt_object *dt, /* one less inode */ rc = osd_declare_quota(env, osd, obj->oo_attr.la_uid, - obj->oo_attr.la_gid, -1, oh, false, NULL, false); + obj->oo_attr.la_gid, obj->oo_attr.la_projid, + -1, oh, NULL, OSD_QID_INODE); if (rc) RETURN(rc); /* data to be truncated */ rc = osd_declare_quota(env, osd, obj->oo_attr.la_uid, - obj->oo_attr.la_gid, 0, oh, true, NULL, false); + obj->oo_attr.la_gid, obj->oo_attr.la_projid, + 0, oh, NULL, OSD_QID_BLK); if (rc) RETURN(rc); @@ -905,7 +931,7 @@ static int osd_declare_attr_set(const struct lu_env *env, * anything else */ } - if (attr && (attr->la_valid & (LA_UID | LA_GID))) { + if (attr && (attr->la_valid & (LA_UID | LA_GID | LA_PROJID))) { sa_object_size(obj->oo_sa_hdl, &blksize, &bspace); bspace = toqb(bspace * blksize); } @@ -932,7 +958,38 @@ static int osd_declare_attr_set(const struct lu_env *env, GOTO(out, rc); } } - +#ifdef ZFS_PROJINHERIT + if (attr && attr->la_valid & LA_PROJID) { + if (!osd->od_projectused_dn) + GOTO(out, rc = -EOPNOTSUPP); + + /* Usually, if project quota is upgradable for the device, + * then the upgrade will be done before or when mount the + * device. So when we come here, this project should have + * project ID attribute already (that is zero by default). + * Otherwise, there was something wrong during the former + * upgrade, let's return failure to report that. + * + * Please note that, different from other attributes, you + * can NOT simply set the project ID attribute under such + * case, because adding (NOT change) project ID attribute + * needs to change the object's attribute layout to match + * zfs backend quota accounting requirement. */ + if (unlikely(!obj->oo_with_projid)) + GOTO(out, rc = -ENXIO); + + /* quota enforcement for project */ + if (attr->la_projid != obj->oo_attr.la_projid) { + rc = qsd_transfer(env, osd->od_quota_slave, + &oh->ot_quota_trans, PRJQUOTA, + obj->oo_attr.la_projid, + attr->la_projid, bspace, + &info->oti_qi); + if (rc) + GOTO(out, rc); + } + } +#endif out: up_read(&obj->oo_guard); RETURN(rc); @@ -1017,13 +1074,30 @@ static int osd_attr_set(const struct lu_env *env, struct dt_object *dt, if (rc < 0) { CWARN("%s: failed to set LMA flags: rc = %d\n", osd->od_svname, rc); - RETURN(rc); + GOTO(out, rc); } } } write_lock(&obj->oo_attr_lock); cnt = 0; + + if (valid & LA_PROJID) { +#ifdef ZFS_PROJINHERIT + /* osd_declare_attr_set() must be called firstly. + * If osd::od_projectused_dn is not set, then we + * can not arrive at here. */ + LASSERT(osd->od_projectused_dn); + LASSERT(obj->oo_with_projid); + + osa->projid = obj->oo_attr.la_projid = la->la_projid; + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_PROJID(osd), NULL, + &osa->projid, 8); +#else + valid &= ~LA_PROJID; +#endif + } + if (valid & LA_ATIME) { osa->atime[0] = obj->oo_attr.la_atime = la->la_atime; SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_ATIME(osd), NULL, @@ -1067,6 +1141,10 @@ static int osd_attr_set(const struct lu_env *env, struct dt_object *dt, /* many flags are not supported by zfs, so ensure a good cached * copy */ obj->oo_attr.la_flags = attrs_zfs2fs(osa->flags); +#ifdef ZFS_PROJINHERIT + if (obj->oo_with_projid) + osa->flags |= ZFS_PROJID; +#endif SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_FLAGS(osd), NULL, &osa->flags, 8); } @@ -1180,14 +1258,14 @@ static int osd_declare_create(const struct lu_env *env, struct dt_object *dt, /* will help to find FID->ino mapping at dt_insert() */ osd_idc_find_and_init(env, osd, obj); - rc = osd_declare_quota(env, osd, attr->la_uid, attr->la_gid, 1, oh, - false, NULL, false); + rc = osd_declare_quota(env, osd, attr->la_uid, attr->la_gid, + attr->la_projid, 1, oh, NULL, OSD_QID_INODE); RETURN(rc); } int __osd_attr_init(const struct lu_env *env, struct osd_device *osd, - sa_handle_t *sa_hdl, dmu_tx_t *tx, + struct osd_object *obj, sa_handle_t *sa_hdl, dmu_tx_t *tx, struct lu_attr *la, uint64_t parent, nvlist_t *xattr) { @@ -1216,16 +1294,32 @@ int __osd_attr_init(const struct lu_env *env, struct osd_device *osd, osa->gid = la->la_gid; osa->rdev = la->la_rdev; osa->nlink = la->la_nlink; - osa->flags = attrs_fs2zfs(la->la_flags); + if (la->la_valid & LA_FLAGS) + osa->flags = attrs_fs2zfs(la->la_flags); + else + osa->flags = 0; osa->size = la->la_size; +#ifdef ZFS_PROJINHERIT + if (osd->od_projectused_dn) { + if (la->la_valid & LA_PROJID) + osa->projid = la->la_projid; + else + osa->projid = ZFS_DEFAULT_PROJID; + osa->flags |= ZFS_PROJID; + if (obj) + obj->oo_with_projid = 1; + } else { + osa->flags &= ~ZFS_PROJID; + } +#endif /* * we need to create all SA below upon object create. * * XXX The attribute order matters since the accounting callback relies * on static offsets (i.e. SA_*_OFFSET, see zfs_space_delta_cb()) to - * look up the UID/GID attributes. Moreover, the callback does not seem - * to support the spill block. + * look up the UID/GID/PROJID attributes. Moreover, the callback does + * not seem to support the spill block. * We define attributes in the same order as SA_*_OFFSET in order to * work around the problem. See ORI-610. */ @@ -1242,6 +1336,11 @@ int __osd_attr_init(const struct lu_env *env, struct osd_device *osd, SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(osd), NULL, osa->ctime, 16); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CRTIME(osd), NULL, crtime, 16); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_LINKS(osd), NULL, &osa->nlink, 8); +#ifdef ZFS_PROJINHERIT + if (osd->od_projectused_dn) + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_PROJID(osd), NULL, + &osa->projid, 8); +#endif SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_RDEV(osd), NULL, &osa->rdev, 8); LASSERT(cnt <= ARRAY_SIZE(osd_oti_get(env)->oti_attr_bulk)); @@ -1581,6 +1680,14 @@ static int osd_create(const struct lu_env *env, struct dt_object *dt, obj->oo_attr = *attr; obj->oo_attr.la_valid |= LA_SIZE | LA_NLINK | LA_TYPE; +#ifdef ZFS_PROJINHERIT + if (osd->od_projectused_dn) { + if (!(obj->oo_attr.la_valid & LA_PROJID)) + obj->oo_attr.la_projid = ZFS_DEFAULT_PROJID; + obj->oo_with_projid = 1; + } +#endif + dn = osd_create_type_f(dof->dof_type)(env, obj, &obj->oo_attr, oh); if (IS_ERR(dn)) { rc = PTR_ERR(dn); diff --git a/lustre/osd-zfs/osd_oi.c b/lustre/osd-zfs/osd_oi.c index e350ba8..010e023 100644 --- a/lustre/osd-zfs/osd_oi.c +++ b/lustre/osd-zfs/osd_oi.c @@ -173,10 +173,10 @@ osd_oi_create(const struct lu_env *env, struct osd_device *o, rc = -sa_handle_get(o->od_os, oid, NULL, SA_HDL_PRIVATE, &sa_hdl); if (rc) goto commit; + memset(la, 0, sizeof(*la)); la->la_valid = LA_MODE | LA_UID | LA_GID; la->la_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; - la->la_uid = la->la_gid = 0; - rc = __osd_attr_init(env, o, sa_hdl, tx, la, parent, NULL); + rc = __osd_attr_init(env, o, NULL, sa_hdl, tx, la, parent, NULL); sa_handle_destroy(sa_hdl); if (rc) goto commit; diff --git a/lustre/osd-zfs/osd_quota.c b/lustre/osd-zfs/osd_quota.c index 20da34e..45b68ff 100644 --- a/lustre/osd-zfs/osd_quota.c +++ b/lustre/osd-zfs/osd_quota.c @@ -33,7 +33,7 @@ /** * Helper function to estimate the number of inodes in use for the given - * uid/gid from the block usage + * uid/gid/projid from the block usage */ static uint64_t osd_objset_user_iused(struct osd_device *osd, uint64_t uidbytes) { @@ -60,7 +60,7 @@ static uint64_t osd_objset_user_iused(struct osd_device *osd, uint64_t uidbytes) */ /** - * Return space usage consumed by a given uid or gid. + * Return space usage consumed by a given uid or gid or projid. * Block usage is accurrate since it is maintained by DMU itself. * However, DMU does not provide inode accounting, so the #inodes in use * is estimated from the block usage and statfs information. @@ -91,7 +91,7 @@ static int osd_acct_index_lookup(const struct lu_env *env, rec->bspace = rec->ispace = 0; - /* convert the 64-bit uid/gid into a string */ + /* convert the 64-bit uid/gid/projid into a string */ snprintf(buf, buflen, "%llx", *((__u64 *)dtkey)); if (unlikely(!dn)) { CDEBUG(D_QUOTA, "%s: miss accounting obj for %s\n", @@ -106,7 +106,7 @@ static int osd_acct_index_lookup(const struct lu_env *env, rc = osd_zap_lookup(osd, dn->dn_object, dn, buf, sizeof(uint64_t), 1, &rec->bspace); if (rc == -ENOENT) { - /* user/group has not created anything yet */ + /* user/group/project has not created anything yet */ CDEBUG(D_QUOTA, "%s: id %s not found in DMU accounting ZAP\n", osd->od_svname, buf); /* -ENOENT is normal case, convert it as 1. */ @@ -446,7 +446,7 @@ static int osd_it_acct_load(const struct lu_env *env, * move to the first valid record. * * \param di - osd iterator - * \param key - uid or gid + * \param key - uid or gid or projid * * \retval +ve - di points to exact matched key * \retval 0 - di points to the first valid record @@ -502,26 +502,27 @@ const struct dt_index_operations osd_acct_index_ops = { * \param osd - is the osd_device * \param uid - user id of the inode * \param gid - group id of the inode + * \param projid - project id of the inode * \param space - how many blocks/inodes will be consumed/released * \param oh - osd transaction handle - * \param is_blk - block quota or inode quota? * \param flags - if the operation is write, return no user quota, no * group quota, or sync commit flags to the caller - * \param force - set to 1 when changes are performed by root user and thus - * can't failed with EDQUOT + * \param osd_qid_declare_flags - indicate this is a inode/block accounting + * and whether changes are performed by root user * * \retval 0 - success * \retval -ve - failure */ int osd_declare_quota(const struct lu_env *env, struct osd_device *osd, - qid_t uid, qid_t gid, long long space, - struct osd_thandle *oh, bool is_blk, int *flags, - bool force) + qid_t uid, qid_t gid, qid_t projid, long long space, + struct osd_thandle *oh, int *flags, + enum osd_qid_declare_flags osd_qid_declare_flags) { - struct osd_thread_info *info = osd_oti_get(env); - struct lquota_id_info *qi = &info->oti_qi; - struct qsd_instance *qsd = osd->od_quota_slave; - int rcu, rcg; /* user & group rc */ + struct osd_thread_info *info = osd_oti_get(env); + struct lquota_id_info *qi = &info->oti_qi; + struct qsd_instance *qsd = osd->od_quota_slave; + int rcu, rcg, rcp = 0; /* user & group & project rc */ + bool force = !!(osd_qid_declare_flags & OSD_QID_FORCE); ENTRY; if (unlikely(qsd == NULL)) @@ -532,9 +533,8 @@ int osd_declare_quota(const struct lu_env *env, struct osd_device *osd, qi->lqi_id.qid_uid = uid; qi->lqi_type = USRQUOTA; qi->lqi_space = space; - qi->lqi_is_blk = is_blk; + qi->lqi_is_blk = !!(osd_qid_declare_flags & OSD_QID_BLK); rcu = qsd_op_begin(env, qsd, &oh->ot_quota_trans, qi, flags); - if (force && (rcu == -EDQUOT || rcu == -EINPROGRESS)) /* ignore EDQUOT & EINPROGRESS when changes are done by root */ rcu = 0; @@ -550,10 +550,23 @@ int osd_declare_quota(const struct lu_env *env, struct osd_device *osd, qi->lqi_id.qid_gid = gid; qi->lqi_type = GRPQUOTA; rcg = qsd_op_begin(env, qsd, &oh->ot_quota_trans, qi, flags); - if (force && (rcg == -EDQUOT || rcg == -EINPROGRESS)) /* as before, ignore EDQUOT & EINPROGRESS for root */ rcg = 0; - RETURN(rcu ? rcu : rcg); +#ifdef ZFS_PROJINHERIT + if (rcg && (rcg != -EDQUOT || flags == NULL)) + RETURN(rcg); + + /* for project quota */ + if (osd->od_projectused_dn) { + qi->lqi_id.qid_projid = projid; + qi->lqi_type = PRJQUOTA; + rcp = qsd_op_begin(env, qsd, &oh->ot_quota_trans, qi, flags); + if (force && (rcp == -EDQUOT || rcp == -EINPROGRESS)) + rcp = 0; + } +#endif + + RETURN(rcu ? rcu : (rcg ? rcg : rcp)); } diff --git a/lustre/osd-zfs/osd_xattr.c b/lustre/osd-zfs/osd_xattr.c index cce0e99..aaa94ab 100644 --- a/lustre/osd-zfs/osd_xattr.c +++ b/lustre/osd-zfs/osd_xattr.c @@ -375,6 +375,16 @@ int __osd_sa_attr_init(const struct lu_env *env, struct osd_object *obj, osa->nlink = obj->oo_attr.la_nlink; osa->flags = attrs_fs2zfs(obj->oo_attr.la_flags); osa->size = obj->oo_attr.la_size; +#ifdef ZFS_PROJINHERIT + if (osd->od_projectused_dn) { + if (obj->oo_attr.la_valid & LA_PROJID) + osa->projid = obj->oo_attr.la_projid; + else + osa->projid = ZFS_DEFAULT_PROJID; + osa->flags |= ZFS_PROJID; + obj->oo_with_projid = 1; + } +#endif cnt = 0; SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MODE(osd), NULL, &osa->mode, 8); @@ -382,15 +392,20 @@ int __osd_sa_attr_init(const struct lu_env *env, struct osd_object *obj, SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_GEN(osd), NULL, &gen, 8); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_UID(osd), NULL, &osa->uid, 8); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_GID(osd), NULL, &osa->gid, 8); + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_PARENT(osd), NULL, + &obj->oo_parent, 8); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_FLAGS(osd), NULL, &osa->flags, 8); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_ATIME(osd), NULL, osa->atime, 16); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(osd), NULL, osa->mtime, 16); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(osd), NULL, osa->ctime, 16); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CRTIME(osd), NULL, crtime, 16); SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_LINKS(osd), NULL, &osa->nlink, 8); +#ifdef ZFS_PROJINHERIT + if (osd->od_projectused_dn) + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_PROJID(osd), NULL, + &osa->projid, 8); +#endif SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_RDEV(osd), NULL, &osa->rdev, 8); - SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_PARENT(osd), NULL, - &obj->oo_parent, 8); LASSERT(cnt <= ARRAY_SIZE(osd_oti_get(env)->oti_attr_bulk)); /* Update the SA for additions, modifications, and removals. */ diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index a6dff5d..48f8ad6 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -72,10 +72,21 @@ export QUOTA_AUTO=0 check_and_setup_lustre is_project_quota_supported() { + lsattr -dp > /dev/null 2>&1 || return 1 + [ "$(facet_fstype $SINGLEMDS)" == "ldiskfs" ] && - [ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.9.55) ] && - egrep -q "7." /etc/redhat-release && - man chattr | grep project >&/dev/null + [ $(lustre_version_code $SINGLEMDS) -gt \ + $(version_code 2.9.55) ] && + egrep -q "7." /etc/redhat-release && return 0 + + if [ "$(facet_fstype $SINGLEMDS)" == "zfs" ]; then + [ $(lustre_version_code $SINGLEMDS) -le \ + $(version_code 2.10.53) ] && return 1 + + $ZPOOL upgrade -v | grep project_quota && return 0 + fi + + return 1 } SHOW_QUOTA_USER="$LFS quota -v -u $TSTUSR $DIR" @@ -186,6 +197,8 @@ getquota() { local spec local uuid + sync_all_data > /dev/null 2>&1 || true + [ "$#" != 4 ] && error "getquota: wrong number of arguments: $#" [ "$1" != "-u" -a "$1" != "-g" -a "$1" != "-p" ] && error "getquota: wrong u/g/p specifier $1 passed" @@ -330,6 +343,7 @@ wait_ost_reint() { disable_project_quota() { is_project_quota_supported || return 0 + [ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] && return 0 stopall || error "failed to stopall (1)" for num in $(seq $MDSCOUNT); do @@ -397,7 +411,8 @@ quota_show_check() { } enable_project_quota() { - is_project_quota_supported || return 0 + is_project_quota_supported || return 0 + [ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] && return 0 stopall || error "failed to stopall (1)" for num in $(seq $MDSCOUNT); do @@ -563,7 +578,7 @@ test_1() { # test for Project log "--------------------------------------" - log "project quota (block hardlimit:$LIMIT mb)" + log "Project quota (block hardlimit:$LIMIT mb)" $LFS setquota -p $TSTPRJID -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR || error "set project quota failed" @@ -580,7 +595,7 @@ test_1() { cancel_lru_locks osc sync; sync_all_data || true $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && quota_error p \ - $TSTPRJID "project write success, but expect edquot" + $TSTPRJID "project write success, but expect EDQUOT" # cleanup cleanup_quota_test @@ -1054,6 +1069,9 @@ test_5() { USED=$(getquota -g $TSTUSR global curspace) [ $USED -ne 0 ] && error "Used block($USED) for group $TSTUSR isn't 0." if is_project_quota_supported; then + USED=$(getquota -p $TSTPRJID global curinodes) + [ $USED -ne 0 ] && + error "Used inode($USED) for project $TSTPRJID isn't 0." USED=$(getquota -p $TSTPRJID global curspace) [ $USED -ne 0 ] && error "Used block($USED) for project $TSTPRJID isn't 0." @@ -2447,6 +2465,9 @@ test_33() { USED=$(getquota -g $TSTID global curinodes) [ $USED -eq 0 ] || error "Used inodes for group $TSTID isn't 0. $USED" if is_project_quota_supported; then + USED=$(getquota -p $TSTPRJID global curspace) + [ $USED -eq 0 ] || + error "Used space for project $TSTPRJID isn't 0. $USED" USED=$(getquota -p $TSTPRJID global curinodes) [ $USED -eq 0 ] || error "Used inodes for project $TSTPRJID isn't 0. $USED" @@ -2850,24 +2871,24 @@ test_51() { mkdir $dir && change_project -dp 1 $dir && change_project +P $dir local used=$(getquota -p 1 global curinodes) - [ $used != "1" ] && error "expected 1 got $used" + [ $used != "1" ] && error "expected 1 got $used" touch $dir/1 touch $dir/2 cp $dir/2 $dir/3 used=$(getquota -p 1 global curinodes) - [ $used != "4" ] && error "expected 4 got $used" + [ $used != "4" ] && error "expected 4 got $used" $DD if=/dev/zero of=$DIR/$tdir/6 bs=1M count=1 #try cp to dir cp $DIR/$tdir/6 $dir/6 used=$(getquota -p 1 global curinodes) - [ $used != "5" ] && error "expected 5 got $used" + [ $used != "5" ] && error "expected 5 got $used" #try mv to dir mv $DIR/$tdir/6 $dir/7 used=$(getquota -p 1 global curinodes) - [ $used != "6" ] && error "expected 6 got $used" + [ $used != "6" ] && error "expected 6 got $used" rm -rf $dir cleanup_quota_test -- 1.8.3.1