From 9f463e30893138845eb571cc32426e8d1e04a9fa Mon Sep 17 00:00:00 2001 From: vitaly Date: Thu, 4 Jun 2009 18:38:58 +0000 Subject: [PATCH] Branch HEAD b=1028 i=nathan i=tappro i=fanyong i=vs SOM Preview --- lustre/cmm/cmm_device.c | 4 -- lustre/include/lclient.h | 7 ++++ lustre/include/lustre/lustre_idl.h | 2 +- lustre/include/lustre/lustre_user.h | 2 + lustre/include/lustre_export.h | 6 +++ lustre/include/lustre_quota.h | 6 ++- lustre/include/obd.h | 3 +- lustre/include/obd_class.h | 2 + lustre/lclient/lcommon_cl.c | 4 +- lustre/liblustre/file.c | 11 +++++- lustre/liblustre/llite_lib.h | 1 + lustre/liblustre/super.c | 12 +++--- lustre/llite/file.c | 15 +++++-- lustre/llite/llite_close.c | 11 ++++-- lustre/llite/llite_internal.h | 2 +- lustre/llite/llite_lib.c | 17 ++++---- lustre/llite/rw.c | 3 +- lustre/llite/vvp_io.c | 12 ++---- lustre/llite/vvp_page.c | 4 +- lustre/mds/mds_lov.c | 3 +- lustre/mdt/mdt_handler.c | 19 +++++++++ lustre/mdt/mdt_internal.h | 3 +- lustre/mdt/mdt_lproc.c | 79 +++++++++++++++++++++++++++++++++++++ lustre/mdt/mdt_reint.c | 2 +- lustre/obdclass/cl_lock.c | 1 + lustre/obdclass/lprocfs_status.c | 2 +- lustre/osd/osd_handler.c | 10 ++--- lustre/quota/lproc_quota.c | 5 +++ lustre/quota/quota_master.c | 12 ++++-- lustre/tests/sanity-quota.sh | 6 +++ lustre/tests/sanity.sh | 58 +++++++++++++++++++++++++++ lustre/tests/sanityN.sh | 12 +++--- lustre/tests/test-framework.sh | 8 ++++ 33 files changed, 280 insertions(+), 64 deletions(-) diff --git a/lustre/cmm/cmm_device.c b/lustre/cmm/cmm_device.c index 82c89dc..73636ea 100644 --- a/lustre/cmm/cmm_device.c +++ b/lustre/cmm/cmm_device.c @@ -230,10 +230,6 @@ static int cmm_quota_off(const struct lu_env *env, struct md_device *m, int rc; ENTRY; - /* disable quota for CMD case temporary. */ - if (cmm_dev->cmm_tgt_count) - RETURN(-EOPNOTSUPP); - rc = cmm_child_ops(cmm_dev)->mdo_quota.mqo_off(env, cmm_dev->cmm_child, type); diff --git a/lustre/include/lclient.h b/lustre/include/lclient.h index 2eb2425..8b86063 100644 --- a/lustre/include/lclient.h +++ b/lustre/include/lclient.h @@ -136,6 +136,13 @@ static inline struct ccc_thread_info *ccc_env_info(const struct lu_env *env) return info; } +static inline struct cl_attr *ccc_env_thread_attr(const struct lu_env *env) +{ + struct cl_attr *attr = &ccc_env_info(env)->cti_attr; + memset(attr, 0, sizeof(*attr)); + return attr; +} + struct ccc_session { struct ccc_io cs_ios; }; diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index efc9870..7f81dd4 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -786,7 +786,7 @@ extern void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb); OBD_CONNECT_MDS_CAPA | OBD_CONNECT_OSS_CAPA | \ OBD_CONNECT_MDS_MDS | OBD_CONNECT_FID | \ LRU_RESIZE_CONNECT_FLAG | OBD_CONNECT_VBR | \ - OBD_CONNECT_LOV_V3) + OBD_CONNECT_LOV_V3 | OBD_CONNECT_SOM) #define OST_CONNECT_SUPPORTED (OBD_CONNECT_SRVLOCK | OBD_CONNECT_GRANT | \ OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | \ OBD_CONNECT_TRUNCLOCK | OBD_CONNECT_INDEX | \ diff --git a/lustre/include/lustre/lustre_user.h b/lustre/include/lustre/lustre_user.h index fd0d3f9..8336538 100644 --- a/lustre/include/lustre/lustre_user.h +++ b/lustre/include/lustre/lustre_user.h @@ -238,6 +238,8 @@ static inline char *obd_uuid2str(struct obd_uuid *uuid) #define LUSTRE_Q_FINVALIDATE 0x80000c /* invalidate filter quota data */ #define UGQUOTA 2 /* set both USRQUOTA and GRPQUOTA */ +#define IMMQUOTA 0x4 /* set immutable quota flag, cannot be turned on/off + * on-fly. temporary used by SOM */ struct if_quotacheck { char obd_type[16]; diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h index ef3cd4c..743ce3b 100644 --- a/lustre/include/lustre_export.h +++ b/lustre/include/lustre_export.h @@ -245,6 +245,12 @@ static inline int exp_connect_vbr(struct obd_export *exp) return !!(exp->exp_connect_flags & OBD_CONNECT_VBR); } +static inline int exp_connect_som(struct obd_export *exp) +{ + LASSERT(exp != NULL); + return !!(exp->exp_connect_flags & OBD_CONNECT_SOM); +} + static inline int imp_connect_lru_resize(struct obd_import *imp) { struct obd_connect_data *ocd; diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h index 6f0105a..85b0195 100644 --- a/lustre/include/lustre_quota.h +++ b/lustre/include/lustre_quota.h @@ -231,12 +231,16 @@ struct lustre_quota_ctxt { * 0:Off, 1:On */ lqc_valid:1, /** this qctxt is valid or not */ - lqc_setup:1; /** + lqc_setup:1, /** * tell whether of not quota_type has * been processed, so that the master * knows when it can start processing * incoming acq/rel quota requests */ + lqc_immutable:1; /** + * cannot be turned on/off on-fly; + * temporary used by SOM. + */ /** }@ */ /** * original unit size of file quota and diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 4efbf5d..6c97d07 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -1033,8 +1033,9 @@ struct obd_device { obd_fail:1, /* cleanup with failover */ obd_async_recov:1, /* allow asyncronous orphan cleanup */ obd_no_conn:1, /* deny new connections */ - obd_inactive:1; /* device active/inactive + obd_inactive:1, /* device active/inactive * (for /proc/status only!!) */ + obd_process_conf:1; /* device is processing mgs config */ /* uuid-export hash body */ struct lustre_hash *obd_uuid_hash; /* nid-export hash body */ diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 3ea3a79..96b9a7d 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -540,6 +540,7 @@ obd_process_config(struct obd_device *obd, int datalen, void *data) OBD_CHECK_DEV(obd); + obd->obd_process_conf = 1; ldt = obd->obd_type->typ_lu; d = obd->obd_lu_dev; if (ldt != NULL && d != NULL) { @@ -555,6 +556,7 @@ obd_process_config(struct obd_device *obd, int datalen, void *data) rc = OBP(obd, process_config)(obd, datalen, data); } OBD_COUNTER_INCREMENT(obd, process_config); + obd->obd_process_conf = 0; RETURN(rc); } diff --git a/lustre/lclient/lcommon_cl.c b/lustre/lclient/lcommon_cl.c index 4fbf688..78cdc63 100644 --- a/lustre/lclient/lcommon_cl.c +++ b/lustre/lclient/lcommon_cl.c @@ -665,7 +665,7 @@ void ccc_lock_state(const struct lu_env *env, obj = slice->cls_obj; inode = ccc_object_inode(obj); - attr = &ccc_env_info(env)->cti_attr; + attr = ccc_env_thread_attr(env); /* vmtruncate()->ll_truncate() first sets the i_size and then * the kms under both a DLM lock and the @@ -849,7 +849,7 @@ int ccc_prep_size(const struct lu_env *env, struct cl_object *obj, struct cl_io *io, loff_t start, size_t count, int vfslock, int *exceed) { - struct cl_attr *attr = &ccc_env_info(env)->cti_attr; + struct cl_attr *attr = ccc_env_thread_attr(env); struct inode *inode = ccc_object_inode(obj); loff_t pos = start + count - 1; loff_t kms; diff --git a/lustre/liblustre/file.c b/lustre/liblustre/file.c index 9344a4f..1850fd3 100644 --- a/lustre/liblustre/file.c +++ b/lustre/liblustre/file.c @@ -157,6 +157,15 @@ void obdo_refresh_inode(struct inode *dst, st->st_blocks = src->o_blocks; } +void llu_ioepoch_open(struct llu_inode_info *lli, __u64 ioepoch) +{ + if (ioepoch && lli->lli_ioepoch != ioepoch) { + lli->lli_ioepoch = ioepoch; + CDEBUG(D_INODE, "Epoch "LPU64" opened on "DFID" for truncate\n", + ioepoch, PFID(&lli->lli_fid)); + } +} + int llu_local_open(struct llu_inode_info *lli, struct lookup_intent *it) { struct ptlrpc_request *req = it->d.lustre.it_data; @@ -182,7 +191,7 @@ int llu_local_open(struct llu_inode_info *lli, struct lookup_intent *it) fd->fd_mds_och.och_magic = OBD_CLIENT_HANDLE_MAGIC; fd->fd_mds_och.och_fid = lli->lli_fid; lli->lli_file_data = fd; - + llu_ioepoch_open(lli, body->ioepoch); md_set_open_replay_data(lli->lli_sbi->ll_md_exp, &fd->fd_mds_och, it->d.lustre.it_data); diff --git a/lustre/liblustre/llite_lib.h b/lustre/liblustre/llite_lib.h index 8c1bec6..3f67427 100644 --- a/lustre/liblustre/llite_lib.h +++ b/lustre/liblustre/llite_lib.h @@ -255,6 +255,7 @@ _SYSIO_OFF_T llu_iop_pos(struct inode *ino, _SYSIO_OFF_T off); int llu_vmtruncate(struct inode * inode, loff_t offset, obd_flag obd_flags); void obdo_refresh_inode(struct inode *dst, struct obdo *src, obd_flag valid); int llu_objects_destroy(struct ptlrpc_request *request, struct inode *dir); +void llu_ioepoch_open(struct llu_inode_info *lli, __u64 ioepoch); /* rw.c */ int llu_iop_read(struct inode *ino, struct ioctx *ioctxp); diff --git a/lustre/liblustre/super.c b/lustre/liblustre/super.c index a04d2a8..de5f687 100644 --- a/lustre/liblustre/super.c +++ b/lustre/liblustre/super.c @@ -803,11 +803,7 @@ int llu_setattr_raw(struct inode *inode, struct iattr *attr) if (rc) RETURN(rc); - if (op_data.op_ioepoch) - CDEBUG(D_INODE, "Epoch "LPU64" opened on "DFID" for " - "truncate\n", op_data.op_ioepoch, - PFID(&llu_i2info(inode)->lli_fid)); - + llu_ioepoch_open(llu_i2info(inode), op_data.op_ioepoch); if (!lsm || !S_ISREG(st->st_mode)) { CDEBUG(D_INODE, "no lsm: not setting attrs on OST\n"); GOTO(out, rc); @@ -1993,7 +1989,7 @@ llu_fsswop_mount(const char *source, ocd.ocd_connect_flags = OBD_CONNECT_IBITS | OBD_CONNECT_VERSION | OBD_CONNECT_FID | OBD_CONNECT_AT | - OBD_CONNECT_VBR; + OBD_CONNECT_VBR | OBD_CONNECT_SOM; #ifdef LIBLUSTRE_POSIX_ACL ocd.ocd_connect_flags |= OBD_CONNECT_ACL; #endif @@ -2029,7 +2025,9 @@ llu_fsswop_mount(const char *source, ocd.ocd_connect_flags = OBD_CONNECT_SRVLOCK | OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | OBD_CONNECT_TRUNCLOCK | - OBD_CONNECT_FID | OBD_CONNECT_AT; + OBD_CONNECT_FID | OBD_CONNECT_AT | + OBD_CONNECT_SOM; + ocd.ocd_version = LUSTRE_VERSION_CODE; err = obd_connect(NULL, &sbi->ll_dt_exp, obd, &sbi->ll_sb_uuid, &ocd, NULL); if (err) { diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 68785b9..e595d8f 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -92,8 +92,7 @@ static void ll_prepare_close(struct inode *inode, struct md_op_data *op_data, if (!(och->och_flags & FMODE_WRITE)) goto out; - if (!(ll_i2mdexp(inode)->exp_connect_flags & OBD_CONNECT_SOM) || - !S_ISREG(inode->i_mode)) + if (!(exp_connect_som(ll_i2mdexp(inode))) || !S_ISREG(inode->i_mode)) op_data->op_attr.ia_valid |= ATTR_SIZE | ATTR_BLOCKS; else ll_epoch_close(inode, op_data, &och, 0); @@ -142,7 +141,6 @@ static int ll_close_inode_openhandle(struct obd_export *md_exp, rc = md_close(md_exp, op_data, och->och_mod, &req); if (rc == -EAGAIN) { /* This close must have the epoch closed. */ - LASSERT(exp->exp_connect_flags & OBD_CONNECT_SOM); LASSERT(epoch_close); /* MDS has instructed us to obtain Size-on-MDS attribute from * OSTs and send setattr to back to MDS. */ @@ -403,6 +401,15 @@ out: RETURN(rc); } +void ll_ioepoch_open(struct ll_inode_info *lli, __u64 ioepoch) +{ + if (ioepoch && lli->lli_ioepoch != ioepoch) { + lli->lli_ioepoch = ioepoch; + CDEBUG(D_INODE, "Epoch "LPU64" opened on "DFID"\n", + ioepoch, PFID(&lli->lli_fid)); + } +} + static int ll_och_fill(struct obd_export *md_exp, struct ll_inode_info *lli, struct lookup_intent *it, struct obd_client_handle *och) { @@ -418,7 +425,7 @@ static int ll_och_fill(struct obd_export *md_exp, struct ll_inode_info *lli, och->och_magic = OBD_CLIENT_HANDLE_MAGIC; och->och_fid = lli->lli_fid; och->och_flags = it->it_flags; - lli->lli_ioepoch = body->ioepoch; + ll_ioepoch_open(lli, body->ioepoch); return md_set_open_replay_data(md_exp, och, req); } diff --git a/lustre/llite/llite_close.c b/lustre/llite/llite_close.c index 14c76a0..c9ff428 100644 --- a/lustre/llite/llite_close.c +++ b/lustre/llite/llite_close.c @@ -84,12 +84,13 @@ void vvp_write_complete(struct ccc_object *club, struct ccc_page *page) void ll_queue_done_writing(struct inode *inode, unsigned long flags) { struct ll_inode_info *lli = ll_i2info(inode); - + struct ccc_object *club = cl2ccc(ll_i2info(inode)->lli_clob); spin_lock(&lli->lli_lock); lli->lli_flags |= flags; + ENTRY; if ((lli->lli_flags & LLIF_DONE_WRITING) && - list_empty(&lli->lli_pending_write_llaps)) { + list_empty(&club->cob_pending_list)) { struct ll_close_queue *lcq = ll_i2sbi(inode)->ll_lcq; if (lli->lli_flags & LLIF_MDS_SIZE_LOCK) @@ -117,6 +118,7 @@ void ll_queue_done_writing(struct inode *inode, unsigned long flags) spin_unlock(&lcq->lcq_lock); } spin_unlock(&lli->lli_lock); + EXIT; } /** Closes epoch and sends Size-on-MDS attribute update if possible. Call @@ -125,10 +127,11 @@ void ll_epoch_close(struct inode *inode, struct md_op_data *op_data, struct obd_client_handle **och, unsigned long flags) { struct ll_inode_info *lli = ll_i2info(inode); + struct ccc_object *club = cl2ccc(ll_i2info(inode)->lli_clob); ENTRY; spin_lock(&lli->lli_lock); - if (!(list_empty(&lli->lli_pending_write_llaps))) { + if (!(list_empty(&club->cob_pending_list))) { if (!(lli->lli_flags & LLIF_EPOCH_PENDING)) { LASSERT(*och != NULL); LASSERT(lli->lli_pending_och == NULL); @@ -180,7 +183,7 @@ void ll_epoch_close(struct inode *inode, struct md_op_data *op_data, } } - LASSERT(list_empty(&lli->lli_pending_write_llaps)); + LASSERT(list_empty(&club->cob_pending_list)); lli->lli_flags &= ~LLIF_SOM_DIRTY; spin_unlock(&lli->lli_lock); op_data->op_flags |= MF_SOM_CHANGE; diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index d0d5229..2701b71 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -125,7 +125,6 @@ struct ll_inode_info { /* this lock protects posix_acl, pending_write_llaps, mmap_cnt */ spinlock_t lli_lock; - struct list_head lli_pending_write_llaps; struct list_head lli_close_list; /* handle is to be sent to MDS later on done_writing and setattr. * Open handle data are needed for the recovery to reconstruct @@ -640,6 +639,7 @@ int ll_file_open(struct inode *inode, struct file *file); int ll_file_release(struct inode *inode, struct file *file); int ll_glimpse_ioctl(struct ll_sb_info *sbi, struct lov_stripe_md *lsm, lstat_t *st); +void ll_ioepoch_open(struct ll_inode_info *lli, __u64 ioepoch); int ll_local_open(struct file *file, struct lookup_intent *it, struct ll_file_data *fd, struct obd_client_handle *och); diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 50c45c2..d99700e 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -199,7 +199,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) OBD_CONNECT_OSS_CAPA | OBD_CONNECT_CANCELSET| OBD_CONNECT_FID | OBD_CONNECT_AT | OBD_CONNECT_LOV_V3 | OBD_CONNECT_RMT_CLIENT | - OBD_CONNECT_VBR; + OBD_CONNECT_VBR | OBD_CONNECT_SOM; #ifdef HAVE_LRU_RESIZE_SUPPORT if (sbi->ll_flags & LL_SBI_LRU_RESIZE) @@ -343,7 +343,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) OBD_CONNECT_SRVLOCK | OBD_CONNECT_TRUNCLOCK| OBD_CONNECT_AT | OBD_CONNECT_RMT_CLIENT | OBD_CONNECT_OSS_CAPA | OBD_CONNECT_VBR| - OBD_CONNECT_GRANT_SHRINK; + OBD_CONNECT_GRANT_SHRINK | OBD_CONNECT_SOM; if (!OBD_FAIL_CHECK(OBD_FAIL_OSC_CONNECT_CKSUM)) { /* OBD_CONNECT_CKSUM should always be set, even if checksums are @@ -685,6 +685,8 @@ void client_common_put_super(struct super_block *sb) ll_close_thread_shutdown(sbi->ll_lcq); + cl_sb_fini(sb); + /* destroy inodes in deathrow */ prune_deathrow(sbi, 0); @@ -868,7 +870,6 @@ void ll_lli_init(struct ll_inode_info *lli) lli->lli_flags = 0; lli->lli_maxbytes = PAGE_CACHE_MAXBYTES; spin_lock_init(&lli->lli_lock); - INIT_LIST_HEAD(&lli->lli_pending_write_llaps); INIT_LIST_HEAD(&lli->lli_close_list); lli->lli_inode_magic = LLI_INODE_MAGIC; sema_init(&lli->lli_och_sem, 1); @@ -1000,12 +1001,11 @@ void ll_put_super(struct super_block *sb) } } - cl_sb_fini(sb); - if (sbi->ll_lcq) { /* Only if client_common_fill_super succeeded */ client_common_put_super(sb); } + next = 0; while ((obd = class_devices_in_group(&sbi->ll_sb_uuid, &next)) !=NULL) { class_manual_cleanup(obd); @@ -1340,10 +1340,7 @@ int ll_setattr_raw(struct inode *inode, struct iattr *attr) if (rc) GOTO(out, rc); - if (op_data->op_ioepoch) - CDEBUG(D_INODE, "Epoch "LPU64" opened on "DFID" for " - "truncate\n", op_data->op_ioepoch, PFID(&lli->lli_fid)); - + ll_ioepoch_open(lli, op_data->op_ioepoch); if (!lsm || !S_ISREG(inode->i_mode)) { CDEBUG(D_INODE, "no lsm: not setting attrs on OST\n"); GOTO(out, rc = 0); @@ -1651,7 +1648,7 @@ void ll_update_inode(struct inode *inode, struct lustre_md *md) LASSERT(fid_seq(&lli->lli_fid) != 0); if (body->valid & OBD_MD_FLSIZE) { - if ((ll_i2mdexp(inode)->exp_connect_flags & OBD_CONNECT_SOM) && + if (exp_connect_som(ll_i2mdexp(inode)) && S_ISREG(inode->i_mode) && lli->lli_smd) { struct lustre_handle lockh; ldlm_mode_t mode; diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c index b29563f..827f644 100644 --- a/lustre/llite/rw.c +++ b/lustre/llite/rw.c @@ -707,7 +707,7 @@ int ll_readahead(const struct lu_env *env, struct cl_io *io, { struct vvp_io *vio = vvp_env_io(env); struct vvp_thread_info *vti = vvp_env_info(env); - struct ccc_thread_info *cti = ccc_env_info(env); + struct cl_attr *attr = ccc_env_thread_attr(env); unsigned long start = 0, end = 0, reserved; unsigned long ra_end, len; struct inode *inode; @@ -715,7 +715,6 @@ int ll_readahead(const struct lu_env *env, struct cl_io *io, struct ra_io_arg *ria = &vti->vti_ria; struct ll_inode_info *lli; struct cl_object *clob; - struct cl_attr *attr = &cti->cti_attr; int ret = 0; __u64 kms; ENTRY; diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c index 0df098a..e398f2c 100644 --- a/lustre/llite/vvp_io.c +++ b/lustre/llite/vvp_io.c @@ -661,10 +661,7 @@ static int vvp_page_sync_io(const struct lu_env *env, struct cl_io *io, int to, enum cl_req_type crt) { struct cl_2queue *queue; - struct ccc_object *cobo = cl2ccc(page->cp_obj); struct cl_sync_io *anchor = &ccc_env_info(env)->cti_sync_io; - - int writing = io->ci_type == CIT_WRITE; int result; LASSERT(io->ci_type == CIT_READ || io->ci_type == CIT_WRITE); @@ -673,10 +670,6 @@ static int vvp_page_sync_io(const struct lu_env *env, struct cl_io *io, cl_2queue_init_page(queue, page); - if (writing) - /* Do not pass llap here as it is sync write. */ - vvp_write_pending(cobo, cp); - cl_sync_io_init(anchor, 1); cp->cpg_sync_io = anchor; cl_page_clip(env, page, 0, to); @@ -707,7 +700,7 @@ static int vvp_io_prepare_partial(const struct lu_env *env, struct cl_io *io, struct ccc_page *cp, unsigned from, unsigned to) { - struct cl_attr *attr = &ccc_env_info(env)->cti_attr; + struct cl_attr *attr = ccc_env_thread_attr(env); loff_t offset = cl_offset(obj, pg->cp_index); int result; @@ -829,6 +822,9 @@ static int vvp_io_commit_write(const struct lu_env *env, tallyop = LPROC_LL_DIRTY_MISSES; vvp_write_pending(cl2ccc(obj), cp); set_page_dirty(vmpage); + /* ll_set_page_dirty() does the same for now, but + * it will not soon. */ + vvp_write_pending(cl2ccc(obj), cp); result = cl_page_cache_add(env, io, pg, CRT_WRITE); if (result == -EDQUOT) /* diff --git a/lustre/llite/vvp_page.c b/lustre/llite/vvp_page.c index 537cce7..8db2b38 100644 --- a/lustre/llite/vvp_page.c +++ b/lustre/llite/vvp_page.c @@ -209,6 +209,7 @@ static int vvp_page_prep_write(const struct lu_env *env, if (clear_page_dirty_for_io(vmpage)) { set_page_writeback(vmpage); + vvp_write_pending(cl2ccc(slice->cpl_obj), cl2ccc_page(slice)); result = 0; } else result = -EALREADY; @@ -361,7 +362,8 @@ static int vvp_page_make_ready(const struct lu_env *env, * tree. */ set_page_writeback(vmpage); - + vvp_write_pending(cl2ccc(slice->cpl_obj), + cl2ccc_page(slice)); CL_PAGE_HEADER(D_PAGE, env, pg, "readied\n"); result = 0; } else diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index a423c0b..8c0de10 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -653,7 +653,8 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) OBD_CONNECT_OSS_CAPA | OBD_CONNECT_FID | OBD_CONNECT_BRW_SIZE | OBD_CONNECT_CKSUM | OBD_CONNECT_CHANGE_QS | OBD_CONNECT_AT | - OBD_CONNECT_MDS | OBD_CONNECT_SKIP_ORPHAN; + OBD_CONNECT_MDS | OBD_CONNECT_SKIP_ORPHAN | + OBD_CONNECT_SOM; #ifdef HAVE_LRU_RESIZE_SUPPORT data->ocd_connect_flags |= OBD_CONNECT_LRU_RESIZE; #endif diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index fb53bf6..61d5880 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -1765,6 +1765,13 @@ static int mdt_quotactl_handle(struct mdt_thread_info *info) switch (oqctl->qc_cmd) { case Q_QUOTAON: + if (info->mti_mdt->mdt_som_conf) { + /* Quota cannot be used together with SOM while + * SOM stored blocks in i_blocks but not in SOM EA. */ + LCONSOLE_ERROR("Fail to turn Quota on: SOM is enabled " + "and temporary conflicts with quota.\n"); + RETURN(-ENOTSUPP); + } rc = mqo->mqo_on(info->mti_env, next, oqctl->qc_type); break; case Q_QUOTAOFF: @@ -4522,6 +4529,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, m->mdt_max_mdsize = MAX_MD_SIZE; m->mdt_max_cookiesize = sizeof(struct llog_cookie); + m->mdt_som_conf = 0; m->mdt_opts.mo_user_xattr = 0; m->mdt_opts.mo_acl = 0; @@ -4942,6 +4950,9 @@ static int mdt_connect_internal(struct obd_export *exp, if (!mdt->mdt_opts.mo_user_xattr) data->ocd_connect_flags &= ~OBD_CONNECT_XATTR; + if (!mdt->mdt_som_conf) + data->ocd_connect_flags &= ~OBD_CONNECT_SOM; + spin_lock(&exp->exp_lock); exp->exp_connect_flags = data->ocd_connect_flags; spin_unlock(&exp->exp_lock); @@ -4964,6 +4975,14 @@ static int mdt_connect_internal(struct obd_export *exp, return -EBADE; } + if (mdt->mdt_som_conf && + !(exp->exp_connect_flags & OBD_CONNECT_MDS_MDS) && + !(exp->exp_connect_flags & OBD_CONNECT_SOM)) { + CWARN("%s: MDS has SOM enabled, but client does not support " + "it\n", mdt->mdt_md_dev.md_lu_dev.ld_obd->obd_name); + return -EBADE; + } + return 0; } diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index aa6c6cc..1195605 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -152,7 +152,8 @@ struct mdt_device { cfs_timer_t mdt_ck_timer; struct ptlrpc_thread mdt_ck_thread; struct lustre_capa_key mdt_capa_keys[2]; - unsigned int mdt_capa_conf:1; + unsigned int mdt_capa_conf:1, + mdt_som_conf:1; /* root squash */ uid_t mdt_squash_uid; diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c index 5d77e35..1874ff8 100644 --- a/lustre/mdt/mdt_lproc.c +++ b/lustre/mdt/mdt_lproc.c @@ -645,6 +645,83 @@ static int lprocfs_wr_nosquash_nids(struct file *file, const char *buffer, RETURN(rc); } +static int lprocfs_rd_mdt_som(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct obd_device *obd = data; + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + + return snprintf(page, count, "%sabled\n", + mdt->mdt_som_conf ? "en" : "dis"); +} + +static int mdt_quota_off(struct mdt_device *mdt) +{ + struct md_device *next = mdt->mdt_child; + const struct md_quota_operations *mqo = &next->md_ops->mdo_quota; + struct lu_env env; + int rc; + + lu_env_init(&env, LCT_MD_THREAD); + rc = mqo->mqo_off(&env, next, UGQUOTA | IMMQUOTA); + lu_env_fini(&env); + return rc; +} + +static int lprocfs_wr_mdt_som(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct obd_export *exp; + struct obd_device *obd = data; + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + char kernbuf[16]; + unsigned long val = 0; + int rc; + + if (count > (sizeof(kernbuf) - 1)) + return -EINVAL; + + if (copy_from_user(kernbuf, buffer, count)) + return -EFAULT; + + kernbuf[count] = '\0'; + + if (!strcmp(kernbuf, "enabled")) + val = 1; + else if (strcmp(kernbuf, "disabled")) + return -EINVAL; + + if (mdt->mdt_som_conf == val) + return count; + + if (!obd->obd_process_conf) { + CERROR("Temporary SOM change is not supported, use lctl " + "conf_param for permanent setting\n"); + return count; + } + + /* 1 stands for self export. */ + list_for_each_entry(exp, &obd->obd_exports, exp_obd_chain) { + if (exp == obd->obd_self_export) + continue; + if (exp->exp_connect_flags & OBD_CONNECT_MDS_MDS) + continue; + /* Some clients are already connected, skip the change */ + LCONSOLE_INFO("%s is already connected, SOM will be %s on " + "the next mount\n", exp->exp_client_uuid.uuid, + val ? "enabled" : "disabled"); + return count; + } + + if ((rc = mdt_quota_off(mdt))) + return rc; + + mdt->mdt_som_conf = val; + LCONSOLE_INFO("Enabling SOM\n"); + + return count; +} + static struct lprocfs_vars lprocfs_mdt_obd_vars[] = { { "uuid", lprocfs_rd_uuid, 0, 0 }, { "recovery_status", lprocfs_obd_rd_recovery_status, 0, 0 }, @@ -674,6 +751,8 @@ static struct lprocfs_vars lprocfs_mdt_obd_vars[] = { lprocfs_wr_root_squash, 0 }, { "nosquash_nids", lprocfs_rd_nosquash_nids, lprocfs_wr_nosquash_nids, 0 }, + { "som", lprocfs_rd_mdt_som, + lprocfs_wr_mdt_som, 0 }, { 0 } }; diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c index 26f923d..3b005b1 100644 --- a/lustre/mdt/mdt_reint.c +++ b/lustre/mdt/mdt_reint.c @@ -286,7 +286,7 @@ int mdt_attr_set(struct mdt_thread_info *info, struct mdt_object *mo, int flags) /* attr shouldn't be set on remote object */ LASSERT(mdt_object_exists(mo) >= 0); - if (info->mti_epoch) + if (exp_connect_som(info->mti_exp) && info->mti_epoch) som_update = (info->mti_epoch->flags & MF_SOM_CHANGE); /* Try to avoid object_lock if another epoch has been started diff --git a/lustre/obdclass/cl_lock.c b/lustre/obdclass/cl_lock.c index 6f6251d..786bca5 100644 --- a/lustre/obdclass/cl_lock.c +++ b/lustre/obdclass/cl_lock.c @@ -542,6 +542,7 @@ struct cl_lock *cl_lock_peek(const struct lu_env *env, const struct cl_io *io, if (ok) { cl_lock_hold_add(env, lock, scope, source); cl_lock_user_add(env, lock); + cl_lock_put(env, lock); } cl_lock_mutex_put(env, lock); if (!ok) { diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index a1faf78..6a337c6 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -739,7 +739,7 @@ static const char *obd_connect_names[] = { "mds_capability", "oss_capability", "early_lock_cancel", - "size_on_mds", + "som", "adaptive_timeouts", "lru_resize", "mds_mds_connection", diff --git a/lustre/osd/osd_handler.c b/lustre/osd/osd_handler.c index 904b9d7..117547f 100644 --- a/lustre/osd/osd_handler.c +++ b/lustre/osd/osd_handler.c @@ -1296,14 +1296,14 @@ static int osd_inode_setattr(const struct lu_env *env, LDISKFS_I(inode)->i_disksize = attr->la_size; i_size_write(inode, attr->la_size); } -# if 0 - /* - * OSD should not change "i_blocks" which is used by quota. + + /* OSD should not change "i_blocks" which is used by quota. * "i_blocks" should be changed by ldiskfs only. - * Disable this assignment until SOM to fix some EA field. */ + * Enable this assignment for SOM purpose now, until it is + * stored in SOM EA. */ if (bits & LA_BLOCKS) inode->i_blocks = attr->la_blocks; -#endif + if (bits & LA_MODE) inode->i_mode = (inode->i_mode & S_IFMT) | (attr->la_mode & ~S_IFMT); diff --git a/lustre/quota/lproc_quota.c b/lustre/quota/lproc_quota.c index ba9492f..58d88f7 100644 --- a/lustre/quota/lproc_quota.c +++ b/lustre/quota/lproc_quota.c @@ -231,6 +231,11 @@ static int auto_quota_on(struct obd_device *obd, int type, rc = 0; goto out; } + if (obt->obt_qctxt.lqc_immutable) { + LCONSOLE_ERROR("Failed to turn Quota on, immutable mode " + "(is SOM enabled?)\n"); + goto out; + } oqctl->qc_type = type; oqctl->qc_cmd = Q_QUOTAON; diff --git a/lustre/quota/quota_master.c b/lustre/quota/quota_master.c index f884d9d..2d50d8b 100644 --- a/lustre/quota/quota_master.c +++ b/lustre/quota/quota_master.c @@ -857,6 +857,7 @@ int mds_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) RETURN(-EBUSY); } + LASSERT(!obt->obt_qctxt.lqc_immutable); down(&mds->mds_qonoff_sem); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); rc = mds_admin_quota_on(obd, oqctl); @@ -883,9 +884,12 @@ int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) struct mds_obd *mds = &obd->u.mds; struct obd_device_target *obt = &obd->u.obt; struct lvfs_run_ctxt saved; - int rc, rc2; + int rc, rc2, imm; ENTRY; + imm = oqctl->qc_type & IMMQUOTA; + oqctl->qc_type &= ~IMMQUOTA; + if (oqctl->qc_type != USRQUOTA && oqctl->qc_type != GRPQUOTA && oqctl->qc_type != UGQUOTA) @@ -904,9 +908,11 @@ int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) rc = obd_quotactl(mds->mds_osc_exp, oqctl); rc2 = fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); - if (!rc2) + if (!rc2) { + if (imm) + obt->obt_qctxt.lqc_immutable = 1; obt->obt_qctxt.lqc_flags &= ~UGQUOTA2LQC(oqctl->qc_type); - + } pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); up(&mds->mds_qonoff_sem); atomic_inc(&obt->obt_quotachecking); diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 6e80581..f45fad5 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -65,6 +65,12 @@ DIR2=${DIR2:-$MOUNT2} check_and_setup_lustre +if [ x"$(som_check)" = x"enabled" ]; then + echo "Som is enabled, Quota is temporary conflicts with it" + check_and_cleanup_lustre + exit 0 +fi + LOVNAME=`lctl get_param -n llite.*.lov.common_name | tail -n 1` OSTCOUNT=`lctl get_param -n lov.$LOVNAME.numobd` diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 9f332e3..0258e2a 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -5899,6 +5899,64 @@ test_131e() { } run_test 131e "test read hitting hole" +get_ost_param() { + local token=$1 + local gl_sum=0 + for node in $(osts_nodes); do + gl=$(do_node $node "$LCTL get_param -n ost.OSS.ost.stats" | awk '/'$token'/ {print $2}' | head -n 1) + [ x$gl = x"" ] && gl=0 + gl_sum=$((gl_sum + gl)) + done + echo $gl +} + +som_mode_switch() { + local som=$1 + local gl1=$2 + local gl2=$3 + + if [ x$som = x"enabled" ]; then + [ $((gl2 - gl1)) -gt 0 ] && error "no glimpse RPC is expected" + do_facet mgs "$LCTL conf_param $FSNAME.mdt.som=disabled" + else + [ $((gl2 - gl1)) -gt 0 ] || error "some glimpse RPC is expected" + do_facet mgs "$LCTL conf_param $FSNAME.mdt.som=enabled" + fi + + # do remount to make new mount-conf parameters actual + echo remounting... + sync + stopall 1&>2 2>/dev/null + setupall 1&>2 2>/dev/null +} + +test_132() { #1028, SOM + local num=$(get_mds_dir $DIR) + local mymds=mds${num} + + dd if=/dev/zero of=$DIR/$tfile count=1 2>/dev/null + cancel_lru_locks osc + + som1=$(do_facet $mymds "$LCTL get_param mdt.*.som" | awk -F= ' {print $2}' | head -n 1) + + gl1=$(get_ost_param "ldlm_glimpse_enqueue") + stat $DIR/$tfile >/dev/null + gl2=$(get_ost_param "ldlm_glimpse_enqueue") + echo "SOM is "$som1", "$((gl2 - gl1))" glimpse RPC occured" + cancel_lru_locks osc + som_mode_switch $som1 $gl1 $gl2 + + som2=$(do_facet $mymds "$LCTL get_param mdt.*.som" | awk -F= ' {print $2}' | head -n 1) + [ $som1 != $som2 ] || error "som is still "$som2 + + gl1=$(get_ost_param "ldlm_glimpse_enqueue") + stat $DIR/$tfile >/dev/null + gl2=$(get_ost_param "ldlm_glimpse_enqueue") + echo "SOM is "$som2", "$((gl2 - gl1))" glimpse RPC occured" + som_mode_switch $som2 $gl1 $gl2 +} +run_test 132 "som avoids glimpse rpc" + test_140() { #bug-17379 mkdir -p $DIR/$tdir || error "Creating dir $DIR/$tdir" cd $DIR/$tdir || error "Changing to $DIR/$tdir" diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityN.sh index c322f95..6a41463 100644 --- a/lustre/tests/sanityN.sh +++ b/lustre/tests/sanityN.sh @@ -44,7 +44,7 @@ SETUP=${SETUP:-:} init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} -[ "$SLOW" = "no" ] && EXCEPT_SLOW="12 16 33a" +[ "$SLOW" = "no" ] && EXCEPT_SLOW="12 16 23 33a" SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} FAIL_ON_ERROR=false @@ -436,19 +436,21 @@ test_23() { # Bug 5972 cancel_lru_locks osc time1=`date +%s` - sleep 2 + #MAX_ATIME_DIFF 60, we update atime only if older than 60 seconds + sleep 61 multiop_bg_pause $DIR1/f23 or20_c || return 1 - MULTIPID=$! + # with SOM and opencache enabled, we need to close a file and cancel + # open lock to get atime propogated to MDS + kill -USR1 $! + cancel_lru_locks mdc time2=`stat -c "%X" $DIR2/f23` if (( $time2 <= $time1 )); then - kill -USR1 $MULTIPID error "atime doesn't update among nodes" fi - kill -USR1 $MULTIPID || return 1 rm -f $DIR1/f23 || error "rm -f $DIR1/f23 failed" true } diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 06d70cc..09d2766 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1687,6 +1687,11 @@ mds_sanity_check () { [ $period -lt $timeout ] || log "$count OST are inactive after $timeout seconds, give up" } +som_check() { + SOM_ENABLED=$(do_facet $SINGLEMDS "$LCTL get_param mdt.*.som" | awk -F= ' {print $2}' | head -n 1) + echo $SOM_ENABLED +} + init_param_vars () { if ! remote_ost_nodsh && ! remote_mds_nodsh; then export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2) @@ -1701,6 +1706,9 @@ init_param_vars () { mds_sanity_check $TIMEOUT + if [ x"$(som_check)" = x"enabled" ]; then + ENABLE_QUOTA="" + fi if [ "$ENABLE_QUOTA" ]; then setup_quota $MOUNT || return 2 fi -- 1.8.3.1