From 1e3b123a8a39417f6b442f82a635c281fecbebd6 Mon Sep 17 00:00:00 2001 From: lsy Date: Thu, 12 Oct 2006 15:35:35 +0000 Subject: [PATCH] * all capabilities of opened inode should always renew because many file operations do from file->f_dentry->d_inode directly. * all dir capabilities should always renew because ll_revalid_it needs parent dir capability. * filter update capability key more intelligently. * rename ll_i2mdscapa -> ll_mdscapa_get, ll_lookup_oss_capa -> ll_osscapa_get. * other code cleanup. --- lustre/cmm/cmm_object.c | 6 +- lustre/include/dt_object.h | 3 +- lustre/include/lustre/lustre_idl.h | 11 ++-- lustre/include/lustre_capa.h | 5 +- lustre/include/md_object.h | 7 ++- lustre/llite/dcache.c | 2 +- lustre/llite/dir.c | 8 +-- lustre/llite/file.c | 14 ++--- lustre/llite/llite_capa.c | 121 +++++++++++++++++-------------------- lustre/llite/llite_internal.h | 16 +++-- lustre/llite/llite_lib.c | 14 ++--- lustre/llite/llite_nfs.c | 4 +- lustre/llite/remote_perm.c | 2 +- lustre/llite/rw.c | 6 +- lustre/llite/rw26.c | 2 +- lustre/llite/symlink.c | 2 +- lustre/llite/xattr.c | 4 +- lustre/lov/lov_request.c | 2 - lustre/mdc/mdc_request.c | 8 ++- lustre/mdd/mdd_handler.c | 12 ++-- lustre/mdd/mdd_lov.c | 4 +- lustre/mdt/mdt_capa.c | 26 +++++--- lustre/mdt/mdt_handler.c | 42 ++++++++----- lustre/mdt/mdt_open.c | 4 +- lustre/mdt/mdt_reint.c | 4 +- lustre/obdclass/capa.c | 7 +-- lustre/obdfilter/filter_capa.c | 44 +++++++++----- lustre/osc/osc_request.c | 3 +- lustre/osd/osd_handler.c | 29 +++++---- lustre/osd/osd_internal.h | 1 - 30 files changed, 222 insertions(+), 191 deletions(-) diff --git a/lustre/cmm/cmm_object.c b/lustre/cmm/cmm_object.c index a126289..86c8714 100644 --- a/lustre/cmm/cmm_object.c +++ b/lustre/cmm/cmm_object.c @@ -333,11 +333,11 @@ static int cml_readpage(const struct lu_env *env, struct md_object *mo, } static int cml_capa_get(const struct lu_env *env, struct md_object *mo, - struct lustre_capa *capa) + struct lustre_capa *capa, int renewal) { int rc; ENTRY; - rc = mo_capa_get(env, md_object_next(mo), capa); + rc = mo_capa_get(env, md_object_next(mo), capa, renewal); RETURN(rc); } @@ -721,7 +721,7 @@ static int cmr_readpage(const struct lu_env *env, struct md_object *mo, } static int cmr_capa_get(const struct lu_env *env, struct md_object *mo, - struct lustre_capa *capa) + struct lustre_capa *capa, int renewal) { RETURN(-EFAULT); } diff --git a/lustre/include/dt_object.h b/lustre/include/dt_object.h index 4c3ae54..833e42e 100644 --- a/lustre/include/dt_object.h +++ b/lustre/include/dt_object.h @@ -253,7 +253,8 @@ struct dt_object_operations { struct dt_object *dt, const struct lu_rdpg *rdpg, struct lustre_capa *capa); struct obd_capa *(*do_capa_get)(const struct lu_env *env, - struct dt_object *dt, __u64 opc); + struct dt_object *dt, + struct lustre_capa *old, __u64 opc); }; /* diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 67fd1ab..c872f9f 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -1920,26 +1920,23 @@ extern void lustre_swab_lustre_capa(struct lustre_capa *c); /* lustre_capa.lc_opc */ enum { - /* MDS only fid capability */ CAPA_OPC_BODY_WRITE = 1<<0, /* write fid data */ CAPA_OPC_BODY_READ = 1<<1, /* read fid data */ CAPA_OPC_INDEX_LOOKUP = 1<<2, /* lookup fid */ CAPA_OPC_INDEX_INSERT = 1<<3, /* insert fid */ CAPA_OPC_INDEX_DELETE = 1<<4, /* delete fid */ - /* OSS only fid capability */ CAPA_OPC_OSS_WRITE = 1<<5, /* write oss object data */ CAPA_OPC_OSS_READ = 1<<6, /* read oss object data */ CAPA_OPC_OSS_TRUNC = 1<<7, /* truncate oss object */ - /* MDS & OSS both might have */ CAPA_OPC_META_WRITE = 1<<8, /* write fid meta data */ CAPA_OPC_META_READ = 1<<9, /* read fid meta data */ }; -#define CAPA_OPC_MDS_ONLY \ - (CAPA_OPC_BODY_WRITE | CAPA_OPC_BODY_READ | \ - CAPA_OPC_INDEX_LOOKUP | CAPA_OPC_INDEX_INSERT | CAPA_OPC_INDEX_DELETE) -#define CAPA_OPC_OSS_ONLY \ +#define CAPA_OPC_MDS_ONLY \ + (CAPA_OPC_BODY_WRITE | CAPA_OPC_BODY_READ | CAPA_OPC_INDEX_LOOKUP | \ + CAPA_OPC_INDEX_INSERT | CAPA_OPC_INDEX_DELETE) +#define CAPA_OPC_OSS_ONLY \ (CAPA_OPC_OSS_WRITE | CAPA_OPC_OSS_READ | CAPA_OPC_OSS_TRUNC) #define CAPA_OPC_MDS_DEFAULT ~CAPA_OPC_OSS_ONLY #define CAPA_OPC_OSS_DEFAULT ~(CAPA_OPC_MDS_ONLY | CAPA_OPC_OSS_ONLY) diff --git a/lustre/include/lustre_capa.h b/lustre/include/lustre_capa.h index 329cebc..a37fe7c 100644 --- a/lustre/include/lustre_capa.h +++ b/lustre/include/lustre_capa.h @@ -55,11 +55,10 @@ struct capa_hmac_alg { struct client_capa { struct inode *inode; struct list_head lli_list; /* link to lli_oss_capas */ - atomic_t open_count; /* open count */ }; struct target_capa { - struct hlist_node c_hash; /* link to capa hash */ + struct hlist_node c_hash; /* link to capa hash */ }; struct obd_capa { @@ -300,7 +299,7 @@ static inline unsigned long capa_renewal_time(struct obd_capa *ocapa) * to OSS before capability expires. */ return ocapa->c_expiry - - ((ocapa->c_capa.lc_flags & CAPA_FL_SHORT_EXPIRY) ? 40:1200) * HZ; + ((ocapa->c_capa.lc_flags & CAPA_FL_SHORT_EXPIRY) ? 300:1200) * HZ; } #ifdef __KERNEL__ diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 4cf1906..d994cb3 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -182,7 +182,7 @@ struct md_object_operations { int (*moo_close)(const struct lu_env *env, struct md_object *obj, struct md_attr *ma); int (*moo_capa_get)(const struct lu_env *, struct md_object *, - struct lustre_capa *); + struct lustre_capa *, int renewal); }; /* @@ -437,10 +437,11 @@ static inline int mo_ref_del(const struct lu_env *env, static inline int mo_capa_get(const struct lu_env *env, struct md_object *m, - struct lustre_capa *c) + struct lustre_capa *c, + int renewal) { LASSERT(m->mo_ops->moo_capa_get); - return m->mo_ops->moo_capa_get(env, m, c); + return m->mo_ops->moo_capa_get(env, m, c, renewal); } static inline int mdo_lookup(const struct lu_env *env, diff --git a/lustre/llite/dcache.c b/lustre/llite/dcache.c index 9606726..9ba0b80 100644 --- a/lustre/llite/dcache.c +++ b/lustre/llite/dcache.c @@ -639,7 +639,7 @@ do_lookup: unlock_kernel(); handle = (flag) ? &ldd->lld_mnt_och : &ldd->lld_cwd_och; - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = obd_pin(sbi->ll_md_exp, ll_inode2fid(inode), oc, handle, flag); capa_put(oc); if (rc) { diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index 37dc3fd..9d15b41 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -151,7 +151,7 @@ static int ll_dir_readpage(struct file *file, struct page *page) CDEBUG(D_VFSTRACE, "VFS Op:inode=%lu/%u(%p) off %lu\n", inode->i_ino, inode->i_generation, inode, (unsigned long)hash); - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_readpage(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), oc, hash, page, &request); capa_put(oc); @@ -596,7 +596,7 @@ static int ll_dir_ioctl(struct inode *inode, struct file *file, GOTO(out, rc = -EINVAL); } - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getattr_name(sbi->ll_md_exp, ll_inode2fid(inode), oc, filename, namelen, OBD_MD_FLID, 0, &request); @@ -675,7 +675,7 @@ static int ll_dir_ioctl(struct inode *inode, struct file *file, if (IS_ERR(filename)) RETURN(PTR_ERR(filename)); - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getattr_name(sbi->ll_md_exp, ll_inode2fid(inode), oc, filename, strlen(filename) + 1, @@ -688,7 +688,7 @@ static int ll_dir_ioctl(struct inode *inode, struct file *file, GOTO(out_name, rc); } } else { - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getattr(sbi->ll_md_exp, ll_inode2fid(inode), oc, OBD_MD_FLEASIZE | OBD_MD_FLDIREA, lmmsize, &request); diff --git a/lustre/llite/file.c b/lustre/llite/file.c index bfb45c4..100f956 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -61,7 +61,7 @@ void ll_pack_inode2opdata(struct inode *inode, struct md_op_data *op_data, ((struct ll_iattr *)&op_data->attr)->ia_attr_flags = inode->i_flags; op_data->ioepoch = ll_i2info(inode)->lli_ioepoch; memcpy(&op_data->handle, fh, sizeof(op_data->handle)); - op_data->mod_capa1 = ll_i2mdscapa(inode); + op_data->mod_capa1 = ll_mdscapa_get(inode); } static void ll_prepare_close(struct inode *inode, struct md_op_data *op_data, @@ -268,7 +268,7 @@ int ll_md_close(struct obd_export *md_exp, struct inode *inode, LUSTRE_FPRIVATE(file) = NULL; ll_file_data_put(fd); - ll_oss_capa_close(inode, file); + ll_capa_close(inode); RETURN(rc); } @@ -577,7 +577,7 @@ int ll_file_open(struct inode *inode, struct file *file) if (!S_ISREG(inode->i_mode)) GOTO(out, rc); - ll_oss_capa_open(inode, file); + ll_capa_open(inode); lsm = lli->lli_smd; if (lsm == NULL) { @@ -628,7 +628,7 @@ int ll_inode_getattr(struct inode *inode, struct obdo *obdo) OBD_MD_FLSIZE | OBD_MD_FLBLOCKS | OBD_MD_FLBLKSZ | OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLGROUP; - oinfo.oi_capa = ll_i2mdscapa(inode); + oinfo.oi_capa = ll_mdscapa_get(inode); set = ptlrpc_prep_set(); if (set == NULL) { @@ -2229,7 +2229,7 @@ int ll_fsync(struct file *file, struct dentry *dentry, int data) rc = err; } - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); err = md_sync(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), oc, &req); capa_put(oc); @@ -2251,7 +2251,7 @@ int ll_fsync(struct file *file, struct dentry *dentry, int data) OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLGROUP); - ocapa = ll_lookup_oss_capa(inode, CAPA_OPC_OSS_WRITE); + ocapa = ll_osscapa_get(inode, CAPA_OPC_OSS_WRITE); err = obd_sync(ll_i2sbi(inode)->ll_dt_exp, oa, lsm, 0, OBD_OBJECT_EOF, ocapa); capa_put(ocapa); @@ -2466,7 +2466,7 @@ int ll_inode_revalidate_it(struct dentry *dentry, struct lookup_intent *it) RETURN(rc); valid |= OBD_MD_FLEASIZE | OBD_MD_FLMODEASIZE; } - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getattr(sbi->ll_md_exp, ll_inode2fid(inode), oc, valid, ealen, &req); capa_put(oc); diff --git a/lustre/llite/llite_capa.c b/lustre/llite/llite_capa.c index 729039e..25da02f 100644 --- a/lustre/llite/llite_capa.c +++ b/lustre/llite/llite_capa.c @@ -108,18 +108,10 @@ static void sort_add_capa(struct obd_capa *ocapa, struct list_head *head) list_add(&ocapa->c_list, before ?: head); } -static int inode_have_md_lock(struct inode *inode, __u64 inodebits) +static inline int obd_capa_open_count(struct obd_capa *oc) { - struct obd_export *exp = ll_i2mdexp(inode); - ldlm_policy_data_t policy = { .l_inodebits = {inodebits}}; - struct lustre_handle lockh; - int flags = LDLM_FL_BLOCK_GRANTED|LDLM_FL_CBPENDING|LDLM_FL_TEST_LOCK; - int rc; - ENTRY; - - rc = md_lock_match(exp, flags, ll_inode2fid(inode), - LDLM_IBITS, &policy, LCK_CR|LCK_CW|LCK_PR, &lockh); - RETURN(rc); + struct ll_inode_info *lli = ll_i2info(oc->u.cli.inode); + return atomic_read(&lli->lli_open_count); } static void ll_delete_capa(struct obd_capa *ocapa) @@ -127,7 +119,6 @@ static void ll_delete_capa(struct obd_capa *ocapa) struct ll_inode_info *lli = ll_i2info(ocapa->u.cli.inode); if (capa_for_mds(&ocapa->c_capa)) { - capa_put(ocapa); LASSERT(lli->lli_mds_capa == ocapa); lli->lli_mds_capa = NULL; } else if (capa_for_oss(&ocapa->c_capa)) { @@ -139,6 +130,19 @@ static void ll_delete_capa(struct obd_capa *ocapa) free_capa(ocapa); } +static void inline assert_expired_capa(struct obd_capa *oc) +{ + if (S_ISDIR(oc->u.cli.inode->i_mode)) + DEBUG_CAPA(D_ERROR, &oc->c_capa, "shouldn't release dir"); + else if (obd_capa_open_count(oc)) + DEBUG_CAPA(D_ERROR, &oc->c_capa, "shouldn't release opened"); + else if (obd_capa_is_root(oc)) + DEBUG_CAPA(D_ERROR, &oc->c_capa, "shouldn't release root"); + else + return; + LBUG(); +} + /* three places where client capa is deleted: * 1. capa_thread_main(), main place to delete expired capa. * 2. ll_clear_inode_capas() in ll_clear_inode(). @@ -176,10 +180,14 @@ static int capa_thread_main(void *unused) } if (capa_for_mds(&ocapa->c_capa) && + !S_ISDIR(ocapa->u.cli.inode->i_mode) && + obd_capa_open_count(ocapa) == 0 && + !obd_capa_is_root(ocapa) && !ll_have_md_lock(ocapa->u.cli.inode, - MDS_INODELOCK_LOOKUP) && - !obd_capa_is_root(ocapa)) { - /* MDS capa without LOOKUP lock won't renew, + MDS_INODELOCK_LOOKUP)) { + assert_expired_capa(ocapa); + /* MDS capa without LOOKUP lock, and the related + * inode is not opened, it won't renew, * move to idle list (except root fid) */ DEBUG_CAPA(D_SEC, &ocapa->c_capa, "skip renewal for"); @@ -189,8 +197,9 @@ static int capa_thread_main(void *unused) } if (capa_for_oss(&ocapa->c_capa) && - atomic_read(&ocapa->u.cli.open_count) == 0) { - /* oss capa with open_count == 0 won't renew, + obd_capa_open_count(ocapa) == 0) { + assert_expired_capa(ocapa); + /* oss capa with open count == 0 won't renew, * move to idle list */ list_del_init(&ocapa->c_list); sort_add_capa(ocapa, &ll_idle_capas); @@ -213,16 +222,17 @@ static int capa_thread_main(void *unused) rc = md_renew_capa(ll_i2mdexp(inode), ocapa, ll_update_capa); spin_lock(&capa_lock); - if (rc) + if (rc) { + DEBUG_CAPA(D_ERROR, &ocapa->c_capa, + "renew failed: %d", rc); sort_add_capa(ocapa, &ll_idle_capas); + } } if (next) update_capa_timer(next, capa_renewal_time(next)); list_for_each_entry_safe(ocapa, tmp, &ll_idle_capas, c_list) { - LASSERT(atomic_read(&ocapa->u.cli.open_count) == 0); - if (!capa_is_expired(ocapa)) { if (!next) update_capa_timer(ocapa, ocapa->c_expiry); @@ -306,7 +316,7 @@ static struct obd_capa *do_lookup_oss_capa(struct inode *inode, int opc) return NULL; } -struct obd_capa *ll_lookup_oss_capa(struct inode *inode, __u64 opc) +struct obd_capa *ll_osscapa_get(struct inode *inode, __u64 opc) { struct ll_inode_info *lli = ll_i2info(inode); struct obd_capa *ocapa; @@ -359,7 +369,7 @@ struct obd_capa *ll_lookup_oss_capa(struct inode *inode, __u64 opc) RETURN(ocapa); } -struct obd_capa *ll_i2mdscapa(struct inode *inode) +struct obd_capa *ll_mdscapa_get(struct inode *inode) { struct ll_inode_info *lli = ll_i2info(inode); struct obd_capa *ocapa; @@ -375,7 +385,8 @@ struct obd_capa *ll_i2mdscapa(struct inode *inode) spin_unlock(&capa_lock); if (ocapa && !obd_capa_is_valid(ocapa)) { - DEBUG_CAPA(D_ERROR, &ocapa->c_capa, "invalid"); + DEBUG_CAPA(D_ERROR, &ocapa->c_capa, "invalid (flags %d)", + ocapa->c_flags); capa_put(ocapa); ocapa = NULL; } @@ -384,8 +395,12 @@ struct obd_capa *ll_i2mdscapa(struct inode *inode) CDEBUG(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ? D_ERROR : D_SEC, "no MDS capability for fid "DFID"\n", PFID(ll_inode2fid(inode))); - if (inode_have_md_lock(inode, MDS_INODELOCK_LOOKUP)) - LBUG(); +#if 0 + LASSERT(!S_ISDIR(inode->i_mode)); + LASSERT(!obd_capa_open_count(ocapa)); + LASSERT(!ll_have_md_lock(ocapa->u.cli.inode, + MDS_INODELOCK_LOOKUP)); +#endif atomic_set(&ll_capa_debug, 0); } @@ -401,7 +416,7 @@ static inline int do_add_mds_capa(struct inode *inode, struct obd_capa **pcapa) if (!old) { ocapa->u.cli.inode = inode; - lli->lli_mds_capa = capa_get(ocapa); + lli->lli_mds_capa = ocapa; obd_capa_clear_new(ocapa); obd_capa_set_valid(ocapa); @@ -457,7 +472,6 @@ static inline int do_add_oss_capa(struct inode *inode, struct obd_capa **pcapa) old = do_lookup_oss_capa(inode, capa->lc_opc & CAPA_OPC_OSS_ONLY); if (!old) { ocapa->u.cli.inode = inode; - atomic_set(&ocapa->u.cli.open_count, 0); INIT_LIST_HEAD(&ocapa->u.cli.lli_list); obd_capa_set_valid(ocapa); @@ -489,8 +503,8 @@ struct obd_capa *ll_add_capa(struct inode *inode, struct obd_capa *ocapa) int rc; spin_lock(&capa_lock); - rc = capa_for_mds(&ocapa->c_capa) ? do_add_mds_capa(inode, pcapa) : - do_add_oss_capa(inode, pcapa); + rc = capa_for_mds(&ocapa->c_capa) ? do_add_mds_capa(inode, pcapa) : + do_add_oss_capa(inode, pcapa); ocapa = *pcapa; /* truncate capa won't renew, or no existed capa changed, don't update @@ -526,8 +540,8 @@ int ll_update_capa(struct obd_capa *ocapa, struct lustre_capa *capa) /* failed capa won't be renewed any longer, but if -EIO, client * might be doing recovery, retry in 1 min. */ spin_lock(&capa_lock); - if (rc == -EIO) { - expiry = cfs_time_current() + cfs_time_seconds(60); + if (rc == -EIO && !capa_is_expired(ocapa)) { + expiry = jiffies + 60 * HZ; DEBUG_CAPA(D_SEC, &ocapa->c_capa, "renewal failed: -EIO, retry in 1 min"); goto retry; @@ -566,57 +580,32 @@ out: return rc; } -void ll_oss_capa_open(struct inode *inode, struct file *file) +void ll_capa_open(struct inode *inode) { - struct obd_capa *ocapa; - int opc = capa_open_opc(open_flags_to_accmode(file->f_flags)); + struct ll_inode_info *lli = ll_i2info(inode); - if ((ll_i2sbi(inode)->ll_flags & LL_SBI_OSS_CAPA) == 0) + if ((ll_i2sbi(inode)->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA)) + == 0) return; if (!S_ISREG(inode->i_mode)) return; - spin_lock(&capa_lock); - ocapa = do_lookup_oss_capa(inode, opc); - if (!ocapa) { - if (atomic_read(&ll_capa_debug)) { - CDEBUG(D_ERROR, "no opc %x capability for fid "DFID"\n", - opc, PFID(ll_inode2fid(inode))); - atomic_set(&ll_capa_debug, 0); - } - spin_unlock(&capa_lock); - return; - } - atomic_inc(&ocapa->u.cli.open_count); - spin_unlock(&capa_lock); - - DEBUG_CAPA(D_SEC, &ocapa->c_capa, "open (count: %d)", - atomic_read(&ocapa->u.cli.open_count)); + atomic_inc(&lli->lli_open_count); } -void ll_oss_capa_close(struct inode *inode, struct file *file) +void ll_capa_close(struct inode *inode) { - struct obd_capa *ocapa; - int opc = capa_open_opc(open_flags_to_accmode(file->f_flags)); + struct ll_inode_info *lli = ll_i2info(inode); - if ((ll_i2sbi(inode)->ll_flags & LL_SBI_OSS_CAPA) == 0) + if ((ll_i2sbi(inode)->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA)) + == 0) return; if (!S_ISREG(inode->i_mode)) return; - spin_lock(&capa_lock); - ocapa = do_lookup_oss_capa(inode, opc); - if (!ocapa) { - spin_unlock(&capa_lock); - return; - } - atomic_dec(&ocapa->u.cli.open_count); - spin_unlock(&capa_lock); - - DEBUG_CAPA(D_SEC, &ocapa->c_capa, "close (count: %d)", - atomic_read(&ocapa->u.cli.open_count)); + atomic_dec(&lli->lli_open_count); } /* delete CAPA_OPC_OSS_TRUNC only */ diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 2c19a92..19f10a1 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -137,8 +137,10 @@ struct ll_inode_info { struct lov_stripe_md *lli_smd; /* fid capability */ + /* open count currently used by capability only, indicate whether + * capability needs renewal */ + atomic_t lli_open_count; struct obd_capa *lli_mds_capa; - /* oss capability list */ struct list_head lli_oss_capas; }; @@ -746,13 +748,17 @@ extern cfs_timer_t ll_capa_timer; int ll_capa_thread_start(void); void ll_capa_thread_stop(void); void ll_capa_timer_callback(unsigned long unused); -struct obd_capa *ll_lookup_oss_capa(struct inode *inode, __u64 opc); + struct obd_capa *ll_add_capa(struct inode *inode, struct obd_capa *ocapa); -void ll_oss_capa_open(struct inode *inode, struct file *file); -void ll_oss_capa_close(struct inode *inode, struct file *file); int ll_update_capa(struct obd_capa *ocapa, struct lustre_capa *capa); + +void ll_capa_open(struct inode *inode); +void ll_capa_close(struct inode *inode); + +struct obd_capa *ll_mdscapa_get(struct inode *inode); +struct obd_capa *ll_osscapa_get(struct inode *inode, __u64 opc); + void ll_truncate_free_capa(struct obd_capa *ocapa); void ll_clear_inode_capas(struct inode *inode); -struct obd_capa *ll_i2mdscapa(struct inode *inode); #endif /* LLITE_INTERNAL_H */ diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index ded9328..1f0cbcd 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -783,9 +783,9 @@ void ll_lli_init(struct ll_inode_info *lli) lli->lli_open_fd_read_count = lli->lli_open_fd_write_count = 0; lli->lli_open_fd_exec_count = 0; INIT_LIST_HEAD(&lli->lli_dead_list); - INIT_LIST_HEAD(&lli->lli_oss_capas); lli->lli_remote_perms = NULL; sema_init(&lli->lli_rmtperm_sem, 1); + INIT_LIST_HEAD(&lli->lli_oss_capas); } /* COMPAT_146 */ @@ -1532,7 +1532,7 @@ int ll_setattr_raw(struct inode *inode, struct iattr *attr) oinfo.oi_oa = oa; oinfo.oi_md = lsm; - oinfo.oi_capa = ll_i2mdscapa(inode); + oinfo.oi_capa = ll_mdscapa_get(inode); /* XXX: this looks unnecessary now. */ rc = obd_setattr_rqset(sbi->ll_dt_exp, &oinfo, NULL); @@ -1923,7 +1923,7 @@ int ll_iocontrol(struct inode *inode, struct file *file, struct mdt_body *body; struct obd_capa *oc; - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getattr(sbi->ll_md_exp, ll_inode2fid(inode), oc, OBD_MD_FLFLAGS, 0, &req); capa_put(oc); @@ -2259,13 +2259,13 @@ ll_prep_md_op_data(struct md_op_data *op_data, struct inode *i1, ll_i2gids(op_data->suppgids, i1, i2); op_data->fid1 = ll_i2info(i1)->lli_fid; - op_data->mod_capa1 = ll_i2mdscapa(i1); + op_data->mod_capa1 = ll_mdscapa_get(i1); /* @i2 may be NULL. In this case caller itself has to initialize ->fid2 * if needed. */ if (i2) { op_data->fid2 = *ll_inode2fid(i2); - op_data->mod_capa2 = ll_i2mdscapa(i2); + op_data->mod_capa2 = ll_mdscapa_get(i2); } op_data->name = name; @@ -2303,7 +2303,7 @@ int ll_ioctl_getfacl(struct inode *inode, struct rmtacl_ioctl_data *ioc) if (copy_from_user(cmd, ioc->cmd, ioc->cmd_len)) GOTO(out, rc = -EFAULT); - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getxattr(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), oc, OBD_MD_FLXATTR, XATTR_NAME_LUSTRE_ACL, cmd, ioc->cmd_len, ioc->res_len, 0, &req); @@ -2347,7 +2347,7 @@ int ll_ioctl_setfacl(struct inode *inode, struct rmtacl_ioctl_data *ioc) if (copy_from_user(cmd, ioc->cmd, ioc->cmd_len)) GOTO(out, rc = -EFAULT); - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_setxattr(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), oc, OBD_MD_FLXATTR, XATTR_NAME_LUSTRE_ACL, cmd, ioc->cmd_len, ioc->res_len, 0, &req); diff --git a/lustre/llite/llite_nfs.c b/lustre/llite/llite_nfs.c index 62845d9..0e22950 100644 --- a/lustre/llite/llite_nfs.c +++ b/lustre/llite/llite_nfs.c @@ -282,7 +282,7 @@ int ll_encode_fh(struct dentry *de, __u32 *fh, int *plen, int connectable) { struct inode *inode = de->d_inode; struct lu_fid *fid = ll_inode2fid(inode); - struct obd_capa *ocapa = ll_i2mdscapa(inode); + struct obd_capa *ocapa = ll_mdscapa_get(inode); int len = (sizeof(*fid) + sizeof(struct lustre_capa) + 3)/4; char *p = (char *)fh; @@ -328,7 +328,7 @@ struct dentry *ll_get_parent(struct dentry *dchild) sbi = ll_s2sbi(dir->i_sb); - oc = ll_i2mdscapa(dir); + oc = ll_mdscapa_get(dir); rc = md_getattr_name(sbi->ll_md_exp, ll_inode2fid(dir), oc, dotdot, strlen(dotdot) + 1, 0, 0, &req); if (rc) { diff --git a/lustre/llite/remote_perm.c b/lustre/llite/remote_perm.c index 0b2eabf..d472951 100644 --- a/lustre/llite/remote_perm.c +++ b/lustre/llite/remote_perm.c @@ -254,7 +254,7 @@ check: LBUG(); } - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_get_remote_perm(sbi->ll_md_exp, ll_inode2fid(inode), oc, &req); capa_put(oc); if (rc) { diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c index 7e1de81..0346c61 100644 --- a/lustre/llite/rw.c +++ b/lustre/llite/rw.c @@ -99,7 +99,7 @@ static int ll_brw(int cmd, struct inode *inode, struct obdo *oa, /* NB partial write, so we might not have CAPA_OPC_OSS_READ capa */ opc = cmd & OBD_BRW_WRITE ? CAPA_OPC_OSS_WRITE : CAPA_OPC_OSS_WRITE | CAPA_OPC_OSS_READ; - oinfo.oi_capa = ll_lookup_oss_capa(inode, opc); + oinfo.oi_capa = ll_osscapa_get(inode, opc); rc = obd_brw(cmd, ll_i2dtexp(inode), &oinfo, 1, &pg, NULL); capa_put(oinfo.oi_capa); if (rc == 0) @@ -187,7 +187,7 @@ void ll_truncate(struct inode *inode) ll_inode_size_unlock(inode, 0); - oinfo.oi_capa = ll_lookup_oss_capa(inode, CAPA_OPC_OSS_TRUNC); + oinfo.oi_capa = ll_osscapa_get(inode, CAPA_OPC_OSS_TRUNC); rc = obd_punch_rqset(ll_i2dtexp(inode), &oinfo, NULL); ll_truncate_free_capa(oinfo.oi_capa); if (rc) @@ -416,7 +416,7 @@ static struct obd_capa *ll_ap_lookup_capa(void *data, int cmd) int opc = cmd & OBD_BRW_WRITE ? CAPA_OPC_OSS_WRITE : CAPA_OPC_OSS_WRITE | CAPA_OPC_OSS_READ; - return ll_lookup_oss_capa(llap->llap_page->mapping->host, opc); + return ll_osscapa_get(llap->llap_page->mapping->host, opc); } static struct obd_async_page_ops ll_async_page_ops = { diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index 10e4418..2ea27a0 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -175,7 +175,7 @@ static ssize_t ll_direct_IO_26_seg(int rw, struct file *file, LPROC_LL_DIRECT_READ, size); opc = CAPA_OPC_OSS_READ | CAPA_OPC_OSS_WRITE; } - ocapa = ll_lookup_oss_capa(inode, opc); + ocapa = ll_osscapa_get(inode, opc); rc = obd_brw_rqset(rw == WRITE ? OBD_BRW_WRITE : OBD_BRW_READ, ll_i2dtexp(inode), &oa, lsm, page_count, pga, NULL, ocapa); diff --git a/lustre/llite/symlink.c b/lustre/llite/symlink.c index b23d6b8..d95369c 100644 --- a/lustre/llite/symlink.c +++ b/lustre/llite/symlink.c @@ -47,7 +47,7 @@ static int ll_readlink_internal(struct inode *inode, RETURN(0); } - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getattr(sbi->ll_md_exp, ll_inode2fid(inode), oc, OBD_MD_LINKNAME, symlen, request); capa_put(oc); diff --git a/lustre/llite/xattr.c b/lustre/llite/xattr.c index 6038ef8..a4daa63 100644 --- a/lustre/llite/xattr.c +++ b/lustre/llite/xattr.c @@ -125,7 +125,7 @@ int ll_setxattr_common(struct inode *inode, const char *name, if (xattr_type == XATTR_TRUSTED_T && strcmp(name, "trusted.lov") == 0) RETURN(0); - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_setxattr(sbi->ll_md_exp, ll_inode2fid(inode), oc, valid, name, value, size, 0, flags, &req); capa_put(oc); @@ -225,7 +225,7 @@ int ll_getxattr_common(struct inode *inode, const char *name, #endif do_getxattr: - oc = ll_i2mdscapa(inode); + oc = ll_mdscapa_get(inode); rc = md_getxattr(sbi->ll_md_exp, ll_inode2fid(inode), oc, valid, name, NULL, 0, size, 0, &req); capa_put(oc); diff --git a/lustre/lov/lov_request.c b/lustre/lov/lov_request.c index 95975d6..6bc5fd5 100644 --- a/lustre/lov/lov_request.c +++ b/lustre/lov/lov_request.c @@ -1400,8 +1400,6 @@ int lov_prep_sync_set(struct obd_export *exp, struct obd_info *oinfo, req->rq_oi.oi_policy.l_extent.end = re; req->rq_oi.oi_policy.l_extent.gid = -1; - req->rq_oi.oi_capa = oinfo->oi_capa; - lov_set_add_req(req, set); } if (!set->set_count) diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index d344ab3..0404517 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1616,9 +1616,11 @@ static int mdc_interpret_renew_capa(struct ptlrpc_request *req, void *unused, struct lustre_capa *capa; ENTRY; - if (status) - DEBUG_CAPA(D_ERROR, &oc->c_capa, "renew failed: %d for", - status); + if (status) { + DEBUG_CAPA(status == -ENOENT ? D_SEC : D_ERROR, &oc->c_capa, + "renew failed: %d for", status); + GOTO(out, capa = ERR_PTR(status)); + } body = lustre_swab_repbuf(req, REPLY_REC_OFF, sizeof(*body), lustre_swab_mdt_body); diff --git a/lustre/mdd/mdd_handler.c b/lustre/mdd/mdd_handler.c index d14d5fc..7daf4d3 100644 --- a/lustre/mdd/mdd_handler.c +++ b/lustre/mdd/mdd_handler.c @@ -3345,23 +3345,27 @@ static int mdd_permission(const struct lu_env *env, struct md_object *obj, } static int mdd_capa_get(const struct lu_env *env, struct md_object *obj, - struct lustre_capa *capa) + struct lustre_capa *capa, int renewal) { struct dt_object *next; struct mdd_object *mdd_obj = md2mdd_obj(obj); struct obd_capa *oc; + int rc = 0; ENTRY; LASSERT(lu_object_exists(mdd2lu_obj(mdd_obj))); next = mdd_object_child(mdd_obj); - oc = next->do_ops->do_capa_get(env, next, capa->lc_opc); - if (oc) { + oc = next->do_ops->do_capa_get(env, next, renewal ? capa : NULL, + capa->lc_opc); + if (IS_ERR(oc)) { + rc = PTR_ERR(oc); + } else { capa_cpy(capa, oc); capa_put(oc); } - RETURN(0); + RETURN(rc); } struct md_device_operations mdd_ops = { diff --git a/lustre/mdd/mdd_lov.c b/lustre/mdd/mdd_lov.c index f0fc2b4..6bbae1c 100644 --- a/lustre/mdd/mdd_lov.c +++ b/lustre/mdd/mdd_lov.c @@ -507,7 +507,7 @@ int mdd_lov_create(const struct lu_env *env, struct mdd_device *mdd, oa->o_valid |= OBD_MD_FLFID | OBD_MD_FLGENER; oinfo->oi_oa = oa; oinfo->oi_md = lsm; - oinfo->oi_capa = next->do_ops->do_capa_get(env, next, + oinfo->oi_capa = next->do_ops->do_capa_get(env, next, NULL, CAPA_OPC_MDS_DEFAULT); rc = obd_setattr(lov_exp, oinfo, oti); @@ -580,7 +580,7 @@ int mdd_lov_setattr_async(const struct lu_env *env, struct mdd_object *obj, if (rc) RETURN(rc); - oc = next->do_ops->do_capa_get(env, next, CAPA_OPC_MDS_DEFAULT); + oc = next->do_ops->do_capa_get(env, next, NULL, CAPA_OPC_MDS_DEFAULT); rc = mds_osc_setattr_async(obd, tmp_la->la_uid, tmp_la->la_gid, lmm, lmm_size, NULL, seq, oid, oc); capa_put(oc); diff --git a/lustre/mdt/mdt_capa.c b/lustre/mdt/mdt_capa.c index 384db5a..37b870e 100644 --- a/lustre/mdt/mdt_capa.c +++ b/lustre/mdt/mdt_capa.c @@ -194,7 +194,9 @@ static int mdt_ck_thread_main(void *args) { struct mdt_device *mdt = args; struct ptlrpc_thread *thread = &mdt->mdt_ck_thread; - struct lustre_capa_key *tmp, *key = &mdt->mdt_capa_keys[1]; + struct lustre_capa_key *bkey = &mdt->mdt_capa_keys[0], + *rkey = &mdt->mdt_capa_keys[1]; + struct lustre_capa_key *tmp; struct lu_env env; struct mdt_thread_info *info; struct md_device *next; @@ -233,26 +235,30 @@ static int mdt_ck_thread_main(void *args) if (time_after(mdt->mdt_ck_expiry, jiffies)) break; - *tmp = *key; - make_capa_key(tmp, mdsnum, key->lk_keyid); + *tmp = *rkey; + make_capa_key(tmp, mdsnum, rkey->lk_keyid); next = mdt->mdt_child; rc = next->md_ops->mdo_update_capa_key(&env, next, tmp); if (!rc) { + spin_lock(&capa_lock); + *bkey = *rkey; + *rkey = *tmp; + spin_unlock(&capa_lock); + rc = write_capa_keys(&env, mdt, mdt->mdt_capa_keys); - if (!rc) { + if (rc) { spin_lock(&capa_lock); - mdt->mdt_capa_keys[0] = *key; - *key = *tmp; + *rkey = *bkey; + memset(bkey, 0, sizeof(*bkey)); spin_unlock(&capa_lock); - + } else { set_capa_key_expiry(mdt); - - DEBUG_CAPA_KEY(D_SEC, key, "new"); + DEBUG_CAPA_KEY(D_SEC, rkey, "new"); } } if (rc) { - DEBUG_CAPA_KEY(D_ERROR, key, "update failed for"); + DEBUG_CAPA_KEY(D_ERROR, rkey, "update failed for"); /* next retry is in 300 sec */ mdt->mdt_ck_expiry = jiffies + 300 * HZ; } diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 4564401..659e397 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -185,7 +185,8 @@ static int mdt_getstatus(struct mdt_thread_info *info) LASSERT(capa); capa->lc_opc = CAPA_OPC_MDS_DEFAULT; - rc = mo_capa_get(info->mti_env, mdt_object_child(root), capa); + rc = mo_capa_get(info->mti_env, mdt_object_child(root), capa, + 0); mdt_object_put(info->mti_env, root); if (rc == 0) body->valid |= OBD_MD_FLMDSCAPA; @@ -413,7 +414,7 @@ static int mdt_getattr_internal(struct mdt_thread_info *info, capa = req_capsule_server_get(&info->mti_pill, &RMF_CAPA1); LASSERT(capa); capa->lc_opc = CAPA_OPC_MDS_DEFAULT; - rc = mo_capa_get(env, next, capa); + rc = mo_capa_get(env, next, capa, 0); if (rc) RETURN(rc); repbody->valid |= OBD_MD_FLMDSCAPA; @@ -431,14 +432,17 @@ static int mdt_renew_capa(struct mdt_thread_info *info) int rc; ENTRY; + /* NB: see mdt_unpack_req_pack_rep */ + if (!obj) + GOTO(out, rc = -ENOENT); + body = req_capsule_server_get(&info->mti_pill, &RMF_MDT_BODY); LASSERT(body); c = req_capsule_client_get(&info->mti_pill, &RMF_CAPA1); LASSERT(c); - if ((capa_for_mds(c) && !mdt->mdt_opts.mo_mds_capa) || - (capa_for_oss(c) && !mdt->mdt_opts.mo_oss_capa)) { + if (!mdt->mdt_opts.mo_mds_capa) { DEBUG_CAPA(D_SEC, c, "mds has disabled capability, skip renew for"); GOTO(out, rc = -ENOENT); @@ -448,8 +452,9 @@ static int mdt_renew_capa(struct mdt_thread_info *info) LASSERT(capa); *capa = *c; - capa->lc_expiry = 0; - rc = mo_capa_get(info->mti_env, mdt_object_child(obj), capa); + rc = mo_capa_get(info->mti_env, mdt_object_child(obj), capa, 1); + if (rc) + GOTO(out, rc); body->valid |= OBD_MD_FLOSSCAPA; EXIT; @@ -465,9 +470,6 @@ static int mdt_getattr(struct mdt_thread_info *info) int rc; ENTRY; - LASSERT(obj != NULL); - LASSERT(lu_object_assert_exists(&obj->mot_obj.mo_lu)); - reqbody = req_capsule_client_get(&info->mti_pill, &RMF_MDT_BODY); if (reqbody == NULL) GOTO(out, rc = -EFAULT); @@ -478,6 +480,9 @@ static int mdt_getattr(struct mdt_thread_info *info) RETURN(rc); } + LASSERT(obj != NULL); + LASSERT(lu_object_assert_exists(&obj->mot_obj.mo_lu)); + if (reqbody->valid & OBD_MD_FLRMTPERM) { rc = mdt_init_ucred(info, reqbody); if (rc) @@ -1505,7 +1510,14 @@ static int mdt_body_unpack(struct mdt_thread_info *info, __u32 flags) if ((flags & HABEO_CORPUS) && !mdt_object_exists(obj)) { mdt_object_put(env, obj); - rc = -ENOENT; +#if 0 + /* for capability renew ENOENT will be handled in + * mdt_renew_capa */ + if (body->valid & OBD_MD_FLOSSCAPA) + rc = 0; + else +#endif + rc = -ENOENT; } else { info->mti_object = obj; rc = 0; @@ -1659,7 +1671,7 @@ static int mdt_req_handle(struct mdt_thread_info *info, } } - /* capability setting changed by /proc, needs reinitialize ctxt */ + /* capability setting changed via /proc, needs reinitialize ctxt */ if (info->mti_mdt && info->mti_mdt->mdt_capa_conf) { mdt_init_capa_ctxt(info->mti_env, info->mti_mdt); info->mti_mdt->mdt_capa_conf = 0; @@ -3292,7 +3304,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, spin_lock_init(&m->mdt_ioepoch_lock); m->mdt_opts.mo_compat_resname = 0; - m->mdt_capa_timeout = CAPA_TIMEOUT; + m->mdt_capa_timeout = 320; // CAPA_TIMEOUT; m->mdt_capa_alg = CAPA_HMAC_ALG_SHA1; m->mdt_ck_timeout = CAPA_KEY_TIMEOUT; obd->obd_replayable = 1; @@ -3369,12 +3381,12 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, GOTO(err_free_ns, rc); } - rc = mdt_ck_thread_start(m); - if (rc) - GOTO(err_free_ns, rc); m->mdt_ck_timer.function = mdt_ck_timer_callback; m->mdt_ck_timer.data = (unsigned long)m; init_timer(&m->mdt_ck_timer); + rc = mdt_ck_thread_start(m); + if (rc) + GOTO(err_free_ns, rc); rc = mdt_start_ptlrpc_service(m); if (rc) diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c index fad899d..ac4968c 100644 --- a/lustre/mdt/mdt_open.c +++ b/lustre/mdt/mdt_open.c @@ -351,7 +351,7 @@ static int mdt_mfd_open(struct mdt_thread_info *info, capa = req_capsule_server_get(&info->mti_pill, &RMF_CAPA1); LASSERT(capa); capa->lc_opc = CAPA_OPC_MDS_DEFAULT; - rc = mo_capa_get(info->mti_env, mdt_object_child(o), capa); + rc = mo_capa_get(info->mti_env, mdt_object_child(o), capa, 0); if (rc) RETURN(rc); repbody->valid |= OBD_MD_FLMDSCAPA; @@ -363,7 +363,7 @@ static int mdt_mfd_open(struct mdt_thread_info *info, capa = req_capsule_server_get(&info->mti_pill, &RMF_CAPA2); LASSERT(capa); capa->lc_opc = CAPA_OPC_OSS_DEFAULT | capa_open_opc(flags); - rc = mo_capa_get(info->mti_env, mdt_object_child(o), capa); + rc = mo_capa_get(info->mti_env, mdt_object_child(o), capa, 0); if (rc) RETURN(rc); repbody->valid |= OBD_MD_FLOSSCAPA; diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c index 1cb6f7e..e8844b9 100644 --- a/lustre/mdt/mdt_reint.c +++ b/lustre/mdt/mdt_reint.c @@ -285,12 +285,14 @@ static int mdt_reint_setattr(struct mdt_thread_info *info, if (mdt->mdt_opts.mo_oss_capa && S_ISREG(lu_object_attr(&mo->mot_obj.mo_lu))) { + /* FIXME: only sent truncate capability back in size change + * case */ struct lustre_capa *capa; capa = req_capsule_server_get(&info->mti_pill, &RMF_CAPA1); LASSERT(capa); capa->lc_opc = CAPA_OPC_OSS_DEFAULT | CAPA_OPC_OSS_TRUNC; - rc = mo_capa_get(info->mti_env, mdt_object_child(mo), capa); + rc = mo_capa_get(info->mti_env, mdt_object_child(mo), capa, 0); if (rc) RETURN(rc); repbody->valid |= OBD_MD_FLOSSCAPA; diff --git a/lustre/obdclass/capa.c b/lustre/obdclass/capa.c index f814784..28994f0 100644 --- a/lustre/obdclass/capa.c +++ b/lustre/obdclass/capa.c @@ -122,11 +122,8 @@ static struct obd_capa *find_capa(struct lustre_capa *capa, { struct hlist_node *pos; struct obd_capa *ocapa; - int len = offsetof(struct lustre_capa, lc_hmac); - - /* MDS get capa case */ - if (capa->lc_expiry == 0) - len = offsetof(struct lustre_capa, lc_keyid); + int len = capa->lc_expiry ? sizeof(*capa) : + offsetof(struct lustre_capa, lc_keyid); hlist_for_each_entry(ocapa, pos, head, u.tgt.c_hash) { if (memcmp(&ocapa->c_capa, capa, len)) diff --git a/lustre/obdfilter/filter_capa.c b/lustre/obdfilter/filter_capa.c index b0bb3c7..78306f0 100644 --- a/lustre/obdfilter/filter_capa.c +++ b/lustre/obdfilter/filter_capa.c @@ -39,31 +39,45 @@ static inline __u32 filter_ck_keyid(struct filter_capa_key *key) return key->k_key.lk_keyid; } -int filter_update_capa_key(struct obd_device *obd, struct lustre_capa_key *key) +int filter_update_capa_key(struct obd_device *obd, struct lustre_capa_key *new) { struct filter_obd *filter = &obd->u.filter; - struct filter_capa_key *k, *rkey = NULL, *bkey = NULL; + struct filter_capa_key *k, *keys[2] = { NULL, NULL }; + int i; spin_lock(&capa_lock); list_for_each_entry(k, &filter->fo_capa_keys, k_list) { - if (k->k_key.lk_mdsid != key->lk_mdsid) + if (k->k_key.lk_mdsid != new->lk_mdsid) continue; - if (rkey) - bkey = k; - else - rkey = k; + if (keys[0]) { + keys[1] = k; + if (filter_ck_keyid(keys[1]) > filter_ck_keyid(keys[0])) + keys[1] = keys[0], keys[0] = k; + } else { + keys[0] = k; + } } spin_unlock(&capa_lock); - if (rkey && bkey && filter_ck_keyid(rkey) < filter_ck_keyid(bkey)) { - k = rkey; - rkey = bkey; - bkey = k; + for (i = 0; i < 2; i++) { + if (!keys[i]) + continue; + if (filter_ck_keyid(keys[i]) != new->lk_keyid) + continue; + /* maybe because of recovery or other reasons, MDS sent the + * the old capability key again. + */ + spin_lock(&capa_lock); + keys[i]->k_key = *new; + spin_unlock(&capa_lock); + + RETURN(0); } - if (bkey) { - k = bkey; + if (keys[1]) { + /* if OSS already have two keys, update the old one */ + k = keys[1]; } else { OBD_ALLOC_PTR(k); if (!k) @@ -72,12 +86,12 @@ int filter_update_capa_key(struct obd_device *obd, struct lustre_capa_key *key) } spin_lock(&capa_lock); - k->k_key = *key; + k->k_key = *new; if (list_empty(&k->k_list)) list_add(&k->k_list, &filter->fo_capa_keys); spin_unlock(&capa_lock); - DEBUG_CAPA_KEY(D_SEC, key, "new"); + DEBUG_CAPA_KEY(D_SEC, new, "new"); RETURN(0); } diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index e02419a..1c0cfbc 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -537,8 +537,7 @@ static int osc_sync(struct obd_export *exp, struct obdo *oa, RETURN(-EINVAL); } - if (capa) - size[REQ_REC_OFF + 1] = sizeof(struct lustre_capa); + size[REQ_REC_OFF + 1] = capa ? sizeof(struct lustre_capa) : 0; req = ptlrpc_prep_req(class_exp2cliimp(exp), LUSTRE_OST_VERSION, OST_SYNC, 3, size, NULL); diff --git a/lustre/osd/osd_handler.c b/lustre/osd/osd_handler.c index cf8ee3c..ede82c4 100644 --- a/lustre/osd/osd_handler.c +++ b/lustre/osd/osd_handler.c @@ -837,17 +837,14 @@ static int capa_is_sane(const struct lu_env *env, { struct osd_thread_info *oti; struct obd_capa *oc; - int i, rc = 1; + int i, rc = 0; ENTRY; oti = lu_context_key_get(&env->le_ctx, &osd_key); oc = capa_lookup(capa); if (oc) { - if (memcmp(&oc->c_capa, capa, sizeof(*capa))) { - DEBUG_CAPA(D_ERROR, capa, "HMAC mismatch"); - rc = -EACCES; - } else if (capa_is_expired(oc)) { + if (capa_is_expired(oc)) { DEBUG_CAPA(D_ERROR, capa, "expired"); rc = -ESTALE; } @@ -869,10 +866,11 @@ static int capa_is_sane(const struct lu_env *env, RETURN(-ESTALE); } - rc = capa_hmac(oti->oti_capa_hmac, capa, oti->oti_capa_key.lk_key); + rc = capa_hmac(oti->oti_capa.lc_hmac, capa, oti->oti_capa_key.lk_key); if (rc) RETURN(rc); - if (memcmp(oti->oti_capa_hmac, capa->lc_hmac, sizeof(capa->lc_hmac))) { + if (memcmp(oti->oti_capa.lc_hmac, capa->lc_hmac, sizeof(capa->lc_hmac))) + { DEBUG_CAPA(D_ERROR, capa, "HMAC mismatch"); RETURN(-EACCES); } @@ -880,7 +878,7 @@ static int capa_is_sane(const struct lu_env *env, oc = capa_add(capa); capa_put(oc); - RETURN(1); + RETURN(0); } static int osd_object_auth(const struct lu_env *env, struct dt_object *dt, @@ -888,6 +886,7 @@ static int osd_object_auth(const struct lu_env *env, struct dt_object *dt, { const struct lu_fid *fid = lu_object_fid(&dt->do_lu); struct osd_device *dev = osd_dev(dt->do_lu.lo_dev); + int rc; if (!dev->od_fl_capa) return 0; @@ -911,8 +910,8 @@ static int osd_object_auth(const struct lu_env *env, struct dt_object *dt, return -EACCES; } - if (!capa_is_sane(env, capa, dev->od_capa_keys)) { - DEBUG_CAPA(D_ERROR, capa, "insane"); + if ((rc = capa_is_sane(env, capa, dev->od_capa_keys))) { + DEBUG_CAPA(D_ERROR, capa, "insane (rc %d)", rc); return -EACCES; } @@ -1541,6 +1540,7 @@ static int osd_readpage(const struct lu_env *env, static struct obd_capa *osd_capa_get(const struct lu_env *env, struct dt_object *dt, + struct lustre_capa *old, __u64 opc) { struct osd_thread_info *info = lu_context_key_get(&env->le_ctx, @@ -1555,16 +1555,21 @@ static struct obd_capa *osd_capa_get(const struct lu_env *env, ENTRY; if (!dev->od_fl_capa) - RETURN(NULL); + RETURN(ERR_PTR(-ENOENT)); LASSERT(dt_object_exists(dt)); LASSERT(osd_invariant(obj)); + /* renewal sanity check */ + if (old && osd_object_auth(env, dt, old, opc)) + RETURN(ERR_PTR(-EACCES)); + capa->lc_fid = *fid; capa->lc_opc = opc; capa->lc_flags |= dev->od_capa_alg << 24; if (dev->od_capa_timeout < CAPA_TIMEOUT) capa->lc_flags |= CAPA_FL_SHORT_EXPIRY; + capa->lc_expiry = 0; /* this is flag to get a not-expired one */ oc = capa_lookup(capa); if (oc) { @@ -1582,7 +1587,7 @@ static struct obd_capa *osd_capa_get(const struct lu_env *env, rc = capa_hmac(capa->lc_hmac, capa, key->lk_key); if (rc) { DEBUG_CAPA(D_ERROR, capa, "HMAC failed: %d for", rc); - RETURN(NULL); + RETURN(ERR_PTR(rc)); } oc = capa_add(capa); diff --git a/lustre/osd/osd_internal.h b/lustre/osd/osd_internal.h index 2c9aae9..ec53c65 100644 --- a/lustre/osd/osd_internal.h +++ b/lustre/osd/osd_internal.h @@ -80,7 +80,6 @@ struct osd_thread_info { /* *XXX temporary: for capa operations. */ - char oti_capa_hmac[CAPA_HMAC_KEY_MAX_LEN]; struct lustre_capa_key oti_capa_key; struct lustre_capa oti_capa; }; -- 1.8.3.1