From a120bb13525727654713f008ffcaf0e3aec7cb65 Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Thu, 11 Jul 2019 16:37:55 +0800 Subject: [PATCH] LU-12526 pcc: Auto attach for PCC during IO PCC uses the layout lock to protect the cache validity. Currently PCC only supports auto attach at the next open. However, the layout lock can be revoked at any time by LRU/manual lock shrinking or lock conflict callback. For example, the layout lock can be revoked when performing I/Os after opened the file. At this time, the cached file will be detached involuntary. The I/O originally directed into PCC will redirect to OSTs after the data restore into OSTs' objects. The cost of this unwilling behavior may be expensive. To avoid this problem, this patch implements auto attach for PCC even during IOs (not only at the open time). For debug purpose, now we have three auto attach options: - open_attach: auto attach at the next open; - io_attach: auto attach during IO - stat_attach: auto attach at stat() call. The reason to add the stat_attach option is that: when check PCC state via "lfs pcc state", it will not only open the file but also stat() on the file, to verify the feature of auto attach during IO, we need to both disable open_attach and stat_attach. And all these auto attach options are enabled by default. This patch also fixed the bug for auto cache at create time: In the current Lustre, the truncate operation will revoke the LOOKUP ibits lock, and the file dentry cache will be invalidated. The following open with O_CREAT flag will call into ->atomic_open, the file was wrongly though as newly created file and try to auto cache the file. So after client known it is not a DISP_OPEN_CREATE, it should cleanup the already created PCC copy. Test-Parameters: clientcount=3 testlist=sanity-pcc,sanity-pcc,sanity-pcc Signed-off-by: Qian Yingjin Change-Id: I1e0a84ca125f00076cf88ee26f9b7da8d17a960c Reviewed-on: https://review.whamcloud.com/36005 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Li Xi Reviewed-by: Oleg Drokin --- lustre/include/uapi/linux/lustre/lustre_user.h | 8 ++ lustre/llite/namei.c | 44 +++--- lustre/llite/pcc.c | 159 +++++++++++++++++----- lustre/llite/pcc.h | 45 ++++++- lustre/tests/sanity-pcc.sh | 178 +++++++++++++++++++------ 5 files changed, 339 insertions(+), 95 deletions(-) diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index a71d6f8..d7d8d48 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -2571,6 +2571,14 @@ enum lu_pcc_state_flags { PCC_STATE_FL_ATTACHING = 0x02, /* Allow to auto attach at open */ PCC_STATE_FL_OPEN_ATTACH = 0x04, + /* Allow to auto attach during I/O after layout lock revocation */ + PCC_STATE_FL_IO_ATTACH = 0x08, + /* Allow to auto attach at stat */ + PCC_STATE_FL_STAT_ATTACH = 0x10, + /* Allow to auto attach at the next open or layout refresh */ + PCC_STATE_FL_AUTO_ATTACH = PCC_STATE_FL_OPEN_ATTACH | + PCC_STATE_FL_IO_ATTACH | + PCC_STATE_FL_STAT_ATTACH, }; struct lu_pcc_state { diff --git a/lustre/llite/namei.c b/lustre/llite/namei.c index d097614..3c4d375 100644 --- a/lustre/llite/namei.c +++ b/lustre/llite/namei.c @@ -710,11 +710,6 @@ out: return rc; } -struct pcc_create_attach { - struct pcc_dataset *pca_dataset; - struct dentry *pca_dentry; -}; - static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry, struct lookup_intent *it, void **secctx, __u32 *secctxlen, @@ -965,8 +960,7 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry, void *secctx = NULL; __u32 secctxlen = 0; struct ll_sb_info *sbi; - struct pcc_create_attach pca = {NULL, NULL}; - struct pcc_dataset *dataset = NULL; + struct pcc_create_attach pca = { NULL, NULL }; int rc = 0; ENTRY; @@ -1007,6 +1001,7 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry, if (!filename_is_volatile(dentry->d_name.name, dentry->d_name.len, NULL)) { struct pcc_matcher item; + struct pcc_dataset *dataset; item.pm_uid = from_kuid(&init_user_ns, current_uid()); item.pm_gid = from_kgid(&init_user_ns, current_gid()); @@ -1041,17 +1036,30 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry, dput(de); goto out_release; } - if (dataset != NULL && dentry->d_inode) { - rc = pcc_inode_create_fini(dataset, - dentry->d_inode, - pca.pca_dentry); - if (rc) { - if (de != NULL) - dput(de); - GOTO(out_release, rc); - } + + rc = pcc_inode_create_fini(dentry->d_inode, &pca); + if (rc) { + if (de != NULL) + dput(de); + GOTO(out_release, rc); } + ll_set_created(opened, file); + } else { + /* Open the file with O_CREAT, but the file already + * existed on MDT. This may happend in the case that + * the LOOKUP ibits lock is revoked and the + * corresponding dentry cache is deleted. + * i.e. In the current Lustre, the truncate operation + * will revoke the LOOKUP ibits lock, and the file + * dentry cache will be invalidated. The following open + * with O_CREAT flag will call into ->atomic_open, the + * file was wrongly though as newly created file and + * try to auto cache the file. So after client knows it + * is not a DISP_OPEN_CREATE, it should cleanup the + * already created PCC copy. + */ + pcc_create_attach_cleanup(dir->i_sb, &pca); } if (dentry->d_inode && it_disposition(it, DISP_OPEN_OPEN)) { @@ -1073,11 +1081,11 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry, } else { rc = finish_no_open(file, de); } + } else { + pcc_create_attach_cleanup(dir->i_sb, &pca); } out_release: - if (dataset != NULL) - pcc_dataset_put(dataset); ll_intent_release(it); OBD_FREE(it, sizeof(*it)); diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c index b99c702..e6f9d8f 100644 --- a/lustre/llite/pcc.c +++ b/lustre/llite/pcc.c @@ -459,12 +459,30 @@ pcc_parse_value_pair(struct pcc_cmd *cmd, char *buffer) if (id <= 0) return -EINVAL; cmd->u.pccc_add.pccc_roid = id; + } else if (strcmp(key, "auto_attach") == 0) { + rc = kstrtoul(val, 10, &id); + if (rc) + return rc; + if (id == 0) + cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_AUTO_ATTACH; } else if (strcmp(key, "open_attach") == 0) { rc = kstrtoul(val, 10, &id); if (rc) return rc; - if (id > 0) - cmd->u.pccc_add.pccc_flags |= PCC_DATASET_OPEN_ATTACH; + if (id == 0) + cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_OPEN_ATTACH; + } else if (strcmp(key, "io_attach") == 0) { + rc = kstrtoul(val, 10, &id); + if (rc) + return rc; + if (id == 0) + cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_IO_ATTACH; + } else if (strcmp(key, "stat_attach") == 0) { + rc = kstrtoul(val, 10, &id); + if (rc) + return rc; + if (id == 0) + cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_STAT_ATTACH; } else if (strcmp(key, "rwpcc") == 0) { rc = kstrtoul(val, 10, &id); if (rc) @@ -491,6 +509,18 @@ pcc_parse_value_pairs(struct pcc_cmd *cmd, char *buffer) char *token; int rc; + switch (cmd->pccc_cmd) { + case PCC_ADD_DATASET: + /* Enable auto attach by default */ + cmd->u.pccc_add.pccc_flags |= PCC_DATASET_AUTO_ATTACH; + break; + case PCC_DEL_DATASET: + case PCC_CLEAR_ALL: + break; + default: + return -EINVAL; + } + val = buffer; while (val != NULL && strlen(val) != 0) { token = strsep(&val, " "); @@ -974,7 +1004,6 @@ static void pcc_inode_init(struct pcc_inode *pcci, struct ll_inode_info *lli) { pcci->pcci_lli = lli; lli->lli_pcc_inode = pcci; - lli->lli_pcc_state = PCC_STATE_FL_NONE; atomic_set(&pcci->pcci_refcount, 0); pcci->pcci_type = LU_PCC_NONE; pcci->pcci_layout_gen = CL_LAYOUT_GEN_NONE; @@ -1044,9 +1073,9 @@ void pcc_file_init(struct pcc_file *pccf) pccf->pccf_type = LU_PCC_NONE; } -static inline bool pcc_open_attach_enabled(struct pcc_dataset *dataset) +static inline bool pcc_auto_attach_enabled(struct pcc_dataset *dataset) { - return dataset->pccd_flags & PCC_DATASET_OPEN_ATTACH; + return dataset->pccd_flags & PCC_DATASET_AUTO_ATTACH; } static const char pcc_xattr_layout[] = XATTR_USER_PREFIX "PCC.layout"; @@ -1059,7 +1088,7 @@ static int pcc_layout_xattr_set(struct pcc_inode *pcci, __u32 gen) ENTRY; - if (!(lli->lli_pcc_state & PCC_STATE_FL_OPEN_ATTACH)) + if (!(lli->lli_pcc_state & PCC_STATE_FL_AUTO_ATTACH)) RETURN(0); #ifndef HAVE_VFS_SETXATTR @@ -1121,6 +1150,8 @@ static void pcc_inode_attach_init(struct pcc_dataset *dataset, struct dentry *dentry, enum lu_pcc_type type) { + struct ll_inode_info *lli = pcci->pcci_lli; + pcci->pcci_path.mnt = mntget(dataset->pccd_path.mnt); pcci->pcci_path.dentry = dentry; LASSERT(atomic_read(&pcci->pcci_refcount) == 0); @@ -1128,11 +1159,12 @@ static void pcc_inode_attach_init(struct pcc_dataset *dataset, pcci->pcci_type = type; pcci->pcci_attr_valid = false; - if (pcc_open_attach_enabled(dataset)) { - struct ll_inode_info *lli = pcci->pcci_lli; - + if (dataset->pccd_flags & PCC_DATASET_OPEN_ATTACH) lli->lli_pcc_state |= PCC_STATE_FL_OPEN_ATTACH; - } + if (dataset->pccd_flags & PCC_DATASET_IO_ATTACH) + lli->lli_pcc_state |= PCC_STATE_FL_IO_ATTACH; + if (dataset->pccd_flags & PCC_DATASET_STAT_ATTACH) + lli->lli_pcc_state |= PCC_STATE_FL_STAT_ATTACH; } static inline void pcc_layout_gen_set(struct pcc_inode *pcci, @@ -1244,7 +1276,7 @@ static int pcc_try_datasets_attach(struct inode *inode, __u32 gen, down_read(&super->pccs_rw_sem); list_for_each_entry_safe(dataset, tmp, &super->pccs_datasets, pccd_linkage) { - if (!pcc_open_attach_enabled(dataset)) + if (!pcc_auto_attach_enabled(dataset)) continue; rc = pcc_try_dataset_attach(inode, gen, type, dataset, cached); if (rc < 0 || (!rc && *cached)) @@ -1255,13 +1287,15 @@ static int pcc_try_datasets_attach(struct inode *inode, __u32 gen, RETURN(rc); } -static int pcc_try_open_attach(struct inode *inode, bool *cached) +static int pcc_try_auto_attach(struct inode *inode, bool *cached, bool is_open) { struct pcc_super *super = &ll_i2sbi(inode)->ll_pcc_super; struct cl_layout clt = { .cl_layout_gen = 0, .cl_is_released = false, }; + struct ll_inode_info *lli = ll_i2info(inode); + __u32 gen; int rc; ENTRY; @@ -1277,13 +1311,25 @@ static int pcc_try_open_attach(struct inode *inode, bool *cached) * obtain valid layout lock from MDT (i.e. the file is being * HSM restoring). */ - if (ll_layout_version_get(ll_i2info(inode)) == CL_LAYOUT_GEN_NONE) - RETURN(0); + if (is_open) { + if (ll_layout_version_get(lli) == CL_LAYOUT_GEN_NONE) + RETURN(0); + } else { + rc = ll_layout_refresh(inode, &gen); + if (rc) + RETURN(rc); + } rc = pcc_get_layout_info(inode, &clt); if (rc) RETURN(rc); + if (!is_open && gen != clt.cl_layout_gen) { + CDEBUG(D_CACHE, DFID" layout changed from %d to %d.\n", + PFID(ll_inode2fid(inode)), gen, clt.cl_layout_gen); + RETURN(-EINVAL); + } + if (clt.cl_is_released) rc = pcc_try_datasets_attach(inode, clt.cl_layout_gen, LU_PCC_READWRITE, cached); @@ -1315,7 +1361,9 @@ int pcc_file_open(struct inode *inode, struct file *file) GOTO(out_unlock, rc = 0); if (!pcci || !pcc_inode_has_layout(pcci)) { - rc = pcc_try_open_attach(inode, &cached); + if (lli->lli_pcc_state & PCC_STATE_FL_OPEN_ATTACH) + rc = pcc_try_auto_attach(inode, &cached, true); + if (rc < 0 || !cached) GOTO(out_unlock, rc); @@ -1383,8 +1431,9 @@ out: RETURN_EXIT; } -static void pcc_io_init(struct inode *inode, bool *cached) +static void pcc_io_init(struct inode *inode, enum pcc_io_type iot, bool *cached) { + struct ll_inode_info *lli = ll_i2info(inode); struct pcc_inode *pcci; pcc_inode_lock(inode); @@ -1395,6 +1444,17 @@ static void pcc_io_init(struct inode *inode, bool *cached) *cached = true; } else { *cached = false; + if ((lli->lli_pcc_state & PCC_STATE_FL_IO_ATTACH && + iot != PIT_GETATTR) || + (iot == PIT_GETATTR && + lli->lli_pcc_state & PCC_STATE_FL_STAT_ATTACH)) { + (void) pcc_try_auto_attach(inode, cached, false); + if (*cached) { + pcci = ll_i2pcci(inode); + LASSERT(atomic_read(&pcci->pcci_refcount) > 0); + atomic_inc(&pcci->pcci_active_ios); + } + } } pcc_inode_unlock(inode); } @@ -1460,7 +1520,7 @@ ssize_t pcc_file_read_iter(struct kiocb *iocb, RETURN(0); } - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_READ, cached); if (!*cached) RETURN(0); @@ -1532,7 +1592,7 @@ ssize_t pcc_file_write_iter(struct kiocb *iocb, RETURN(-EAGAIN); } - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_WRITE, cached); if (!*cached) RETURN(0); @@ -1568,7 +1628,7 @@ int pcc_inode_setattr(struct inode *inode, struct iattr *attr, RETURN(0); } - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_SETATTR, cached); if (!*cached) RETURN(0); @@ -1604,7 +1664,7 @@ int pcc_inode_getattr(struct inode *inode, bool *cached) RETURN(0); } - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_GETATTR, cached); if (!*cached) RETURN(0); @@ -1668,7 +1728,7 @@ ssize_t pcc_file_splice_read(struct file *in_file, loff_t *ppos, if (!file_inode(pcc_file)->i_fop->splice_read) RETURN(-ENOTSUPP); - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_SPLICE_READ, cached); if (!*cached) RETURN(0); @@ -1695,7 +1755,7 @@ int pcc_fsync(struct file *file, loff_t start, loff_t end, RETURN(0); } - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_FSYNC, cached); if (!*cached) RETURN(0); @@ -1811,7 +1871,7 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, CDEBUG(D_MMAP, "%s: PCC backend fs not support ->page_mkwrite()\n", ll_i2sbi(inode)->ll_fsname); - pcc_ioctl_detach(inode, PCC_DETACH_OPT_NONE); + pcc_ioctl_detach(inode, PCC_DETACH_OPT_UNCACHE); up_read(&mm->mmap_sem); *cached = true; RETURN(VM_FAULT_RETRY | VM_FAULT_NOPAGE); @@ -1819,7 +1879,7 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, /* Pause to allow for a race with concurrent detach */ OBD_FAIL_TIMEOUT(OBD_FAIL_LLITE_PCC_MKWRITE_PAUSE, cfs_fail_val); - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_PAGE_MKWRITE, cached); if (!*cached) { /* This happens when the file is detached from PCC after got * the fault page via ->fault() on the inode of the PCC copy. @@ -1852,7 +1912,7 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, */ if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_PCC_DETACH_MKWRITE)) { pcc_io_fini(inode); - pcc_ioctl_detach(inode, PCC_DETACH_OPT_NONE); + pcc_ioctl_detach(inode, PCC_DETACH_OPT_UNCACHE); up_read(&mm->mmap_sem); RETURN(VM_FAULT_RETRY | VM_FAULT_NOPAGE); } @@ -1886,7 +1946,7 @@ int pcc_fault(struct vm_area_struct *vma, struct vm_fault *vmf, RETURN(0); } - pcc_io_init(inode, cached); + pcc_io_init(inode, PIT_FAULT, cached); if (!*cached) RETURN(0); @@ -2107,15 +2167,23 @@ int pcc_inode_create(struct super_block *sb, struct pcc_dataset *dataset, return rc; } -int pcc_inode_create_fini(struct pcc_dataset *dataset, struct inode *inode, - struct dentry *pcc_dentry) +int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca) { + struct dentry *pcc_dentry = pca->pca_dentry; const struct cred *old_cred; struct pcc_inode *pcci; - int rc = 0; + int rc; ENTRY; + if (!pca->pca_dataset) + RETURN(0); + + if (!inode) + GOTO(out_dataset_put, rc = 0); + + LASSERT(pcc_dentry); + old_cred = override_creds(pcc_super_cred(inode->i_sb)); pcc_inode_lock(inode); LASSERT(ll_i2pcci(inode) == NULL); @@ -2129,7 +2197,8 @@ int pcc_inode_create_fini(struct pcc_dataset *dataset, struct inode *inode, GOTO(out_put, rc); pcc_inode_init(pcci, ll_i2info(inode)); - pcc_inode_attach_init(dataset, pcci, pcc_dentry, LU_PCC_READWRITE); + pcc_inode_attach_init(pca->pca_dataset, pcci, pcc_dentry, + LU_PCC_READWRITE); rc = pcc_layout_xattr_set(pcci, 0); if (rc) { @@ -2152,9 +2221,36 @@ out_put: out_unlock: pcc_inode_unlock(inode); revert_creds(old_cred); +out_dataset_put: + pcc_dataset_put(pca->pca_dataset); RETURN(rc); } +void pcc_create_attach_cleanup(struct super_block *sb, + struct pcc_create_attach *pca) +{ + if (!pca->pca_dataset) + return; + + if (pca->pca_dentry) { + const struct cred *old_cred; + int rc; + + old_cred = override_creds(pcc_super_cred(sb)); + rc = ll_vfs_unlink(pca->pca_dentry->d_parent->d_inode, + pca->pca_dentry); + if (rc) + CWARN("failed to unlink PCC file %.*s, rc = %d\n", + pca->pca_dentry->d_name.len, + pca->pca_dentry->d_name.name, rc); + /* ignore the unlink failure */ + revert_creds(old_cred); + dput(pca->pca_dentry); + } + + pcc_dataset_put(pca->pca_dataset); +} + static int pcc_filp_write(struct file *filp, const void *buf, ssize_t count, loff_t *offset) { @@ -2323,7 +2419,6 @@ int pcc_readwrite_attach_fini(struct file *file, struct inode *inode, old_cred = override_creds(pcc_super_cred(inode->i_sb)); pcc_inode_lock(inode); pcci = ll_i2pcci(inode); - lli->lli_pcc_state &= ~PCC_STATE_FL_ATTACHING; if (rc || lease_broken) { if (attached && pcci) pcc_inode_put(pcci); @@ -2340,6 +2435,7 @@ int pcc_readwrite_attach_fini(struct file *file, struct inode *inode, if (rc) GOTO(out_put, rc); + LASSERT(lli->lli_pcc_state & PCC_STATE_FL_ATTACHING); rc = ll_layout_refresh(inode, &gen2); if (!rc) { if (gen2 == gen) { @@ -2358,6 +2454,7 @@ out_put: pcc_inode_put(pcci); } out_unlock: + lli->lli_pcc_state &= ~PCC_STATE_FL_ATTACHING; pcc_inode_unlock(inode); revert_creds(old_cred); RETURN(rc); diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h index d2c8512..7323a03 100644 --- a/lustre/llite/pcc.h +++ b/lustre/llite/pcc.h @@ -92,12 +92,19 @@ struct pcc_matcher { enum pcc_dataset_flags { PCC_DATASET_NONE = 0x0, - /* Try auto attach at open, disabled by default */ - PCC_DATASET_OPEN_ATTACH = 0x1, + /* Try auto attach at open, enabled by default */ + PCC_DATASET_OPEN_ATTACH = 0x01, + /* Try auto attach during IO when layout refresh, enabled by default */ + PCC_DATASET_IO_ATTACH = 0x02, + /* Try auto attach at stat */ + PCC_DATASET_STAT_ATTACH = 0x04, + PCC_DATASET_AUTO_ATTACH = PCC_DATASET_OPEN_ATTACH | + PCC_DATASET_IO_ATTACH | + PCC_DATASET_STAT_ATTACH, /* PCC backend is only used for RW-PCC */ - PCC_DATASET_RWPCC = 0x2, + PCC_DATASET_RWPCC = 0x08, /* PCC backend is only used for RO-PCC */ - PCC_DATASET_ROPCC = 0x4, + PCC_DATASET_ROPCC = 0x10, /* PCC backend provides caching services for both RW-PCC and RO-PCC */ PCC_DATASET_PCC_ALL = PCC_DATASET_RWPCC | PCC_DATASET_ROPCC, }; @@ -153,6 +160,25 @@ struct pcc_file { enum lu_pcc_type pccf_type; }; +enum pcc_io_type { + /* read system call */ + PIT_READ = 1, + /* write system call */ + PIT_WRITE, + /* truncate, utime system calls */ + PIT_SETATTR, + /* stat system call */ + PIT_GETATTR, + /* mmap write handling */ + PIT_PAGE_MKWRITE, + /* page fault handling */ + PIT_FAULT, + /* fsync system call handling */ + PIT_FSYNC, + /* splice_read system call */ + PIT_SPLICE_READ +}; + enum pcc_cmd_type { PCC_ADD_DATASET = 0, PCC_DEL_DATASET, @@ -176,6 +202,11 @@ struct pcc_cmd { } u; }; +struct pcc_create_attach { + struct pcc_dataset *pca_dataset; + struct dentry *pca_dentry; +}; + int pcc_super_init(struct pcc_super *super); void pcc_super_fini(struct pcc_super *super); int pcc_cmd_handle(char *buffer, unsigned long count, @@ -211,12 +242,12 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, bool *cached); int pcc_inode_create(struct super_block *sb, struct pcc_dataset *dataset, struct lu_fid *fid, struct dentry **pcc_dentry); -int pcc_inode_create_fini(struct pcc_dataset *dataset, struct inode *inode, - struct dentry *pcc_dentry); +int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca); +void pcc_create_attach_cleanup(struct super_block *sb, + struct pcc_create_attach *pca); struct pcc_dataset *pcc_dataset_match_get(struct pcc_super *super, struct pcc_matcher *matcher); void pcc_dataset_put(struct pcc_dataset *dataset); void pcc_inode_free(struct inode *inode); void pcc_layout_invalidate(struct inode *inode); - #endif /* LLITE_PCC_H */ diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index e18ad79..a209f1b 100644 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -181,12 +181,32 @@ setup_pcc_mapping() { do_facet $facet $LCTL pcc add $MOUNT $hsm_root -p $param } +setup_loopdev() { + local facet=$1 + local file=$2 + local mntpt=$3 + local size=${4:-50} + + do_facet $facet mkdir -p $mntpt || error "mkdir -p $hsm_root failed" + stack_trap "do_facet $facet rm -rf $mntpt" EXIT + do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size + stack_trap "do_facet $facet rm -f $file" EXIT + do_facet $facet mkfs.ext4 $file || + error "mkfs.ext4 $file failed" + do_facet $facet file $file + do_facet $facet mount -t ext4 -o loop,usrquota,grpquota $file $mntpt || + error "mount -o loop,usrquota,grpquota $file $mntpt failed" + stack_trap "do_facet $facet $UMOUNT $mntpt" EXIT +} + lpcc_rw_test() { local restore="$1" local project="$2" local project_id=100 local agt_facet=$SINGLEAGT - local hsm_root=$(hsm_root) + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" local file=$DIR/$tdir/$tfile local -a state local -a lpcc_path @@ -195,6 +215,7 @@ lpcc_rw_test() { $project && enable_project_quota do_facet $SINGLEAGT rm -rf $hsm_root + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" is_project_quota_supported || project=false @@ -302,7 +323,8 @@ test_1e() { local -a lpcc_path copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" $LCTL pcc list $MOUNT mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed" chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed" @@ -353,15 +375,20 @@ run_test 1e "Test RW-PCC with non-root user" test_1f() { local project_id=100 local agt_facet=$SINGLEAGT - local hsm_root=$(hsm_root) + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" local file=$DIR/$tdir/$tfile ! is_project_quota_supported && skip "project quota is not supported" enable_project_quota + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0" + do_facet $SINGLEAGT mkdir -p $DIR/$tdir chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed" $LFS project -sp $project_id $DIR/$tdir || @@ -377,7 +404,7 @@ test_1f() { error "failed to truncate $file" do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 || error "failed to truncate $file" - do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 || + do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=256 count=1 || error "failed to dd write from $file" check_lpcc_state $file "readwrite" @@ -392,14 +419,18 @@ test_1f() { do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file || error "failed to detach file $file" - check_lpcc_state $file "none" wait_request_state $(path2fid $file) REMOVE SUCCEED + check_lpcc_state $file "none" } run_test 1f "Test auto RW-PCC cache with non-root user" test_1g() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" local file=$DIR/$tfile + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping @@ -435,13 +466,16 @@ run_test 1g "General permission test for RW-PCC" test_2a() { local project_id=100 local agt_facet=$SINGLEAGT - local hsm_root=$(hsm_root) + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" local agt_host=$(facet_active_host $SINGLEAGT) ! is_project_quota_supported && skip "project quota is not supported" && return enable_project_quota + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping file=$DIR/$tdir/multiop @@ -463,6 +497,8 @@ test_2a() { # HSM released exists archived status check_hsm_flags $file "0x0000000d" + do_facet $SINGLEAGT $LFS pcc detach $file || + error "failed to detach $file" rmultiop_stop $agt_host || error "close $file failed" } run_test 2a "Test multi open when creating" @@ -486,14 +522,17 @@ get_remote_client() { # test_2b() { local agt_facet=$SINGLEAGT - local hsm_root=$(hsm_root) local agt_host=$(facet_active_host $SINGLEAGT) + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" needclients 2 || return 0 remote_client=$(get_remote_client) enable_project_quota + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping file=$DIR/$tdir/multiop @@ -525,10 +564,14 @@ run_test 2b "Test multi remote open when creating" test_2c() { local agt_host=$(facet_active_host $SINGLEAGT) + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" local file=$DIR/$tdir/$tfile local file2=$DIR2/$tdir/$tfile enable_project_quota + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping mkdir -p $DIR/$tdir @@ -560,7 +603,8 @@ test_3a() { local file=$DIR/$tdir/$tfile copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed" dd if=/dev/zero of=$file bs=1024 count=1 || @@ -593,7 +637,7 @@ test_3b() { # Start all of the copytools and setup PCC for n in $(seq $AGTCOUNT); do copytool setup -f agt$n -a $n -m $MOUNT - setup_pcc_mapping agt$n "projid={100}\ rwid=$n" + setup_pcc_mapping agt$n "projid={100}\ rwid=$n\ auto_attach=0" done mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed" @@ -634,11 +678,15 @@ run_test 3b "Repeat attach/detach operations on multiple clients" test_4() { local project_id=100 + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" ! is_project_quota_supported && skip "project quota is not supported" && return enable_project_quota + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping @@ -652,12 +700,18 @@ test_4() { $LUSTRE/tests/mmap_sanity -d $DIR/$tdir -m $DIR2/$tdir -e 7 || error "mmap_sanity test failed" sync; sleep 1; sync + + rm -rf $DIR/$tdir || error "failed to remove $DIR/$tdir" } run_test 4 "Auto cache test for mmap" test_5() { local file=$DIR/$tfile + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping @@ -683,24 +737,6 @@ test_5() { } run_test 5 "Mmap & cat a RW-PCC cached file" -setup_loopdev() { - local facet=$1 - local file=$2 - local mntpt=$3 - local size=${4:-50} - - do_facet $facet mkdir -p $mntpt || error "mkdir -p $hsm_root failed" - stack_trap "do_facet $facet rm -rf $mntpt" EXIT - do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size - stack_trap "do_facet $facet rm -f $file" EXIT - do_facet $facet mkfs.ext4 $file || - error "mkfs.ext4 $file failed" - do_facet $facet file $file - do_facet $facet mount -t ext4 -o loop,usrquota,grpquota $file $mntpt || - error "mount -o loop,usrquota,grpquota $file $mntpt failed" - stack_trap "do_facet $facet $UMOUNT $mntpt" EXIT -} - test_6() { local loopfile="$TMP/$tfile" local mntpt="/mnt/pcc.$tdir" @@ -727,8 +763,9 @@ test_6() { error "mmap write data mismatch: $content" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach -k $file || + do_facet $SINGLEAGT $LFS pcc detach $file || error "failed to detach file $file" + wait_request_state $(path2fid $file) REMOVE SUCCEED content=$(do_facet $SINGLEAGT $MMAP_CAT $file) [[ $content == "nmap_write_data" ]] || @@ -776,7 +813,8 @@ test_7b() { setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" echo "QQQQQ" > $file do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file || @@ -792,6 +830,7 @@ test_7b() { do_facet $SINGLEAGT $MULTIOP $file OSMWUc & pid=$! + sleep 3 do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach file $file" @@ -803,8 +842,12 @@ test_7b() { run_test 7b "Test the race with concurrent mkwrite and detach" test_8() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" local file=$DIR/$tfile + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping @@ -968,7 +1011,8 @@ test_12() { local pid copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" echo -n race_rw_attach_hsmremove > $file lpcc_path=$(lpcc_fid2path $hsm_root $file) @@ -1000,7 +1044,8 @@ test_rule_id() { local myRUNAS="$3" local file=$DIR/$tdir/$tfile - setup_pcc_mapping $SINGLEAGT "$rule\ rwid=$HSM_ARCHIVE_NUMBER" + setup_pcc_mapping $SINGLEAGT \ + "$rule\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" $LCTL pcc list $MOUNT do_facet $SINGLEAGT mkdir -p $DIR/$tdir @@ -1020,9 +1065,12 @@ test_rule_id() { error "failed to dd write from $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file || + do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file || error "failed to detach file $file" + wait_request_state $(path2fid $file) REMOVE SUCCEED check_lpcc_state $file "none" + + cleanup_pcc_mapping } test_13a() { @@ -1037,7 +1085,7 @@ test_13b() { copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping $SINGLEAGT \ - "fname={*.h5\ suffix.*\ Mid*dle}\ rwid=$HSM_ARCHIVE_NUMBER" + "fname={*.h5\ suffix.*\ Mid*dle}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" $LCTL pcc list $MOUNT do_facet $SINGLEAGT mkdir -p $DIR/$tdir @@ -1081,11 +1129,15 @@ run_test 13b "Test auto RW-PCC create caching for file name with wildcard" test_13c() { local file local myRUNAS + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" ! is_project_quota_supported && echo "Skip project quota is not supported" && return 0 enable_project_quota + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping $SINGLEAGT \ "projid={100\ 200}\&fname={*.h5},uid={500}\&gid={1000}\ rwid=$HSM_ARCHIVE_NUMBER" @@ -1143,7 +1195,8 @@ test_14() { local file=$DIR/$tdir/$tfile copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" mkdir -p $DIR/$tdir || error "mkdir -p $DIR/$tdir failed" do_facet $SINGLEAGT "echo -n autodetach_data > $file" @@ -1168,8 +1221,7 @@ test_15() { setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping $SINGLEAGT \ - "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=1" + setup_pcc_mapping mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed" chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed" @@ -1239,8 +1291,7 @@ test_16() { setup_loopdev $SINGLEAGT $loopfile $mntpt 50 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" - setup_pcc_mapping $SINGLEAGT \ - "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=1" + setup_pcc_mapping do_facet $SINGLEAGT "echo -n detach_data > $file" lpcc_path=$(lpcc_fid2path $hsm_root $file) @@ -1273,6 +1324,55 @@ test_16() { } run_test 16 "Test detach with different options" +test_17() { + local agt_host=$(facet_active_host $SINGLEAGT) + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local file=$DIR/$tfile + + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 + copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0" + + do_facet $SINGLEAGT $LCTL pcc list $MOUNT + + do_facet $SINGLEAGT "echo -n layout_refresh_data > $file" + do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file || + error "PCC attach $file failed" + check_lpcc_state $file "readwrite" + + do_facet $SINGLEAGT $LFS pcc detach -k $file || + error "PCC detach $file failed" + check_lpcc_state $file "none" + + # Truncate should attach the file into PCC automatically + # as the PCC copy is still valid. + echo "Verify auto attach during IO for truncate" + do_facet $SINGLEAGT $TRUNCATE $file 4 || error "truncate $file failed" + check_lpcc_state $file "readwrite" + + echo "Verify auto attach during IO for read/write" + rmultiop_start $agt_host $file O_r || error "open $file failed" + sleep 3 + + # Revoke the layout lock, the PCC-cached file will be + # detached automatically. + do_facet $SINGLEAGT $LCTL \ + set_param ldlm.namespaces.*mdc*.lru_size=clear + + check_lpcc_state $file "none" + rmultiop_stop $agt_host || error "close $file failed" + sleep 3 + check_lpcc_state $file "readwrite" + + do_facet $SINGLEAGT $LFS pcc detach -k $file || + error "PCC detach $file failed" + check_lpcc_state $file "none" +} +run_test 17 "Test auto attach for layout refresh" + complete $SECONDS check_and_cleanup_lustre exit_status -- 1.8.3.1