From 5be466e87304101e582df4c30edd830e528d08fb Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Mon, 22 Mar 2021 17:16:15 +0800 Subject: [PATCH] LU-10499 pcc: async attach in the background for PCC-RO file In current PCC, it may have a long delay while the whole file is being copied into the cache before it can be used. There is a significant delay for the first file access if the file is large, which wastes valuable computing time. Being able to shorten this time to first access may help application efficiency. In this patch, it adds an tuning parameter "async_threshold", which means the size threshold to determine doing PCC-RO attach asynchronously in the background. When the file size is samller than the threshold, the PCC attach during open() will be performed in synchronous way. Otherwise, the client will start a dedicated kernel thread to copy data from Lustre OSTs to the PCC copy in the background, but reads could fall back to the normal Lustre I/O path from Lustre OSTs until the file is fully cached. This may double the reads to the Lustre filesystem initially if the file is not read sequentially, but would avoid the high latency for data access. This may be some cache sharing (avoiding double reads) if the PCC copy and the application both shared the filesystem cached pages on the client. The tuning parameter "llite.*.pcc_async_threshold" is set with 256MiB by default. EX-3880 pcc: add pcc_async_affinity for async PCC attach This patch adds a tunable parameter "llite.*.pcc_async_affinity" that enables or disables the CPT selection in PCC-RO asynchronous attach for testing. Was-Change-Id: I1473a7547555a2d6c615d37182b6cc359194aae0 EX-bug-id: EX-2873 EX-3880 Test-Parameters: clientcount=3 testlist=sanity-pcc,sanity-pcc,sanity-pcc Signed-off-by: Qian Yingjin Change-Id: Ia80992e9050cc6e4c7f61949fc4013dec303e150 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54379 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Reviewed-by: Li Xi Reviewed-by: Oleg Drokin --- lustre/llite/lproc_llite.c | 63 ++++++++++++ lustre/llite/pcc.c | 245 +++++++++++++++++++++++++++++++++++---------- lustre/llite/pcc.h | 13 ++- lustre/tests/sanity-pcc.sh | 131 ++++++++++++++++++++++++ 4 files changed, 397 insertions(+), 55 deletions(-) diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c index 34aecfa..5c61219 100644 --- a/lustre/llite/lproc_llite.c +++ b/lustre/llite/lproc_llite.c @@ -612,6 +612,67 @@ out_unlock: } LDEBUGFS_SEQ_FOPS(ll_max_cached_mb); +static ssize_t pcc_async_threshold_show(struct kobject *kobj, + struct attribute *attr, char *buffer) +{ + struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, + ll_kset.kobj); + struct pcc_super *super = &sbi->ll_pcc_super; + + return scnprintf(buffer, PAGE_SIZE, "%llu\n", + super->pccs_async_threshold); +} + +static ssize_t pcc_async_threshold_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, size_t count) +{ + struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, + ll_kset.kobj); + struct pcc_super *super = &sbi->ll_pcc_super; + u64 threshold; + int rc; + + rc = sysfs_memparse(buffer, count, &threshold, "B"); + if (rc) + return rc; + + super->pccs_async_threshold = threshold; + + return count; +} +LUSTRE_RW_ATTR(pcc_async_threshold); + +static ssize_t pcc_async_affinity_show(struct kobject *kobj, + struct attribute *attr, char *buffer) +{ + struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, + ll_kset.kobj); + struct pcc_super *super = &sbi->ll_pcc_super; + + return scnprintf(buffer, PAGE_SIZE, "%d\n", super->pccs_async_affinity); +} + +static ssize_t pcc_async_affinity_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, size_t count) +{ + struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, + ll_kset.kobj); + struct pcc_super *super = &sbi->ll_pcc_super; + bool val; + int rc; + + rc = kstrtobool(buffer, &val); + if (rc) + return rc; + + super->pccs_async_affinity = val; + + return count; +} +LUSTRE_RW_ATTR(pcc_async_affinity); + static ssize_t checksums_show(struct kobject *kobj, struct attribute *attr, char *buf) { @@ -2215,6 +2276,8 @@ static struct attribute *llite_attrs[] = { #if defined(CONFIG_LL_ENCRYPTION) || defined(HAVE_LUSTRE_CRYPTO) &lustre_attr_filename_enc_use_old_base64.attr, #endif + &lustre_attr_pcc_async_threshold.attr, + &lustre_attr_pcc_async_affinity.attr, NULL, }; diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c index 49189d7..dd08925 100644 --- a/lustre/llite/pcc.c +++ b/lustre/llite/pcc.c @@ -129,6 +129,7 @@ int pcc_super_init(struct pcc_super *super) init_rwsem(&super->pccs_rw_sem); INIT_LIST_HEAD(&super->pccs_datasets); super->pccs_generation = 1; + super->pccs_async_threshold = PCC_DEFAULT_ASYNC_THRESHOLD; return 0; } @@ -1024,6 +1025,8 @@ pcc_dataset_add(struct pcc_super *super, struct pcc_cmd *cmd) rc = kern_path(pathname, LOOKUP_DIRECTORY, &dataset->pccd_path); if (unlikely(rc)) { + CDEBUG(D_CACHE, "%s: cache path lookup error: rc = %d\n", + pathname, rc); OBD_FREE_PTR(dataset); return rc; } @@ -1375,6 +1378,8 @@ static int pcc_fid2dataset_path(struct pcc_dataset *dataset, char *buf, (__u32)((fid)->f_oid ^ (fid)->f_seq) & 0XFFFF, PFID(fid)); default: + CERROR(DFID ": unknown archive format %u: rc = %d\n", + PFID(fid), dataset->pccd_hsmtool_type, -EINVAL); return -EINVAL; } } @@ -1708,8 +1713,159 @@ static int pcc_try_datasets_attach(struct inode *inode, enum pcc_io_type iot, RETURN(rc); } -static int pcc_readonly_ioctl_attach(struct file *file, struct inode *inode, - __u32 roid); +static struct pcc_attach_context * +pcc_attach_context_alloc(struct file *file, struct inode *inode, __u32 id) +{ + struct pcc_attach_context *pccx; + + OBD_ALLOC_PTR(pccx); + if (!pccx) + RETURN(NULL); + + pccx->pccx_file = get_file(file); + pccx->pccx_inode = inode; + pccx->pccx_attach_id = id; + + return pccx; +} + +static inline void pcc_attach_context_free(struct pcc_attach_context *pccx) +{ + LASSERT(pccx->pccx_file != NULL); + fput(pccx->pccx_file); + OBD_FREE_PTR(pccx); +} + +static int pcc_attach_check_set(struct inode *inode) +{ + struct ll_inode_info *lli = ll_i2info(inode); + struct pcc_inode *pcci; + int rc = 0; + + ENTRY; + + pcc_inode_lock(inode); + if (lli->lli_pcc_state & PCC_STATE_FL_ATTACHING) + GOTO(out_unlock, rc = -EINPROGRESS); + + pcci = ll_i2pcci(inode); + if (pcci && pcc_inode_has_layout(pcci)) + GOTO(out_unlock, rc = -EEXIST); + + lli->lli_pcc_state |= PCC_STATE_FL_ATTACHING; +out_unlock: + pcc_inode_unlock(inode); + RETURN(rc); +} + +static inline void pcc_readonly_attach_fini(struct inode *inode) +{ + pcc_inode_lock(inode); + ll_i2info(inode)->lli_pcc_state &= ~PCC_STATE_FL_ATTACHING; + pcc_inode_unlock(inode); +} + +static int pcc_readonly_attach(struct file *file, struct inode *inode, + __u32 roid); + +static int pcc_readonly_attach_thread(void *arg) +{ + struct pcc_attach_context *pccx = (struct pcc_attach_context *)arg; + struct file *file = pccx->pccx_file; + int rc; + + ENTRY; + + /* + * For asynchronous open attach, it can not reuse the Lustre file + * handle directly when the file is opening for read as the file + * position in the file handle can not be shared by both user thread + * and asynchronous attach thread in kenerl on the background. + * It must reopen the file without O_DIRECT flag and use this new + * file hanlde to do data copy from Lustre OSTs to the PCC copy. + */ + file = dentry_open(&file->f_path, file->f_flags & ~O_DIRECT, + pcc_super_cred(pccx->pccx_inode->i_sb)); + if (IS_ERR_OR_NULL(file)) + GOTO(out, rc = file == NULL ? -EINVAL : PTR_ERR(file)); + + rc = pcc_readonly_attach(file, pccx->pccx_inode, + pccx->pccx_attach_id); + fput(file); +out: + pcc_readonly_attach_fini(pccx->pccx_inode); + CDEBUG(D_CACHE, "PCC-RO attach in background for %pd "DFID" rc = %d\n", + file_dentry(pccx->pccx_file), + PFID(ll_inode2fid(pccx->pccx_inode)), rc); + pcc_attach_context_free(pccx); + RETURN(rc); +} + +static int pcc_readonly_attach_async(struct file *file, + struct inode *inode, __u32 roid) +{ + struct pcc_attach_context *pccx = NULL; + struct task_struct *task; + int rc; + + ENTRY; + + rc = pcc_attach_check_set(inode); + if (rc) + RETURN(rc); + + pccx = pcc_attach_context_alloc(file, inode, roid); + if (!pccx) + GOTO(out, rc = -ENOMEM); + + if (ll_i2pccs(inode)->pccs_async_affinity) { + /* Create a attach kthread on the current node. */ + task = kthread_create(pcc_readonly_attach_thread, pccx, + "ll_pcc_%u", current->pid); + } else { + int node = cfs_cpt_spread_node(cfs_cpt_tab, CFS_CPT_ANY); + + task = kthread_create_on_node(pcc_readonly_attach_thread, pccx, + node, "ll_pcc_%u", current->pid); + } + + if (IS_ERR(task)) { + rc = PTR_ERR(task); + CERROR("%s: cannot start ll_pcc thread for "DFID": rc = %d\n", + ll_i2sbi(inode)->ll_fsname, PFID(ll_inode2fid(inode)), + rc); + GOTO(out, rc); + } + + wake_up_process(task); + RETURN(0); +out: + if (pccx) + pcc_attach_context_free(pccx); + + pcc_readonly_attach_fini(inode); + RETURN(rc); +} + +static int pcc_readonly_attach_sync(struct file *file, + struct inode *inode, __u32 roid); + +static inline int pcc_do_readonly_attach(struct file *file, + struct inode *inode, __u32 roid) +{ + int rc; + + if (max_t(__u64, ll_i2info(inode)->lli_lazysize, i_size_read(inode)) >= + ll_i2pccs(inode)->pccs_async_threshold) { + rc = pcc_readonly_attach_async(file, inode, roid); + if (!rc || rc == -EINPROGRESS) + return rc; + } + + rc = pcc_readonly_attach_sync(file, inode, roid); + + return rc; +} /* Call with pcci_mutex hold */ static int pcc_try_readonly_open_attach(struct inode *inode, struct file *file, @@ -1726,6 +1882,9 @@ static int pcc_try_readonly_open_attach(struct inode *inode, struct file *file, if (!((file->f_flags & O_ACCMODE) == O_RDONLY)) RETURN(0); + if (ll_i2info(inode)->lli_pcc_state & PCC_STATE_FL_ATTACHING) + RETURN(-EINPROGRESS); + item.pm_uid = from_kuid(&init_user_ns, current_uid()); item.pm_gid = from_kgid(&init_user_ns, current_gid()); item.pm_projid = ll_i2info(inode)->lli_projid; @@ -1739,7 +1898,7 @@ static int pcc_try_readonly_open_attach(struct inode *inode, struct file *file, if ((dataset->pccd_flags & PCC_DATASET_PCC_ALL) == PCC_DATASET_PCCRO) { pcc_inode_unlock(inode); - rc = pcc_readonly_ioctl_attach(file, inode, dataset->pccd_roid); + rc = pcc_do_readonly_attach(file, inode, dataset->pccd_roid); pcc_inode_lock(inode); pcci = ll_i2pcci(inode); if (pcci && pcc_inode_has_layout(pcci)) @@ -3338,28 +3497,6 @@ out_free: RETURN(rc); } -static int pcc_attach_allowed_check(struct inode *inode) -{ - struct ll_inode_info *lli = ll_i2info(inode); - struct pcc_inode *pcci; - int rc = 0; - - ENTRY; - - pcc_inode_lock(inode); - if (lli->lli_pcc_state & PCC_STATE_FL_ATTACHING) - GOTO(out_unlock, rc = -EBUSY); - - pcci = ll_i2pcci(inode); - if (pcci && pcc_inode_has_layout(pcci)) - GOTO(out_unlock, rc = -EEXIST); - - lli->lli_pcc_state |= PCC_STATE_FL_ATTACHING; -out_unlock: - pcc_inode_unlock(inode); - RETURN(rc); -} - static int pcc_attach_data_archive(struct file *file, struct inode *inode, struct pcc_dataset *dataset, struct dentry **dentry) @@ -3447,7 +3584,7 @@ int pcc_readwrite_attach(struct file *file, struct inode *inode, ENTRY; - rc = pcc_attach_allowed_check(inode); + rc = pcc_attach_check_set(inode); if (rc) RETURN(rc); @@ -3606,18 +3743,9 @@ repeat: RETURN(rc); } -static void pcc_readonly_attach_fini(struct inode *inode) -{ - pcc_inode_lock(inode); - ll_i2info(inode)->lli_pcc_state &= ~PCC_STATE_FL_ATTACHING; - pcc_inode_unlock(inode); -} - -static int pcc_readonly_ioctl_attach(struct file *file, - struct inode *inode, - __u32 roid) +static int pcc_readonly_attach(struct file *file, + struct inode *inode, __u32 roid) { - struct ll_sb_info *sbi = ll_i2sbi(inode); struct pcc_super *super = ll_i2pccs(inode); struct ll_inode_info *lli = ll_i2info(inode); const struct cred *old_cred; @@ -3631,25 +3759,14 @@ static int pcc_readonly_ioctl_attach(struct file *file, ENTRY; - if (!test_bit(LL_SBI_LAYOUT_LOCK, sbi->ll_flags)) - RETURN(-EOPNOTSUPP); - - rc = pcc_attach_allowed_check(inode); - if (rc) { - CDEBUG(D_CACHE, - "PCC-RO caching for "DFID" not allowed, rc = %d\n", - PFID(ll_inode2fid(inode)), rc); - RETURN(rc); - } - rc = pcc_layout_rdonly_set(inode, &gen); if (rc) - GOTO(out_fini, rc); + RETURN(rc); dataset = pcc_dataset_get(&ll_s2sbi(inode->i_sb)->ll_pcc_super, LU_PCC_READONLY, roid); if (dataset == NULL) - GOTO(out_fini, rc = -ENOENT); + RETURN(-ENOENT); rc = pcc_attach_data_archive(file, inode, dataset, &dentry); if (rc) @@ -3703,12 +3820,33 @@ out_put_unlock: mutex_unlock(&lli->lli_layout_mutex); out_dataset_put: pcc_dataset_put(dataset); -out_fini: - pcc_readonly_attach_fini(inode); RETURN(rc); } +static int pcc_readonly_attach_sync(struct file *file, + struct inode *inode, __u32 roid) +{ + int rc; + + ENTRY; + + if (!test_bit(LL_SBI_LAYOUT_LOCK, ll_i2sbi(inode)->ll_flags)) + RETURN(-EOPNOTSUPP); + + rc = pcc_attach_check_set(inode); + if (rc) { + CDEBUG(D_CACHE, + "PCC-RO caching for "DFID" not allowed, rc = %d\n", + PFID(ll_inode2fid(inode)), rc); + RETURN(rc); + } + + rc = pcc_readonly_attach(file, inode, roid); + pcc_readonly_attach_fini(inode); + RETURN(rc); +} + int pcc_ioctl_attach(struct file *file, struct inode *inode, struct lu_pcc_attach *attach) { @@ -3721,8 +3859,7 @@ int pcc_ioctl_attach(struct file *file, struct inode *inode, rc = -EOPNOTSUPP; break; case LU_PCC_READONLY: - rc = pcc_readonly_ioctl_attach(file, inode, - attach->pcca_id); + rc = pcc_readonly_attach_sync(file, inode, attach->pcca_id); break; default: rc = -EINVAL; diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h index 6d24d6e..a01cf2c 100644 --- a/lustre/llite/pcc.h +++ b/lustre/llite/pcc.h @@ -150,9 +150,11 @@ struct pcc_dataset { struct path pccd_path; /* Root path */ struct list_head pccd_linkage; /* Linked to pccs_datasets */ atomic_t pccd_refcount; /* Reference count */ - enum hsmtool_type pccd_hsmtool_type; /*HSM copytool type */ + enum hsmtool_type pccd_hsmtool_type; /* HSM copytool type */ }; +#define PCC_DEFAULT_ASYNC_THRESHOLD (256 << 20) + struct pcc_super { /* Protect pccs_datasets */ struct rw_semaphore pccs_rw_sem; @@ -166,6 +168,9 @@ struct pcc_super { * parameters for PCC. */ __u64 pccs_generation; + /* Size threshold for asynchrous PCC-RO attach in background. */ + __u64 pccs_async_threshold; + bool pccs_async_affinity; }; struct pcc_inode { @@ -207,6 +212,12 @@ struct pcc_vma { const struct vm_operations_struct *pccv_vm_ops; }; +struct pcc_attach_context { + struct file *pccx_file; + struct inode *pccx_inode; + __u32 pccx_attach_id; +}; + enum pcc_io_type { /* read system call */ PIT_READ = 1, diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index 21d8292..5346625 100755 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -240,6 +240,7 @@ setup_loopdev_project() { stack_trap "do_facet $facet rmdir $mntpt" EXIT do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size stack_trap "do_facet $facet rm -f $file" EXIT + do_facet $facet $UMOUNT $mntpt do_facet $facet mkfs.ext4 -O project,quota $file || error "mkfs.ext4 -O project,quota $file failed" do_facet $facet file $file @@ -3173,6 +3174,136 @@ test_39() { } run_test 39 "Test Project quota on loop PCC device" +wait_readonly_attach_fini() { + local file=$1 + local facet=${2:-$SINGLEAGT} + local cmd="$LFS pcc state $file | grep -E -c 'type: readonly'" + + echo $cmd + wait_update_facet $facet "$cmd" "1" 50 || + error "Async attach $file timed out" +} + +calc_stats_facet() { + local paramfile="$1" + local stat="$2" + local facet=${3:-$SINGLEAGT} + + do_facet $facet $LCTL get_param -n $paramfile | + awk '/^'$stat'/ { sum += $2 } END { printf("%0.0f", sum) }' +} + +test_40() { + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + + is_project_quota_supported || skip "project quota is not supported" + + enable_project_quota + + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local dir=$DIR/$tdir + local file=$dir/$tfile + local id=100 + + setup_loopdev $SINGLEAGT $loopfile $mntpt 200 + do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed" + setup_pcc_mapping $SINGLEAGT \ + "projid={$id}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1" + do_facet $SINGLEAGT $LCTL pcc list $MOUNT + + mkdir -p $dir || error "mkdir $dir failed" + do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=50 || + error "Write $file failed" + + $LFS project -p $id $file || error "failed to set project for $file" + $LFS project -d $file + do_facet $SINGLEAGT $LFS pcc detach $file + do_facet $SINGLEAGT $LFS pcc state $file + + do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear + do_facet $SINGLEAGT $LCTL set_param osc.*.stats=clear + #define OBD_FAIL_OST_BRW_PAUSE_BULK + set_nodes_failloc "$(osts_nodes)" 0x214 1 + echo 3 > /proc/sys/vm/drop_caches + + local stime + local time1 + local time2 + local rpcs_before + local rpcs_after + + do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=5MB + + echo "Test open attach with pcc_async_threshold=5MB" + stime=$SECONDS + # Open with O_RDONLY flag will trigger auto attach + do_facet $SINGLEAGT $MULTIOP $file oc || + error "failed to readonly open $file" + + rpcs_before=$(calc_stats_facet osc.*.stats ost_read) + do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct + rpcs_after=$(calc_stats_facet osc.*.stats ost_read) + echo "Before: $rpcs_before After: $rpcs_after" + [ $rpcs_after -gt $rpcs_before ] || + error "should send read RPCs to OSTs $rpcs_before: $rpcs_after" + time1=$((SECONDS - stime)) + do_facet $SINGLEAGT $LFS pcc state $file + wait_readonly_attach_fini $file + + do_facet $SINGLEAGT $LFS pcc detach $file + do_facet $SINGLEAGT $LFS pcc state $file + do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=1G + do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear + do_facet $SINGLEAGT $LCTL set_param osc.*.stats=clear + + echo "Test open attach with async_threshold=1G" + stime=$SECONDS + # Open with O_RDONLY flag will trigger auto attach + do_facet $SINGLEAGT $MULTIOP $file oc || + error "failed to readonly open $file" + do_facet $SINGLEAGT $LFS pcc state $file + rpcs_before=$(calc_stats_facet osc.*.stats ost_read) + do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct + rpcs_after=$(calc_stats_facet osc.*.stats ost_read) + time2=$((SECONDS - stime)) + echo "Before: $rpcs_before After: $rpcs_after" + [ $rpcs_after -eq $rpcs_before ] || + error "should not send OST_READ RPCs to OSTs" + + echo "Time1: $time1 Time2: $time2" + [ $time1 -le $time2 ] || + error "Total time for async open attach should be smaller" + + do_facet $SINGLEAGT $LFS pcc detach $file + do_facet $SINGLEAGT $LFS pcc state $file + do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=5MB + do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear + + echo "Read 1MB data with async_threshold=5MB" + stime=$SECONDS + do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct + time1=$((SECONDS - stime)) + wait_readonly_attach_fini $file + + do_facet $SINGLEAGT $LFS pcc detach $file + do_facet $SINGLEAGT $LFS pcc state $file + do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=1G + do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear + + echo "Read 1MB data with async_threshold=1G" + stime=$SECONDS + do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct + time2=$((SECONDS - stime)) + + echo "Time1: $time1 Time2: $time2" + [ $time1 -le $time2 ] || + error "Total time for async open attach should be smaller" +} +run_test 40 "Test async open attach in the background for PCC-RO file" + test_41() { local loopfile="$TMP/$tfile" local mntpt="/mnt/pcc.$tdir" -- 1.8.3.1