From 06b44fda8c64900c050a42f895257a5cee0e8388 Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Tue, 15 Aug 2023 04:54:54 -0400 Subject: [PATCH] EX-8027 pcc: add wait option when remove a PCC backend In this patch, we add a "wait" option for the PCC tool when remove PCC backend from a client: lctl pcc del --wait $MOUNT $pcc_path lctl pcc clear --wait $MOUNT With this option, the caller must wait for all in-progress attaches finished when remove the PCC backend from a client. Change-Id: Ic8386329087a7129b0583fa823cbb50673893d0d Signed-off-by: Qian Yingjin Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51944 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/doc/lctl-pcc.8 | 12 +++++--- lustre/doc/llapi_pcc_del.3 | 11 ++++++-- lustre/include/uapi/linux/lustre/lustre_user.h | 4 +++ lustre/llite/pcc.c | 38 ++++++++++++++++++-------- lustre/llite/pcc.h | 2 +- lustre/tests/sanity-pcc.sh | 6 ++-- lustre/utils/liblustreapi_pcc.c | 7 ++++- lustre/utils/obd.c | 16 ++++++++--- 8 files changed, 70 insertions(+), 26 deletions(-) diff --git a/lustre/doc/lctl-pcc.8 b/lustre/doc/lctl-pcc.8 index c1950e8..b2102f6 100644 --- a/lustre/doc/lctl-pcc.8 +++ b/lustre/doc/lctl-pcc.8 @@ -4,9 +4,9 @@ lctl pcc commands used to interact with PCC features. .SH SYNOPSIS .B lctl pcc add \fR<\fImntpath\fR> <\fIpccpath\fR> [\fB--param\fR|\fB-p\fR <\fIparam\fR>] .br -.B lctl pcc del [\fB--keep\fR|\fB-k\fR] <\fImntpath\fR> <\fIpccpath\fR> +.B lctl pcc del [\fB--keep\fR|\fB-k\fR] [\fB--wait\fR|\fB-w\fR] <\fImntpath\fR> <\fIpccpath\fR> .br -.B lctl pcc clear [\fB--keep\fR|\fB-k\fR] <\fImntpath\fR> +.B lctl pcc clear [\fB--keep\fR|\fB-k\fR] [\fB--wait\fR|\fB-w\fR] <\fImntpath\fR> .br .B lctl pcc list <\fImntpath\fR> .SH DESCRIPTION @@ -37,13 +37,13 @@ Currently each PCC backend only has one rule which is configed when setup PCC backend on a client. If a user wants to change the rule, the PCC backend needs to be removed first and then added back with a new rule. .TP -.B lctl pcc del [\fB--keep\fR|\fB-k\fR] <\fImntpath\fR> <\fIpccpath\fR> +.B lctl pcc del [\fB--keep\fR|\fB-k\fR] [\fB--wait\fR|\fB-w\fR] <\fImntpath\fR> <\fIpccpath\fR> Delete a PCC backend specified by path .IR pccpath on a Lustre client referenced by the mount point of .IR mntpath . .TP -.B lctl pcc clear [\fB--keep\fR|\fB-k\fR] <\fImntpath\fR> +.B lctl pcc clear [\fB--keep\fR|\fB-k\fR] [\fB--wait\fR|\fB-w\fR] <\fImntpath\fR> Remove all PCC backend on a Lustre client referenced by the mount point of .IR mntpath . .TP @@ -63,6 +63,10 @@ Lustre client, and retains the data on the cache. At this time, the PCC-RW backend falls back as a tranditional HSM storage solution since the copytool is still running at this client. .TP +.B --wait | -w +The caller must wait all in-progress attaches finished when remove a PCC +backend from a client. +.TP .SH SEE ALSO .BR lfs (1), .BR lfs-hsm (1), diff --git a/lustre/doc/llapi_pcc_del.3 b/lustre/doc/llapi_pcc_del.3 index f6a54b2..f52e274 100644 --- a/lustre/doc/llapi_pcc_del.3 +++ b/lustre/doc/llapi_pcc_del.3 @@ -20,12 +20,19 @@ By default, when remove a PCC backend from a client, the action is to scan the PCC backend fs, uncache (detach and remove) all scanned PCC copies from PCC by FIDs. The input parameter .IR flags -currently only has one value +currently has two flag values .B PCC_CLEANUP_FL_KEEP_DATA -that means it removes the PCC backend from the Lustre client, but retains +and +.B PCC_CLEANUP_FL_WAIT. + +.B PCC_CLEANUP_FL_KEEP_DATA +means it removes the PCC backend from the Lustre client, but retains the data on the cache. In this case, the PCC-RW backend falls back as a traditional HSM storage solution as long as the copytool is still running at this client. +.B PCC_CLEANUP_FL_WAIT +means the caller must wait for the in-progress attaches finished when remove +the PCC backend from the client. .SH RETURN VALUES .PP .B llapi_pcc_del() diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 5a81fe6..51e68df 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -2778,7 +2778,9 @@ static inline const char *pcc_type2string(enum lu_pcc_type type) #define PCC_CMDNAME_ADD "add" #define PCC_CMDNAME_DEL "del" +#define PCC_CMDNAME_DEL_WAIT "del_wait" #define PCC_CMDNAME_CLEAR "clear" +#define PCC_CMDNAME_CLEAR_WAIT "clear_wait" #define PCC_CMDNAME_RWID "rwid" #define PCC_CMDNAME_ROID "roid" #define PCC_CMDNAME_AUTO_ATTACH "auto_attach" @@ -2895,6 +2897,8 @@ enum lu_pcc_cleanup_flags { PCC_CLEANUP_FL_NONE = 0x0, /* Remove the PCC backend but retain the data on the cache */ PCC_CLEANUP_FL_KEEP_DATA = 0x1, + /* Wait in-progress attaches finished when remove the PCC backend */ + PCC_CLEANUP_FL_WAIT = 0x2, }; enum lu_project_type { diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c index aa5e19a..2b5716a 100644 --- a/lustre/llite/pcc.c +++ b/lustre/llite/pcc.c @@ -1162,7 +1162,7 @@ pcc_dataset_attach_put(struct pcc_dataset *dataset) } static int -pcc_dataset_del(struct pcc_super *super, char *pathname) +pcc_dataset_del(struct pcc_super *super, char *pathname, bool wait) { struct list_head *l, *tmp; struct pcc_dataset *dataset; @@ -1174,8 +1174,11 @@ pcc_dataset_del(struct pcc_super *super, char *pathname) if (strcmp(dataset->pccd_pathname, pathname) == 0) { list_del_init(&dataset->pccd_linkage); up_write(&super->pccs_rw_sem); - l_wait_event_abortable(dataset->pccd_attach_waitq, - atomic_read(&dataset->pccd_attach_nr) == 0); + if (wait) + l_wait_event_abortable( + dataset->pccd_attach_waitq, + atomic_read(&dataset->pccd_attach_nr) == + 0); pcc_dataset_put(dataset); super->pccs_generation++; return 0; @@ -1215,7 +1218,7 @@ pcc_super_dump(struct pcc_super *super, struct seq_file *m) return 0; } -static void pcc_remove_datasets(struct pcc_super *super) +static void pcc_remove_datasets(struct pcc_super *super, bool wait) { struct pcc_dataset *dataset; @@ -1225,7 +1228,8 @@ static void pcc_remove_datasets(struct pcc_super *super) pccd_linkage)) != NULL) { list_del(&dataset->pccd_linkage); up_write(&super->pccs_rw_sem); - l_wait_event_abortable(dataset->pccd_attach_waitq, + if (wait) + l_wait_event_abortable(dataset->pccd_attach_waitq, atomic_read(&dataset->pccd_attach_nr) == 0); pcc_dataset_put(dataset); down_write(&super->pccs_rw_sem); @@ -1236,7 +1240,7 @@ static void pcc_remove_datasets(struct pcc_super *super) void pcc_super_fini(struct pcc_super *super) { - pcc_remove_datasets(super); + pcc_remove_datasets(super, true); put_cred(super->pccs_cred); } @@ -1267,6 +1271,13 @@ pcc_cmd_parse(char *buffer, unsigned long count) cmd->pccc_cmd = PCC_CLEAR_ALL; GOTO(out, rc = 0); } + if (strncmp(buffer, PCC_CMDNAME_CLEAR_WAIT, + strlen(PCC_CMDNAME_CLEAR_WAIT)) == 0) { + cmd->pccc_cmd = PCC_CLEAR_ALL; + cmd->u.pccc_del.pccc_attach_wait = true; + GOTO(out, rc = 0); + } + val = buffer; token = strsep(&val, " "); @@ -1274,12 +1285,16 @@ pcc_cmd_parse(char *buffer, unsigned long count) GOTO(out_free_cmd, rc = -EINVAL); /* Type of the command */ - if (strcmp(token, PCC_CMDNAME_ADD) == 0) + if (strcmp(token, PCC_CMDNAME_ADD) == 0) { cmd->pccc_cmd = PCC_ADD_DATASET; - else if (strcmp(token, PCC_CMDNAME_DEL) == 0) + } else if (strcmp(token, PCC_CMDNAME_DEL) == 0) { cmd->pccc_cmd = PCC_DEL_DATASET; - else + } else if (strcmp(token, PCC_CMDNAME_DEL_WAIT) == 0) { + cmd->pccc_cmd = PCC_DEL_DATASET; + cmd->u.pccc_del.pccc_attach_wait = true; + } else { GOTO(out_free_cmd, rc = -EINVAL); + } if (cmd->pccc_cmd == PCC_ADD_DATASET || cmd->pccc_cmd == PCC_DEL_DATASET) { @@ -1350,10 +1365,11 @@ int pcc_cmd_handle(char *buffer, unsigned long count, rc = pcc_dataset_add(super, cmd); break; case PCC_DEL_DATASET: - rc = pcc_dataset_del(super, cmd->pccc_pathname); + rc = pcc_dataset_del(super, cmd->pccc_pathname, + cmd->u.pccc_del.pccc_attach_wait); break; case PCC_CLEAR_ALL: - pcc_remove_datasets(super); + pcc_remove_datasets(super, cmd->u.pccc_del.pccc_attach_wait); break; default: rc = -EINVAL; diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h index 828a288..0e45185 100644 --- a/lustre/llite/pcc.h +++ b/lustre/llite/pcc.h @@ -295,7 +295,7 @@ struct pcc_cmd { enum hsmtool_type pccc_hsmtool_type; } pccc_add; struct pcc_cmd_del { - __u32 pccc_pad; + bool pccc_attach_wait; } pccc_del; } u; }; diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index d305c866..28a604c 100755 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -175,7 +175,7 @@ cleanup_pcc_mapping() { local facet=${1:-$SINGLEAGT} echo "Cleanup PCC backend on $MOUNT" - do_facet $facet $LCTL pcc clear $MOUNT + do_facet $facet $LCTL pcc clear --wait $MOUNT } setup_pcc_mapping() { @@ -4912,7 +4912,7 @@ test_106() { $LFS pcc attach $file & pid=$! sleep 1 - $LCTL set_param llite.*.pcc="clear" + $LCTL set_param llite.*.pcc="clear_wait" wait $pid || error "PCC attach for $file failed" $LFS pcc state $file $LFS pcc detach $file || error "failed to detch $file" @@ -4922,7 +4922,7 @@ test_106() { "projid={100}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1" $LFS pcc attach $file & sleep 1 - $LCTL pcc clear -v $MOUNT + $LCTL pcc clear -v --wait $MOUNT } run_test 106 "PCC backend cleanup shound wait for all attaching finished" diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c index c62f9d1..b7a2647 100644 --- a/lustre/utils/liblustreapi_pcc.c +++ b/lustre/utils/liblustreapi_pcc.c @@ -656,7 +656,12 @@ static int llapi_pcc_del_internal(const char *mntpath, const char *pccpath, char cmd[PATH_MAX]; int rc; - snprintf(cmd, sizeof(cmd), "%s %s", PCC_CMDNAME_DEL, pccpath); + if (flags & PCC_CLEANUP_FL_WAIT) + snprintf(cmd, sizeof(cmd), "%s %s", + PCC_CMDNAME_DEL_WAIT, pccpath); + else + snprintf(cmd, sizeof(cmd), "%s %s", PCC_CMDNAME_DEL, pccpath); + rc = llapi_pccdev_set(mntpath, cmd); if (rc < 0) { llapi_error(LLAPI_MSG_ERROR, rc, diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index 9fdb5a7..62a4d73 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -6221,6 +6221,7 @@ int jt_pcc_del(int argc, char **argv) static struct option long_opts[] = { { .val = 'k', .name = "keep-data", .has_arg = no_argument }, { .val = 'v', .name = "verbose", .has_arg = no_argument }, + { .val = 'w', .name = "wait", .has_arg = no_argument }, { .name = NULL } }; char fsname[MAX_OBD_NAME + 1]; const char *mntpath; @@ -6230,10 +6231,13 @@ int jt_pcc_del(int argc, char **argv) int rc; int c; - while ((c = getopt_long(argc, argv, "kv", long_opts, NULL)) != -1) { + while ((c = getopt_long(argc, argv, "kvw", long_opts, NULL)) != -1) { switch (c) { case 'k': - flags = PCC_CLEANUP_FL_KEEP_DATA; + flags |= PCC_CLEANUP_FL_KEEP_DATA; + break; + case 'w': + flags |= PCC_CLEANUP_FL_WAIT; break; case 'v': verbose++; @@ -6278,6 +6282,7 @@ int jt_pcc_clear(int argc, char **argv) static struct option long_opts[] = { { .val = 'k', .name = "keep-data", .has_arg = no_argument }, { .val = 'v', .name = "verbose", .has_arg = no_argument }, + { .val = 'w', .name = "wait", .has_arg = no_argument }, { .name = NULL } }; char fsname[MAX_OBD_NAME + 1]; const char *mntpath; @@ -6286,10 +6291,13 @@ int jt_pcc_clear(int argc, char **argv) int rc; int c; - while ((c = getopt_long(argc, argv, "kv", long_opts, NULL)) != -1) { + while ((c = getopt_long(argc, argv, "kvw", long_opts, NULL)) != -1) { switch (c) { case 'k': - flags = PCC_CLEANUP_FL_KEEP_DATA; + flags |= PCC_CLEANUP_FL_KEEP_DATA; + break; + case 'w': + flags |= PCC_CLEANUP_FL_WAIT; break; case 'v': verbose++; -- 1.8.3.1