From: Lei Feng Date: Tue, 19 Apr 2022 00:32:47 +0000 (-0400) Subject: LU-10499 utils: add pcc pin/unpin commands X-Git-Tag: 2.16.51~32 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=aae909cf8fbc5ef3090c5385793370fa71360f42;p=fs%2Flustre-release.git LU-10499 utils: add pcc pin/unpin commands Store pin information in xattr trusted.pin. Add lfs pcc pin/unpin commands to set/clear pcc pin flag. A pcc-pinned file will not be removed from local storage cache by lpcc_purge automatically. Examples of xattr trusted.pin: trusted.pin=[hsm: 2] trusted.pin=[hsm: 2,hsm: 3] EX-bug-id: EX-5113 Change-Id: I17ddf5ac8dc4eae48c0f6bdd0f2a19240474b0f5 Signed-off-by: Lei Feng Test-Parameters: trivial testlist=sanity-pcc Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54460 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- diff --git a/lustre/doc/lfs-pcc-pin.1 b/lustre/doc/lfs-pcc-pin.1 new file mode 100644 index 0000000..e6ee72da --- /dev/null +++ b/lustre/doc/lfs-pcc-pin.1 @@ -0,0 +1,37 @@ +.TH LFS-PCC-PIN-UNPIN 1 2022-04-13 "Lustre" "Lustre Utilities" +.SH NAME +lfs-pcc-pin,lfs-pcc-unpin \- set or clear pcc pin flag on Lustre file +.SH SYNOPSIS +.BR "lfs pcc pin " [ --id | -i " \fIID\fR] <" \fIFILE ...> +.br +.BR "lfs pcc unpin " [ --id | -i " \fIID\fR] <" \fIFILE ...> +.SH DESCRIPTION +Set/Clear flag on a lustre file so that it cannot or can be removed from local +cache storage by \fBlpcc_purge\fR. + +.SH OPTIONS +.TP +.BR --id | -i +The ARCHIVE +.I ID +to be pinned or un-pinned. + +.SH EXAMPLES +.TP +.B $ lfs pcc pin -i 2 /mnt/lustre/file +Set pcc pin flag on file \fB/mnt/lustre/file\fR for archive ID \fB2\fR so that +once it is copied to local cache storage, it will not be removed automatically +by \fBlpcc_purge\fR. +.TP +.B $ lfs pcc unpin -i 2 /mnt/lustre/file +Clear pcc pin flag on file \fB/mnt/lustre/file\fR for archived ID \fB2\fR +so that if it is in local cache storage, it can be removed automatically +by \fBlpcc_purge\fR. +.TP +.SH SEE ALSO +.BR lfs (1), +.BR lfs-pcc (1), +.BR llapi_pcc_attach (3) +.BR lfs-pcc-detach (1), +.BR lfs-pcc-state (1), + diff --git a/lustre/doc/lfs-pcc-unpin.1 b/lustre/doc/lfs-pcc-unpin.1 new file mode 100644 index 0000000..a1f4a1f --- /dev/null +++ b/lustre/doc/lfs-pcc-unpin.1 @@ -0,0 +1 @@ +.so man1/lfs-pcc-pin diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 2a799d4..2455e71 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -745,6 +745,9 @@ int llapi_pccdev_get(const char *mntpath); int llapi_pcc_del(const char *mntpath, const char *pccpath, enum lu_pcc_cleanup_flags flags); int llapi_pcc_clear(const char *mntpath, enum lu_pcc_cleanup_flags flags); +int llapi_pcc_pin_file(const char *path, __u32 id); +int llapi_pcc_unpin_file(const char *path, __u32 id); + /** @} llapi */ /* llapi_layout user interface */ diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index 7a30435..8b61084 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -1243,6 +1243,7 @@ struct lov_mds_md_v1 { /* LOV EA mds/wire data (little-endian) */ #define XATTR_NAME_DUMMY "trusted.dummy" #define XATTR_NAME_PROJID "trusted.projid" #define XATTR_NAME_DATAVER "trusted.dataver" +#define XATTR_NAME_PIN "trusted.pin" #define LL_XATTR_NAME_ENCRYPTION_CONTEXT_OLD XATTR_SECURITY_PREFIX"c" #define LL_XATTR_NAME_ENCRYPTION_CONTEXT XATTR_ENCRYPTION_PREFIX"c" diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index 05f62cf..1be67bb 100755 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -4485,6 +4485,9 @@ test_203() { local file=$DIR/$tfile local bs="1024" + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + setup_loopdev client $loopfile $mntpt 10 mkdir $hsm_root || error "mkdir $hsm_root failed" setup_pcc_mapping client \ @@ -4514,6 +4517,78 @@ test_203() { } run_test 203 "Verify attach/hit bytes statistics data" +test_204() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local file=$DIR/$tfile.dat + local xattrname=trusted.pin + local xattrvalue + + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + + (( $MDS1_VERSION >= $(version_code 2.15.61) )) || + skip "Need server version at least 2.15.61" + + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 + do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed" + setup_pcc_mapping $SINGLEAGT \ + "fname={*.dat}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1" + + do_facet $SINGLEAGT touch $file + + # pin with id=2 + do_facet $SINGLEAGT $LFS pcc pin -i $HSM_ARCHIVE_NUMBER $file || + error "failed to pcc pin $file" + do_facet $SINGLEAGT getfattr $file -d -m $xattrname + xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \ + --only-values) + [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" ]] || + error "incorrect xattr $xattrname=$xattrvalue" + + # pin with id=100 + do_facet $SINGLEAGT $LFS pcc pin -i 100 $file || + error "failed to pcc pin $file" + do_facet $SINGLEAGT getfattr $file -d -m $xattrname + xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \ + --only-values) + [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" && + "$xattrvalue" =~ "hsm: 100" ]] || + error "incorrect xattr $xattrname=$xattrvalue" + + # pin with id=2 again + do_facet $SINGLEAGT $LFS pcc pin -i $HSM_ARCHIVE_NUMBER $file || + error "failed to pcc pin $file" + do_facet $SINGLEAGT getfattr $file -d -m $xattrname + + # unpin id=100 + do_facet $SINGLEAGT $LFS pcc unpin -i 100 $file || + error "failed to pcc unpin $file" + do_facet $SINGLEAGT getfattr $file -d -m $xattrname + xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \ + --only-values) + [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" ]] || + error "incorrect xattr $xattrname=$xattrvalue" + + # unpin id=2 + do_facet $SINGLEAGT $LFS pcc unpin -i $HSM_ARCHIVE_NUMBER $file || + error "failed to pcc unpin $file" + do_facet $SINGLEAGT getfattr $file -d -m $xattrname + xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \ + --only-values) + [[ -z "$xattrvalue" ]] || + error "incorrect xattr $xattrname=$xattrvalue" + + # pin/unpin operation should NOT trigger autocache + check_lpcc_state $file "none" + + # pin/unpin operation should NOT block autocache triggered by read + do_facet $SINGLEAGT cat $file + check_lpcc_state $file "readonly" +} +run_test 204 "pin/unpin pcc flag" + complete_test $SECONDS check_and_cleanup_lustre exit_status diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 3ff3505..c365572 100755 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -147,6 +147,8 @@ static int lfs_pcc_attach(int argc, char **argv); static int lfs_pcc_attach_fid(int argc, char **argv); static int lfs_pcc_detach(int argc, char **argv); static int lfs_pcc_detach_fid(int argc, char **argv); +static int lfs_pcc_pin(int argc, char **argv); +static int lfs_pcc_unpin(int argc, char **argv); static int lfs_pcc_state(int argc, char **argv); static int lfs_pcc_delete(int argc, char **argv); static int lfs_pcc(int argc, char **argv); @@ -334,6 +336,13 @@ command_t pcc_cmdlist[] = { { .pc_name = "delete", .pc_func = lfs_pcc_delete, .pc_help = "Delete the PCC layout component for given files.\n" "usage: lfs pcc delete ...\n" }, + { .pc_name = "pin", .pc_func = lfs_pcc_pin, + .pc_help = "Pin files to prevent them from being removed from PCC.\n" + "usage: lfs pcc pin [--id|-i ID] FILE ...\n" + "\t-i: archive ID for PCC\n"}, + { .pc_name = "unpin", .pc_func = lfs_pcc_unpin, + .pc_help = "Un-pin files so that they can be removed from PCC.\n" + "usage: lfs pcc unpin [--id|-i ID] FILE ...\n"}, { .pc_help = NULL } }; @@ -585,7 +594,10 @@ command_t cmdlist[] = { "lfs pcc attach_fid - attach given files into PCC by FID(s)\n" "lfs pcc state - display the PCC state for given files\n" "lfs pcc detach - detach given files from Persistent Client Cache\n" - "lfs pcc detach_fid - detach given files from PCC by FID(s)\n"}, + "lfs pcc detach_fid - detach given files from PCC by FID(s)\n" + "lfs pcc delete - delete the PCC layout componenet for given files\n" + "lfs pcc pin - pin give files for PCC\n" + "lfs pcc unpin - unpin given files for PCC\n"}, { 0, 0, 0, NULL } }; @@ -14416,6 +14428,148 @@ static int lfs_pcc_delete(int argc, char **argv) return rc; } +static int lfs_pcc_pin(int argc, char **argv) +{ + int rc = 0, c; + const char *path; + char *end; + char fullpath[PATH_MAX]; + __u32 id = 0; + struct option long_opts[] = { + { .val = 'i', .name = "id", .has_arg = required_argument }, + { .name = NULL } }; + + optind = 0; + while ((c = getopt_long(argc, argv, "i:", + long_opts, NULL)) != -1) { + switch (c) { + case 'i': + errno = 0; + id = strtoul(optarg, &end, 0); + if (errno != 0 || *end != '\0' || + id == 0 || id >= UINT32_MAX) { + fprintf(stderr, + "error: %s: bad attach ID '%s'\n", + argv[0], optarg); + return CMD_HELP; + } + break; + case '?': + return CMD_HELP; + default: + fprintf(stderr, "%s: option '%s' unrecognized\n", + argv[0], argv[optind - 1]); + return CMD_HELP; + } + } + + /* check parameters */ + if (id == 0) { + fprintf(stderr, "%s: must specify -i|--id option\n", + argv[0]); + return CMD_HELP; + } + if (argc <= 1) { + fprintf(stderr, "%s: must specify one or more file names\n", + argv[0]); + return CMD_HELP; + } + + while (optind < argc) { + int rc2; + + path = argv[optind++]; + if (!realpath(path, fullpath)) { + fprintf(stderr, "%s: could not find path '%s': %s\n", + argv[0], path, strerror(errno)); + if (rc == 0) + rc = -EINVAL; + continue; + } + + rc2 = llapi_pcc_pin_file(fullpath, id); + if (rc2 < 0) { + fprintf(stderr, "%s: cannot pin '%s' for PCC: %s\n", + argv[0], path, strerror(-rc2)); + if (rc == 0) + rc = rc2; + } + } + + return rc; +} + +static int lfs_pcc_unpin(int argc, char **argv) +{ + int rc = 0, c; + const char *path; + char *end; + char fullpath[PATH_MAX]; + __u32 id = 0; + struct option long_opts[] = { + { .val = 'i', .name = "id", .has_arg = required_argument }, + { .name = NULL } }; + + optind = 0; + while ((c = getopt_long(argc, argv, "i:", + long_opts, NULL)) != -1) { + switch (c) { + case 'i': + errno = 0; + id = strtoul(optarg, &end, 0); + if (errno != 0 || *end != '\0' || + id == 0 || id > UINT32_MAX) { + fprintf(stderr, + "error: %s: bad attach ID '%s'\n", + argv[0], optarg); + return CMD_HELP; + } + break; + case '?': + return CMD_HELP; + default: + fprintf(stderr, "%s: option '%s' unrecognized\n", + argv[0], argv[optind - 1]); + return CMD_HELP; + } + } + /* check parameters */ + if (id == 0) { + fprintf(stderr, "%s: must specify -i|--id option\n", + argv[0]); + return CMD_HELP; + } + if (argc <= 1) { + fprintf(stderr, "%s: must specify one or more file names\n", + argv[0]); + return CMD_HELP; + } + + while (optind < argc) { + int rc2; + + path = argv[optind++]; + if (!realpath(path, fullpath)) { + fprintf(stderr, "%s: could not find path '%s': %s\n", + argv[0], path, strerror(errno)); + if (rc == 0) + rc = -EINVAL; + continue; + } + + rc2 = llapi_pcc_unpin_file(fullpath, id); + if (rc2 < 0) { + fprintf(stderr, "%s: cannot unpin '%s' for PCC: %s\n", + argv[0], path, strerror(-rc2)); + if (rc == 0) + rc = rc2; + } + } + + return rc; +} + + /** * lfs_pcc() - Parse and execute lfs pcc commands. * @argc: The count of lfs pcc command line arguments. diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c index 6636d90..2fb354c 100644 --- a/lustre/utils/liblustreapi_pcc.c +++ b/lustre/utils/liblustreapi_pcc.c @@ -31,6 +31,7 @@ */ #include #include +#include #include #include #include @@ -829,3 +830,210 @@ int llapi_pcc_clear(const char *mntpath, enum lu_pcc_cleanup_flags flags) return llapi_pcc_yaml_cb_helper(&pch); } + +#define PIN_YAML_HSM_STR "hsm" + +static int verify_pin_xattr_object(struct cYAML *yaml) +{ + struct cYAML *node = NULL; + + if (yaml->cy_type != CYAML_TYPE_OBJECT) + return -EINVAL; + + for (node = yaml->cy_child; node != NULL; node = node->cy_next) { + if (node->cy_type != CYAML_TYPE_NUMBER) + return -EINVAL; + } + + return 0; +} + +static int dump_pin_object(struct cYAML *yaml, char *buff, int buflen) +{ + int rc = 0, remained; + struct cYAML *node; + char *p = buff; + + if (yaml->cy_child == NULL) { + buff[0] = '\0'; + goto out; + } + + *p++ = '['; + for (node = yaml->cy_child; node != NULL; node = node->cy_next) { + if (node != yaml->cy_child) + *p++ = ','; + + remained = buff + buflen - p - 1; + rc = snprintf(p, remained, "%s: %ld", + node->cy_string, node->cy_valueint); + if (rc <= 0) { + rc = -errno; + goto out; + } else if (rc > remained) { + rc = -EOVERFLOW; + goto out; + } + p += rc; + } + *p++ = ']'; + *p = '\0'; + + rc = 0; +out: + return rc; +} + +static struct cYAML *read_pin_xattr_object(const char *path) +{ + int rc, i; + struct cYAML *yaml = NULL; + char buff[XATTR_SIZE_MAX]; + + rc = getxattr(path, XATTR_NAME_PIN, buff, sizeof(buff)); + if (rc < 0) + goto out; + + if (buff[0] != '[' || buff[rc - 1] != ']') { + llapi_error(LLAPI_MSG_ERROR, EINVAL, + "invalid pin string '%s'.", buff); + rc = -EINVAL; + goto out; + } + + for (i = 0; i < rc; i++) + if (buff[i] == ',') + buff[i] = '\n'; + + yaml = cYAML_build_tree(NULL, buff + 1, rc - 2, NULL, false); + if (yaml == NULL) { + llapi_error(LLAPI_MSG_ERROR, EINVAL, + "invalid pin string '%s'.", buff); + errno = -EINVAL; + goto out; + } + + rc = verify_pin_xattr_object(yaml); + if (rc) { + llapi_error(LLAPI_MSG_ERROR, -rc, "Invalid pin object."); + cYAML_free_tree(yaml); + yaml = NULL; + errno = -rc; + goto out; + } + +out: + return yaml; +} + +int llapi_pcc_pin_file(const char *path, __u32 id) +{ + int rc = 0; + struct cYAML *yaml, *node; + char buff[XATTR_SIZE_MAX]; + + yaml = read_pin_xattr_object(path); + + if (yaml == NULL && errno == ENODATA) { + snprintf(buff, sizeof(buff), "[%s: %d]", PIN_YAML_HSM_STR, id); + goto set; + + } + if (yaml == NULL) { + llapi_error(LLAPI_MSG_ERROR, errno, + "cannot read or parse pin xattr of file '%s'.", + path); + rc = -errno; + goto out; + } + + /* Now we have an valid pin object, search for existing entry */ + for (node = yaml->cy_child; node != NULL; node = node->cy_next) { + if (strcmp(node->cy_string, PIN_YAML_HSM_STR) == 0 && + node->cy_valueint == id) + break; + } + if (node != NULL) { + rc = 0; + goto out; + } + + node = cYAML_create_number(yaml, PIN_YAML_HSM_STR, id); + if (node == NULL) { + rc = -errno; + goto out; + } + + rc = dump_pin_object(yaml, buff, sizeof(buff)); + if (rc) + goto out; + +set: + rc = setxattr(path, XATTR_NAME_PIN, buff, strlen(buff), 0); + if (rc < 0) + rc = -errno; +out: + return rc; +} + +int llapi_pcc_unpin_file(const char *path, __u32 id) +{ + int rc = 0; + struct cYAML *yaml, *node; + char buff[XATTR_SIZE_MAX]; + + yaml = read_pin_xattr_object(path); + + if (yaml == NULL && errno == ENODATA) { + rc = 0; + goto out; + } + if (yaml == NULL) { + llapi_error(LLAPI_MSG_ERROR, errno, + "cannot read or parse pin xattr of file '%s'.", + path); + rc = -errno; + goto out; + } + + /* We have an valid pin object, search for the entry to be deleted */ + for (node = yaml->cy_child; node != NULL; node = node->cy_next) { + if (strcmp(node->cy_string, PIN_YAML_HSM_STR) == 0 && + node->cy_valueint == id) + break; + } + if (node == NULL) { + rc = 0; + goto out; + } + + /* Remove the node */ + if (node == yaml->cy_child) { + /* the first child */ + if (node->cy_next) + node->cy_next->cy_prev = NULL; + yaml->cy_child = node->cy_next; + } else { + /* not the first child */ + node->cy_prev->cy_next = node->cy_next; + if (node->cy_next) + node->cy_next->cy_prev = node->cy_prev; + } + node->cy_prev = node->cy_next = NULL; + cYAML_free_tree(node); + + rc = dump_pin_object(yaml, buff, sizeof(buff)); + if (rc) + goto out; + + if (strlen(buff) == 0) + rc = removexattr(path, XATTR_NAME_PIN); + else + rc = setxattr(path, XATTR_NAME_PIN, buff, strlen(buff), 0); + + if (rc < 0) + rc = -errno; + +out: + return rc; +}