From: Frank Zago Date: Fri, 24 Apr 2015 18:50:53 +0000 (-0500) Subject: LU-6494 hsm: add support to lfs to remove by fid X-Git-Tag: 2.7.55~47 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=refs%2Fchanges%2F81%2F14581%2F7;p=fs%2Flustre-release.git LU-6494 hsm: add support to lfs to remove by fid Once an archived file has been deleted from Lustre, the only way to remove it from an archive is to access the archive and delete it. Enhance lfs hsm_remove so it can take an archive id and a fid to delete that file. The FID has to be known somehow before the file is deleted. The lfs hsm_remove command gains a mntpath option to point to the Lustre mountpoint, and an archive ID option. Signed-off-by: frank zago Change-Id: Ib7c5ab07d66754c71038147aa4524fd3a737a88f Reviewed-on: http://review.whamcloud.com/14581 Tested-by: Jenkins Reviewed-by: Patrick Farrell Reviewed-by: Henri Doreau Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index 5d1f474..45f4255 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -2138,6 +2138,76 @@ test_28() { } run_test 28 "Concurrent archive/file remove" +test_29a() { + # Tests --mntpath and --archive options + + local archive_id=7 + copytool_setup $SINGLEAGT $MOUNT $archive_id + + # Bad archive number + $LFS hsm_remove -m $MOUNT -a 33 0x857765760:0x8:0x2 2>&1 | + grep "Invalid argument" || + error "unexpected hsm_remove failure (1)" + + # mntpath is present but file is given + $LFS hsm_remove --mntpath $MOUNT --archive 30 /qwerty/uyt 2>&1 | + grep "hsm: '/qwerty/uyt' is not a valid FID" || + error "unexpected hsm_remove failure (2)" + + copytool_cleanup +} +run_test 29a "Tests --mntpath and --archive options" + +test_29b() { + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(make_small $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + rm -f $f + + $LFS hsm_remove -m $MOUNT -a $HSM_ARCHIVE_NUMBER $fid + wait_request_state $fid REMOVE SUCCEED + + copytool_cleanup +} +run_test 29b "Archive/delete/remove by FID from the archive." + +test_29c() { + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local fid1=$(make_small $DIR/$tdir/$tfile-1) + local fid2=$(make_small $DIR/$tdir/$tfile-2) + local fid3=$(make_small $DIR/$tdir/$tfile-3) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tdir/$tfile-[1-3] + wait_request_state $fid1 ARCHIVE SUCCEED + wait_request_state $fid2 ARCHIVE SUCCEED + wait_request_state $fid3 ARCHIVE SUCCEED + + rm -f $DIR/$tdir/$tfile-[1-3] + + echo $fid1 > $DIR/$tdir/list + echo $fid2 >> $DIR/$tdir/list + echo $fid3 >> $DIR/$tdir/list + + $LFS hsm_remove -m $MOUNT -a $HSM_ARCHIVE_NUMBER \ + --filelist $DIR/$tdir/list + wait_request_state $fid1 REMOVE SUCCEED + wait_request_state $fid2 REMOVE SUCCEED + wait_request_state $fid3 REMOVE SUCCEED + + copytool_cleanup +} +run_test 29c "Archive/delete/remove by FID, using a file list." + test_30a() { # restore at exec cannot work on agent node (because of Linux kernel # protection of executables) diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index d7535b3..2100ac8 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -339,7 +339,13 @@ command_t cmdlist[] = { "usage: hsm_release [--filelist FILELIST] [--data DATA] ..."}, {"hsm_remove", lfs_hsm_remove, 0, "Remove file copy from external storage.\n" - "usage: hsm_remove [--filelist FILELIST] [--data DATA] ..."}, + "usage: hsm_remove [--filelist FILELIST] [--data DATA]\n" + " [--mntpath MOUNTPATH] [--archive NUM] ...\n" + "\n" + "Note: To remove files from the archive that have been deleted on\n" + "Lustre, set mntpath and optionally archive. In that case, all the\n" + "positional arguments and entries in the file list must be FIDs." + }, {"hsm_cancel", lfs_hsm_cancel, 0, "Cancel requests related to specified files.\n" "usage: hsm_cancel [--filelist FILELIST] [--data DATA] ..."}, @@ -3760,7 +3766,7 @@ static int lfs_hsm_clear(int argc, char **argv) * * \return 0 on success. */ -static int lfs_hsm_prepare_file(char *file, struct lu_fid *fid, +static int lfs_hsm_prepare_file(const char *file, struct lu_fid *fid, dev_t *last_dev) { struct stat st; @@ -3795,16 +3801,62 @@ static int lfs_hsm_prepare_file(char *file, struct lu_fid *fid, return 0; } +/* Fill an HSM HUR item with a given file name. + * + * If mntpath is set, then the filename is actually a FID, and no + * lookup on the filesystem will be performed. + * + * \param[in] hur the user request to fill + * \param[in] idx index of the item inside the HUR to fill + * \param[in] mntpath mountpoint of Lustre + * \param[in] fname filename (if mtnpath is NULL) + * or FID (if mntpath is set) + * \param[in] last_dev pointer to last device id used + * + * \retval 0 on success + * \retval CMD_HELP or a negative errno on error + */ +static int fill_hur_item(struct hsm_user_request *hur, unsigned int idx, + const char *mntpath, const char *fname, + dev_t *last_dev) +{ + struct hsm_user_item *hui = &hur->hur_user_item[idx]; + int rc; + + hui->hui_extent.length = -1; + + if (mntpath != NULL) { + if (*fname == '[') + fname++; + rc = sscanf(fname, SFID, RFID(&hui->hui_fid)); + if (rc == 3) { + rc = 0; + } else { + fprintf(stderr, "hsm: '%s' is not a valid FID\n", + fname); + rc = -EINVAL; + } + } else { + rc = lfs_hsm_prepare_file(fname, &hui->hui_fid, last_dev); + } + + if (rc == 0) + hur->hur_request.hr_itemcount++; + + return rc; +} + static int lfs_hsm_request(int argc, char **argv, int action) { struct option long_opts[] = { {"filelist", 1, 0, 'l'}, {"data", 1, 0, 'D'}, {"archive", 1, 0, 'a'}, + {"mntpath", 1, 0, 'm'}, {0, 0, 0, 0} }; dev_t last_dev = 0; - char short_opts[] = "l:D:a:"; + char short_opts[] = "l:D:a:m:"; struct hsm_user_request *hur, *oldhur; int c, i; size_t len; @@ -3817,7 +3869,8 @@ static int lfs_hsm_request(int argc, char **argv, int action) int archive_id = 0; FILE *fp; int nbfile_alloc = 0; - char some_file[PATH_MAX+1] = ""; + char *some_file = NULL; + char *mntpath = NULL; int rc; if (argc < 2) @@ -3833,14 +3886,21 @@ static int lfs_hsm_request(int argc, char **argv, int action) opaque = optarg; break; case 'a': - if (action != HUA_ARCHIVE) { + if (action != HUA_ARCHIVE && + action != HUA_REMOVE) { fprintf(stderr, "error: -a is supported only " - "when archiving\n"); + "when archiving or removing\n"); return CMD_HELP; } archive_id = atoi(optarg); break; + case 'm': + if (some_file == NULL) { + mntpath = optarg; + some_file = strdup(optarg); + } + break; case '?': return CMD_HELP; default: @@ -3874,20 +3934,12 @@ static int lfs_hsm_request(int argc, char **argv, int action) hur->hur_request.hr_flags = 0; /* All remaining args are files, add them */ - if (nbfile != 0) { - if (strlen(argv[optind]) > sizeof(some_file)-1) { - free(hur); - return -E2BIG; - } - strncpy(some_file, argv[optind], sizeof(some_file)); - } + if (nbfile != 0 && some_file == NULL) + some_file = strdup(argv[optind]); for (i = 0; i < nbfile; i++) { - hur->hur_user_item[i].hui_extent.length = -1; - rc = lfs_hsm_prepare_file(argv[optind + i], - &hur->hur_user_item[i].hui_fid, - &last_dev); - hur->hur_request.hr_itemcount++; + rc = fill_hur_item(hur, i, mntpath, argv[optind + i], + &last_dev); if (rc) goto out_free; } @@ -3905,12 +3957,11 @@ static int lfs_hsm_request(int argc, char **argv, int action) } while ((rc = getline(&line, &len, fp)) != -1) { - struct hsm_user_item *hui; - /* If allocated buffer was too small, get something * larger */ if (nbfile_alloc <= hur->hur_request.hr_itemcount) { ssize_t size; + nbfile_alloc = nbfile_alloc * 2 + 1; oldhur = hur; hur = llapi_hsm_user_request_alloc(nbfile_alloc, @@ -3944,25 +3995,21 @@ static int lfs_hsm_request(int argc, char **argv, int action) if (line[strlen(line) - 1] == '\n') line[strlen(line) - 1] = '\0'; - hui = - &hur->hur_user_item[hur->hur_request.hr_itemcount]; - hui->hui_extent.length = -1; - rc = lfs_hsm_prepare_file(line, &hui->hui_fid, - &last_dev); - hur->hur_request.hr_itemcount++; + rc = fill_hur_item(hur, hur->hur_request.hr_itemcount, + mntpath, line, &last_dev); if (rc) { fclose(fp); goto out_free; } - if ((some_file[0] == '\0') && - (strlen(line) < sizeof(some_file))) - strcpy(some_file, line); + if (some_file == NULL) { + some_file = line; + line = NULL; + } } rc = fclose(fp); - if (line) - free(line); + free(line); } /* If a --data was used, add it to the request */ @@ -3983,6 +4030,7 @@ static int lfs_hsm_request(int argc, char **argv, int action) } out_free: + free(some_file); free(hur); return rc; }