LU-6494 hsm: add support to lfs to remove by fid

Once an archived file has been deleted from Lustre, the only way to
remove it from an archive is to access the archive and delete it.

Enhance lfs hsm_remove so it can take an archive id and a fid to
delete that file. The FID has to be known somehow before the file is
deleted.

The lfs hsm_remove command gains a mntpath option to point to the
Lustre mountpoint, and an archive ID option.

Signed-off-by: frank zago <fzago@cray.com>
Change-Id: Ib7c5ab07d66754c71038147aa4524fd3a737a88f
Reviewed-on: http://review.whamcloud.com/14581
Tested-by: Jenkins
Reviewed-by: Patrick Farrell <paf@cray.com>
Reviewed-by: Henri Doreau <henri.doreau@cea.fr>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index 5d1f474..45f4255 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -2138,6 +2138,76 @@ test_28() {
 }
 run_test 28 "Concurrent archive/file remove"
 
+test_29a() {
+       # Tests --mntpath and --archive options
+
+       local archive_id=7
+       copytool_setup $SINGLEAGT $MOUNT $archive_id
+
+       # Bad archive number
+       $LFS hsm_remove -m $MOUNT -a 33 0x857765760:0x8:0x2 2>&1 |
+               grep "Invalid argument" ||
+               error "unexpected hsm_remove failure (1)"
+
+       # mntpath is present but file is given
+       $LFS hsm_remove --mntpath $MOUNT --archive 30 /qwerty/uyt 2>&1 |
+               grep "hsm: '/qwerty/uyt' is not a valid FID" ||
+               error "unexpected hsm_remove failure (2)"
+
+       copytool_cleanup
+}
+run_test 29a "Tests --mntpath and --archive options"
+
+test_29b() {
+       # test needs a running copytool
+       copytool_setup
+
+       mkdir -p $DIR/$tdir
+       local f=$DIR/$tdir/$tfile
+       local fid=$(make_small $f)
+
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f
+       wait_request_state $fid ARCHIVE SUCCEED
+
+       rm -f $f
+
+       $LFS hsm_remove -m $MOUNT -a $HSM_ARCHIVE_NUMBER $fid
+       wait_request_state $fid REMOVE SUCCEED
+
+       copytool_cleanup
+}
+run_test 29b "Archive/delete/remove by FID from the archive."
+
+test_29c() {
+       # test needs a running copytool
+       copytool_setup
+
+       mkdir -p $DIR/$tdir
+       local fid1=$(make_small $DIR/$tdir/$tfile-1)
+       local fid2=$(make_small $DIR/$tdir/$tfile-2)
+       local fid3=$(make_small $DIR/$tdir/$tfile-3)
+
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tdir/$tfile-[1-3]
+       wait_request_state $fid1 ARCHIVE SUCCEED
+       wait_request_state $fid2 ARCHIVE SUCCEED
+       wait_request_state $fid3 ARCHIVE SUCCEED
+
+       rm -f $DIR/$tdir/$tfile-[1-3]
+
+       echo $fid1 > $DIR/$tdir/list
+       echo $fid2 >> $DIR/$tdir/list
+       echo $fid3 >> $DIR/$tdir/list
+
+       $LFS hsm_remove -m $MOUNT -a $HSM_ARCHIVE_NUMBER \
+               --filelist $DIR/$tdir/list
+       wait_request_state $fid1 REMOVE SUCCEED
+       wait_request_state $fid2 REMOVE SUCCEED
+       wait_request_state $fid3 REMOVE SUCCEED
+
+       copytool_cleanup
+}
+run_test 29c "Archive/delete/remove by FID, using a file list."
+
 test_30a() {
        # restore at exec cannot work on agent node (because of Linux kernel
        # protection of executables)

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index d7535b3..2100ac8 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -339,7 +339,13 @@ command_t cmdlist[] = {
         "usage: hsm_release [--filelist FILELIST] [--data DATA] <file> ..."},
        {"hsm_remove", lfs_hsm_remove, 0,
         "Remove file copy from external storage.\n"
-        "usage: hsm_remove [--filelist FILELIST] [--data DATA] <file> ..."},
+        "usage: hsm_remove [--filelist FILELIST] [--data DATA]\n"
+        "                  [--mntpath MOUNTPATH] [--archive NUM] <file|FID> ...\n"
+        "\n"
+        "Note: To remove files from the archive that have been deleted on\n"
+        "Lustre, set mntpath and optionally archive. In that case, all the\n"
+        "positional arguments and entries in the file list must be FIDs."
+       },
        {"hsm_cancel", lfs_hsm_cancel, 0,
         "Cancel requests related to specified files.\n"
         "usage: hsm_cancel [--filelist FILELIST] [--data DATA] <file> ..."},
@@ -3760,7 +3766,7 @@ static int lfs_hsm_clear(int argc, char **argv)
  *
  * \return 0 on success.
  */
-static int lfs_hsm_prepare_file(char *file, struct lu_fid *fid,
+static int lfs_hsm_prepare_file(const char *file, struct lu_fid *fid,
                                dev_t *last_dev)
 {
        struct stat     st;
@@ -3795,16 +3801,62 @@ static int lfs_hsm_prepare_file(char *file, struct lu_fid *fid,
        return 0;
 }
 
+/* Fill an HSM HUR item with a given file name.
+ *
+ * If mntpath is set, then the filename is actually a FID, and no
+ * lookup on the filesystem will be performed.
+ *
+ * \param[in]  hur         the user request to fill
+ * \param[in]  idx         index of the item inside the HUR to fill
+ * \param[in]  mntpath     mountpoint of Lustre
+ * \param[in]  fname       filename (if mtnpath is NULL)
+ *                         or FID (if mntpath is set)
+ * \param[in]  last_dev    pointer to last device id used
+ *
+ * \retval 0 on success
+ * \retval CMD_HELP or a negative errno on error
+ */
+static int fill_hur_item(struct hsm_user_request *hur, unsigned int idx,
+                        const char *mntpath, const char *fname,
+                        dev_t *last_dev)
+{
+       struct hsm_user_item *hui = &hur->hur_user_item[idx];
+       int rc;
+
+       hui->hui_extent.length = -1;
+
+       if (mntpath != NULL) {
+               if (*fname == '[')
+                       fname++;
+               rc = sscanf(fname, SFID, RFID(&hui->hui_fid));
+               if (rc == 3) {
+                       rc = 0;
+               } else {
+                       fprintf(stderr, "hsm: '%s' is not a valid FID\n",
+                               fname);
+                       rc = -EINVAL;
+               }
+       } else {
+               rc = lfs_hsm_prepare_file(fname, &hui->hui_fid, last_dev);
+       }
+
+       if (rc == 0)
+               hur->hur_request.hr_itemcount++;
+
+       return rc;
+}
+
 static int lfs_hsm_request(int argc, char **argv, int action)
 {
        struct option            long_opts[] = {
                {"filelist", 1, 0, 'l'},
                {"data", 1, 0, 'D'},
                {"archive", 1, 0, 'a'},
+               {"mntpath", 1, 0, 'm'},
                {0, 0, 0, 0}
        };
        dev_t                    last_dev = 0;
-       char                     short_opts[] = "l:D:a:";
+       char                     short_opts[] = "l:D:a:m:";
        struct hsm_user_request *hur, *oldhur;
        int                      c, i;
        size_t                   len;
@@ -3817,7 +3869,8 @@ static int lfs_hsm_request(int argc, char **argv, int action)
        int                      archive_id = 0;
        FILE                    *fp;
        int                      nbfile_alloc = 0;
-       char                     some_file[PATH_MAX+1] = "";
+       char                    *some_file = NULL;
+       char                    *mntpath = NULL;
        int                      rc;
 
        if (argc < 2)
@@ -3833,14 +3886,21 @@ static int lfs_hsm_request(int argc, char **argv, int action)
                        opaque = optarg;
                        break;
                case 'a':
-                       if (action != HUA_ARCHIVE) {
+                       if (action != HUA_ARCHIVE &&
+                           action != HUA_REMOVE) {
                                fprintf(stderr,
                                        "error: -a is supported only "
-                                       "when archiving\n");
+                                       "when archiving or removing\n");
                                return CMD_HELP;
                        }
                        archive_id = atoi(optarg);
                        break;
+               case 'm':
+                       if (some_file == NULL) {
+                               mntpath = optarg;
+                               some_file = strdup(optarg);
+                       }
+                       break;
                case '?':
                        return CMD_HELP;
                default:
@@ -3874,20 +3934,12 @@ static int lfs_hsm_request(int argc, char **argv, int action)
        hur->hur_request.hr_flags = 0;
 
        /* All remaining args are files, add them */
-       if (nbfile != 0) {
-               if (strlen(argv[optind]) > sizeof(some_file)-1) {
-                       free(hur);
-                       return -E2BIG;
-               }
-               strncpy(some_file, argv[optind], sizeof(some_file));
-       }
+       if (nbfile != 0 && some_file == NULL)
+               some_file = strdup(argv[optind]);
 
        for (i = 0; i < nbfile; i++) {
-               hur->hur_user_item[i].hui_extent.length = -1;
-               rc = lfs_hsm_prepare_file(argv[optind + i],
-                                         &hur->hur_user_item[i].hui_fid,
-                                         &last_dev);
-               hur->hur_request.hr_itemcount++;
+               rc = fill_hur_item(hur, i, mntpath, argv[optind + i],
+                                  &last_dev);
                if (rc)
                        goto out_free;
        }
@@ -3905,12 +3957,11 @@ static int lfs_hsm_request(int argc, char **argv, int action)
                }
 
                while ((rc = getline(&line, &len, fp)) != -1) {
-                       struct hsm_user_item *hui;
-
                        /* If allocated buffer was too small, get something
                         * larger */
                        if (nbfile_alloc <= hur->hur_request.hr_itemcount) {
                                ssize_t size;
+
                                nbfile_alloc = nbfile_alloc * 2 + 1;
                                oldhur = hur;
                                hur = llapi_hsm_user_request_alloc(nbfile_alloc,
@@ -3944,25 +3995,21 @@ static int lfs_hsm_request(int argc, char **argv, int action)
                        if (line[strlen(line) - 1] == '\n')
                                line[strlen(line) - 1] = '\0';
 
-                       hui =
-                            &hur->hur_user_item[hur->hur_request.hr_itemcount];
-                       hui->hui_extent.length = -1;
-                       rc = lfs_hsm_prepare_file(line, &hui->hui_fid,
-                                                 &last_dev);
-                       hur->hur_request.hr_itemcount++;
+                       rc = fill_hur_item(hur, hur->hur_request.hr_itemcount,
+                                          mntpath, line, &last_dev);
                        if (rc) {
                                fclose(fp);
                                goto out_free;
                        }
 
-                       if ((some_file[0] == '\0') &&
-                           (strlen(line) < sizeof(some_file)))
-                               strcpy(some_file, line);
+                       if (some_file == NULL) {
+                               some_file = line;
+                               line = NULL;
+                       }
                }
 
                rc = fclose(fp);
-               if (line)
-                       free(line);
+               free(line);
        }
 
        /* If a --data was used, add it to the request */
@@ -3983,6 +4030,7 @@ static int lfs_hsm_request(int argc, char **argv, int action)
        }
 
 out_free:
+       free(some_file);
        free(hur);
        return rc;
 }