Whamcloud - gitweb
LU-13404 utils: fix lfs mirror duplicate file check
[fs/lustre-release.git] / lustre / utils / lfs.c
index 9a4a194..44f2653 100644 (file)
@@ -76,7 +76,7 @@
 #include <linux/lustre/lustre_ver.h>
 #include <linux/lustre/lustre_param.h>
 #include <linux/lnet/nidstr.h>
-#include <cyaml.h>
+#include <lnetconfig/cyaml.h>
 
 #ifndef ARRAY_SIZE
 # define ARRAY_SIZE(a) ((sizeof(a)) / (sizeof((a)[0])))
@@ -186,16 +186,17 @@ static inline int lfs_mirror_delete(int argc, char **argv)
        "                 [--stripe-index|-i <start_ost_idx>]\n"        \
        "                 [--stripe-size|-S <stripe_size>]\n"           \
        "                 [--layout|-L <pattern>]\n"                    \
-       "                 [--pool|-p <pool_name>]\n"                    \
+       "                 [--mirror_count|-N[mirror_count]]\n"          \
        "                 [--ost|-o <ost_indices>]\n"                   \
+       "                 [--pool|-p <pool_name>]\n"                    \
        "                 [--yaml|-y <yaml_template_file>]\n"           \
        "                 [--copy=<lustre_src>]\n"
 
 #define SSM_HELP_COMMON \
        "\tstripe_count: Number of OSTs to stripe over (0=fs default, -1 all)\n" \
        "\t              Using -C instead of -c allows overstriping, which\n" \
-        "\t             will place more than one stripe per OST if\n" \
-        "\t             stripe_count is greater than the number of OSTs\n" \
+       "\t              will place more than one stripe per OST if\n" \
+       "\t              stripe_count is greater than the number of OSTs\n" \
        "\tstart_ost_idx: OST index of first stripe (-1=default round robin)\n"\
        "\tstripe_size:  Number of bytes on each OST (0=fs default)\n" \
        "\t              Can be specified with K, M or G (for KB, MB, GB\n" \
@@ -207,7 +208,7 @@ static inline int lfs_mirror_delete(int argc, char **argv)
        "\t                -o <ost_1>,<ost_i>-<ost_j>,<ost_n>\n"        \
        "\t              Or:\n"                                         \
        "\t                -o <ost_1> -o <ost_i>-<ost_j> -o <ost_n>\n"  \
-       "\t              If --pool is set with --ost, then the OSTs\n"  \
+       "\t              If --pool is set with --ost then the OSTs\n"   \
        "\t              must be the members of the pool.\n"            \
        "\tcomp_end:     Extent end of component, start after previous end.\n"\
        "\t              Can be specified with K, M or G (for KB, MB, GB\n" \
@@ -227,7 +228,7 @@ static inline int lfs_mirror_delete(int argc, char **argv)
        "\t              it must follow the option without a space.\n"         \
        "\t              The option can also be repeated multiple times to\n"  \
        "\t              separate mirrors that have different layouts.\n"      \
-       "\tsetstripe options: Mirror layout\n"                                 \
+       "\tSETSTRIPE_OPTIONS: Mirror layout as with 'setstripe'\n"             \
        "\t              It can be a plain layout or a composite layout.\n"    \
        "\t              If not specified, the stripe options inherited\n"     \
        "\t              from the previous component will be used.\n"          \
@@ -244,8 +245,8 @@ static inline int lfs_mirror_delete(int argc, char **argv)
        "\t              file.\n"
 
 #define MIRROR_EXTEND_USAGE                                                   \
-       "                 <--mirror-count|-N[mirror_count]>\n"                 \
-       "                 [setstripe options|-f <victim_file>]\n"              \
+       "                 {--mirror-count|-N[mirror_count]}\n"                 \
+       "                 [SETSTRIPE_OPTIONS|-f|--file <victim_file>]\n"       \
        "                 [--no-verify]\n"
 
 #define SETSTRIPE_USAGE                                                        \
@@ -275,14 +276,14 @@ static inline int lfs_mirror_delete(int argc, char **argv)
        "\tstripe_count: stripe count of the striped directory\n"       \
        "\tmdt_index: MDT index of first stripe\n"                      \
        "\tmdt_hash:  hash type of the striped directory. mdt types:\n" \
-       "       fnv_1a_64 FNV-1a hash algorithm (default)\n"            \
+       "       crush     CRUSH hash algorithm (default)\n" \
+       "       fnv_1a_64 FNV-1a hash algorithm\n"              \
        "       all_char  sum of characters % MDT_COUNT (not recommended)\n" \
-       "       space     create subdirectories with balanced space usage\n" \
        "\tdefault_stripe: set default dirstripe of the directory\n"    \
        "\tmode: the file access permission of the directory (octal)\n" \
        "To create dir with a foreign (free format) layout :\n" \
        "setdirstripe|mkdir --foreign[=<foreign_type>] -x|-xattr <string> " \
-               "[--mode|-m mode] [--flags <hex>] <dir>\n" \
+               "[--mode|-o mode] [--flags <hex>] <dir>\n" \
        "\tmode: the mode of the directory\n" \
        "\tforeign_type: none or daos\n"
 
@@ -294,7 +295,7 @@ command_t mirror_cmdlist[] = {
          .pc_help = "Create a mirrored file.\n"
                "usage: lfs mirror create "
                "<--mirror-count|-N[mirror_count]> "
-               "[setstripe options] ... <filename|directory> ...\n"
+               "[SETSTRIPE_OPTIONS] ... <filename|directory> ...\n"
          MIRROR_CREATE_HELP },
        { .pc_name = "delete", .pc_func = lfs_mirror_delete,
          .pc_help = "delete a mirror from a file.\n"
@@ -304,7 +305,7 @@ command_t mirror_cmdlist[] = {
          .pc_help = "Extend a mirrored file.\n"
                "usage: lfs mirror extend "
                "<--mirror-count|-N[mirror_count]> [--no-verify] "
-               "[setstripe options|-f <victim_file>] ... <filename> ...\n"
+               "[SETSTRIPE_OPTIONS|-f <victim_file>] ... <filename> ...\n"
          MIRROR_EXTEND_HELP },
        { .pc_name = "split", .pc_func = lfs_mirror_split,
          .pc_help = "Split a mirrored file.\n"
@@ -392,10 +393,10 @@ command_t cmdlist[] = {
         SSM_CMD_COMMON("setstripe --component-add")
         SSM_HELP_COMMON
         "To totally delete the default striping from an existing directory:\n"
-        "usage: setstripe -d <directory>\n"
+        "usage: setstripe [--delete|-d] <directory>\n"
         " or\n"
         "To create a mirrored file or set s default mirror layout on a directory:\n"
-        "usage: setstripe -N[mirror_count] [STRIPE_OPTIONS] <directory|filename>\n"
+        "usage: setstripe {--mirror-count|-N}[mirror_count] [SETSTRIPE_OPTIONS] <directory|filename>\n"
         " or\n"
         "To delete the last component(s) from an existing composite file\n"
         "(note that this will also delete any data in those components):\n"
@@ -474,7 +475,7 @@ command_t cmdlist[] = {
         "     [[!] --component-count [+-]<comp_cnt>]\n"
         "     [[!] --component-start [+-]N[kMGTPE]]\n"
         "     [[!] --component-end|-E [+-]N[kMGTPE]]\n"
-        "     [[!] --component-flags <comp_flags>]\n"
+        "     [[!] --component-flags {init,stale,prefer,offline,nosync,extension}]\n"
         "     [[!] --mirror-count|-N [+-]<n>]\n"
         "     [[!] --mirror-state <[^]state>]\n"
         "     [[!] --mdt-count|-T [+-]<stripes>]\n"
@@ -497,7 +498,8 @@ command_t cmdlist[] = {
        {"df", lfs_df, 0,
         "report filesystem disk space usage or inodes usage "
         "of each MDS and all OSDs or a batch belonging to a specific pool.\n"
-        "Usage: df [-i] [-h] [--lazy|-l] [--pool|-p <fsname>[.<pool>] [path]"},
+        "Usage: df [--inodes|-i] [--human-readable|-h] [--lazy|-l]\n"
+        "          [--pool|-p <fsname>[.<pool>]] [path]"},
        {"getname", lfs_getname, 0,
         "list instances and specified mount points [for specified path only]\n"
         "Usage: getname [--help|-h] [--instance|-i] [--fsname|-n] [path ...]"},
@@ -512,8 +514,8 @@ command_t cmdlist[] = {
          "                [--inode-softlimit <inode-softlimit>]\n"
          "                [--inode-hardlimit <inode-hardlimit>] <filesystem>\n"
         "       setquota [-t] <-u|--user|-g|--group|-p|--projid>\n"
-         "                [--block-grace <block-grace>]\n"
-         "                [--inode-grace <inode-grace>] <filesystem>\n"
+        "                [--block-grace 'notify'|<block-grace>]\n"
+        "                [--inode-grace 'notify'|<inode-grace>] <filesystem>\n"
         "       setquota <-U|-G|-P>\n"
         "                -b <block-softlimit> -B <block-hardlimit>\n"
         "                -i <inode-softlimit> -I <inode-hardlimit> <filesystem>\n"
@@ -536,8 +538,11 @@ command_t cmdlist[] = {
         "      Quota space rebalancing process will stop when this mininum\n"
         "      value is reached. As a result, quota exceeded can be returned\n"
         "      while many targets still have 1MB or 1K inodes of spare\n"
-        "      quota space."},
-        {"quota", lfs_quota, 0, "Display disk usage and limits.\n"
+        "      quota space.\n\n"
+        "      When setting the grace time, 'notify' can be used as grace to\n"
+        "      be notified after the quota is over soft limit but prevents\n"
+        "      the soft limit from becoming the hard limit."},
+       {"quota", lfs_quota, 0, "Display disk usage and limits.\n"
         "usage: quota [-q] [-v] [-h] [-o <obd_uuid>|-i <mdt_idx>|-I "
                       "<ost_idx>]\n"
         "             [<-u|-g|-p> <uname>|<uid>|<gname>|<gid>|<projid>] <filesystem>\n"
@@ -574,7 +579,7 @@ command_t cmdlist[] = {
        {"rmfid", lfs_rmfid, 0, "Remove file(s) by FID(s)\n"
         "usage: rmfid <fsname|rootpath> <fid> ..."},
        {"data_version", lfs_data_version, 0, "Display file data version for "
-        "a given path.\n" "usage: data_version -[n|r|w] <path>"},
+        "a given path.\n" "usage: data_version [-n|-r|-w] <path>"},
        {"hsm_state", lfs_hsm_state, 0, "Display the HSM information (states, "
         "undergoing actions) for given files.\n usage: hsm_state <file> ..."},
        {"hsm_set", lfs_hsm_set, 0, "Set HSM user flag on specified files.\n"
@@ -623,7 +628,8 @@ command_t cmdlist[] = {
         "                      it's the MDT index of first stripe\n"
         "\tmdt_count:  number of MDTs to stripe a directory over\n"
         "\tmdt_hash:   hash type of the striped directory. mdt types:\n"
-        "                      fnv_1a_64 FNV-1a hash algorithm (default)\n"
+        "                      crush     CRUSH hash algorithm (default)\n"
+        "                      fnv_1a_64 FNV-1a hash algorithm\n"
         "                      all_char  sum of characters % MDT_COUNT\n"
         "\n"
         "migrate file objects from one OST "
@@ -816,12 +822,12 @@ migrate_open_files(const char *name, __u64 migration_flags,
                }
 
                /* create, open a volatile file, use caching (ie no directio) */
-               if (param != NULL)
-                       fdv = llapi_file_open_param(volatile_file, open_flags,
-                                                   open_mode, param);
-               else
+               if (layout)
                        fdv = lfs_component_create(volatile_file, open_flags,
                                                   open_mode, layout);
+               else
+                       fdv = llapi_file_open_param(volatile_file, open_flags,
+                                                   open_mode, param);
        } while (fdv < 0 && (rc = fdv) == -EEXIST);
 
        if (rc < 0) {
@@ -3577,6 +3583,11 @@ static int lfs_setstripe_internal(int argc, char **argv,
                goto usage_error;
        }
 
+       /* lfs migrate $filename should keep the file's layout by default */
+       if (migrate_mode && !setstripe_args_specified(&lsa) && !layout &&
+           !from_yaml)
+               from_copy = true;
+
        if (xattr && !foreign_mode) {
                /* only print a warning as this is harmless and will be ignored
                 */
@@ -3727,8 +3738,8 @@ static int lfs_setstripe_internal(int argc, char **argv,
 
        if ((from_yaml || from_copy) &&
            (setstripe_args_specified(&lsa) || layout != NULL)) {
-               fprintf(stderr, "error: %s: can't specify --yaml with "
-                       "-c, -S, -i, -o, -p or -E options.\n",
+               fprintf(stderr, "error: %s: can't specify --yaml or --copy with"
+                       " -c, -S, -i, -o, -p or -E options.\n",
                        argv[0]);
                goto error;
        }
@@ -3774,7 +3785,7 @@ static int lfs_setstripe_internal(int argc, char **argv,
                if (lsa.lsa_pattern != LLAPI_LAYOUT_RAID0)
                        lmu->lum_hash_type = lsa.lsa_pattern;
                else
-                       lmu->lum_hash_type = LMV_HASH_TYPE_DEFAULT;
+                       lmu->lum_hash_type = LMV_HASH_TYPE_UNKNOWN;
                if (lsa.lsa_pool_name) {
                        strncpy(lmu->lum_pool_name, lsa.lsa_pool_name,
                                sizeof(lmu->lum_pool_name) - 1);
@@ -3868,17 +3879,19 @@ static int lfs_setstripe_internal(int argc, char **argv,
                                argv[0], template);
                        goto error;
                }
-       } else if (from_copy) {
-               layout = llapi_layout_get_by_path(template, 0);
-               if (layout == NULL) {
-                       fprintf(stderr,
-                           "%s: can't create composite layout from file %s.\n",
-                               progname, template);
-                       goto error;
-               }
        }
 
        for (fname = argv[optind]; fname != NULL; fname = argv[++optind]) {
+               if (from_copy) {
+                       layout = llapi_layout_get_by_path(template ?: fname, 0);
+                       if (layout == NULL) {
+                               fprintf(stderr, "%s: can't create composite "
+                                       "layout from file %s.\n",
+                                       progname, template ?: fname);
+                               goto error;
+                       }
+               }
+
                if (migrate_mdt_mode) {
                        result = llapi_migrate_mdt(fname, &migrate_mdt_param);
                } else if (migrate_mode) {
@@ -6010,7 +6023,7 @@ static int lfs_setdirstripe(int argc, char **argv)
        if (lsa.lsa_pattern != LLAPI_LAYOUT_RAID0)
                param->lsp_stripe_pattern = lsa.lsa_pattern;
        else
-               param->lsp_stripe_pattern = LMV_HASH_TYPE_DEFAULT;
+               param->lsp_stripe_pattern = LMV_HASH_TYPE_UNKNOWN;
        param->lsp_pool = lsa.lsa_pool_name;
        param->lsp_is_specific = false;
        if (lsa.lsa_nr_tgts > 1) {
@@ -6134,6 +6147,7 @@ static int lfs_mv(int argc, char **argv)
                return CMD_HELP;
        }
 
+       lmu.lum_hash_type = LMV_HASH_TYPE_UNKNOWN;
 
        /* initialize migrate mdt parameters */
        param.fp_lmv_md = &lmu;
@@ -6141,7 +6155,7 @@ static int lfs_mv(int argc, char **argv)
        rc = llapi_migrate_mdt(argv[optind], &param);
        if (rc != 0)
                fprintf(stderr, "%s mv: cannot migrate '%s' to MDT%04x: %s\n",
-                       progname, argv[optind], param.fp_mdt_index,
+                       progname, argv[optind], lmu.lum_stripe_offset,
                        strerror(-rc));
        return rc;
 }
@@ -6493,28 +6507,41 @@ quota_type:
                        }
                        qctl.qc_type = qtype;
                        break;
-                case 'b':
-                        if ((dqi->dqi_bgrace = str2sec(optarg)) == ULONG_MAX) {
-                                fprintf(stderr, "error: bad block-grace: %s\n",
-                                        optarg);
-                                return CMD_HELP;
-                        }
-                        dqb->dqb_valid |= QIF_BTIME;
-                        break;
-                case 'i':
-                        if ((dqi->dqi_igrace = str2sec(optarg)) == ULONG_MAX) {
-                                fprintf(stderr, "error: bad inode-grace: %s\n",
-                                        optarg);
-                                return CMD_HELP;
-                        }
-                        dqb->dqb_valid |= QIF_ITIME;
-                        break;
-                case 't': /* Yes, of course! */
-                        break;
-                default: /* getopt prints error message for us when opterr != 0 */
-                        return CMD_HELP;
-                }
-        }
+               case 'b':
+                       if (strncmp(optarg, NOTIFY_GRACE,
+                                   strlen(NOTIFY_GRACE)) == 0) {
+                               dqi->dqi_bgrace = NOTIFY_GRACE_TIME;
+                       } else {
+                               dqi->dqi_bgrace = str2sec(optarg);
+                               if (dqi->dqi_bgrace >= NOTIFY_GRACE_TIME) {
+                                       fprintf(stderr, "error: bad "
+                                               "block-grace: %s\n", optarg);
+                                       return CMD_HELP;
+                               }
+                       }
+                       dqb->dqb_valid |= QIF_BTIME;
+                       break;
+               case 'i':
+                       if (strncmp(optarg, NOTIFY_GRACE,
+                                   strlen(NOTIFY_GRACE)) == 0) {
+                               dqi->dqi_igrace = NOTIFY_GRACE_TIME;
+                       } else {
+                               dqi->dqi_igrace = str2sec(optarg);
+                               if (dqi->dqi_igrace >= NOTIFY_GRACE_TIME) {
+                                       fprintf(stderr, "error: bad "
+                                               "inode-grace: %s\n", optarg);
+                                       return CMD_HELP;
+                               }
+                       }
+                       dqb->dqb_valid |= QIF_ITIME;
+                       break;
+               case 't': /* Yes, of course! */
+                       break;
+               /* getopt prints error message for us when opterr != 0 */
+               default:
+                       return CMD_HELP;
+               }
+       }
 
        if (qctl.qc_type == ALLQUOTA) {
                fprintf(stderr, "error: neither -u, -g nor -p specified\n");
@@ -6526,12 +6553,6 @@ quota_type:
                 return CMD_HELP;
         }
 
-       if ((dqb->dqb_valid | QIF_BTIME && dqi->dqi_bgrace >= UINT_MAX) ||
-           (dqb->dqb_valid | QIF_ITIME && dqi->dqi_igrace >= UINT_MAX)) {
-               fprintf(stderr, "error: grace time is too large\n");
-               return CMD_HELP;
-       }
-
         mnt = argv[optind];
         rc = llapi_quotactl(mnt, &qctl);
         if (rc) {
@@ -7003,17 +7024,23 @@ static void print_quota(char *mnt, struct if_quotactl *qctl, int type,
                else
                        printf(" %7s %7s %7s %7s", "-", "-", "-", "-");
                printf("\n");
+       } else if (qctl->qc_cmd == LUSTRE_Q_GETINFO ||
+                  qctl->qc_cmd == Q_GETOINFO) {
+               char bgtimebuf[40];
+               char igtimebuf[40];
 
-        } else if (qctl->qc_cmd == LUSTRE_Q_GETINFO ||
-                   qctl->qc_cmd == Q_GETOINFO) {
-                char bgtimebuf[40];
-                char igtimebuf[40];
+               if (qctl->qc_dqinfo.dqi_bgrace == NOTIFY_GRACE_TIME)
+                       strncpy(bgtimebuf, NOTIFY_GRACE, 40);
+               else
+                       sec2str(qctl->qc_dqinfo.dqi_bgrace, bgtimebuf, rc);
+               if (qctl->qc_dqinfo.dqi_igrace == NOTIFY_GRACE_TIME)
+                       strncpy(igtimebuf, NOTIFY_GRACE, 40);
+               else
+                       sec2str(qctl->qc_dqinfo.dqi_igrace, igtimebuf, rc);
 
-                sec2str(qctl->qc_dqinfo.dqi_bgrace, bgtimebuf, rc);
-                sec2str(qctl->qc_dqinfo.dqi_igrace, igtimebuf, rc);
-                printf("Block grace time: %s; Inode grace time: %s\n",
-                       bgtimebuf, igtimebuf);
-        }
+               printf("Block grace time: %s; Inode grace time: %s\n",
+                      bgtimebuf, igtimebuf);
+       }
 }
 
 static int print_obd_quota(char *mnt, struct if_quotactl *qctl, int is_mdt,
@@ -9431,22 +9458,16 @@ static inline int verify_mirror_id_by_fd(int fd, __u16 mirror_id)
  * \retval     1  not the same file
  * \retval     <0 error code
  */
-static inline int check_same_file(const char *f1, const char *f2)
+static inline int check_same_file(int fd, const char *f2)
 {
        struct stat stbuf1;
        struct stat stbuf2;
 
-       if (stat(f1, &stbuf1) < 0) {
-               fprintf(stderr, "%s: cannot stat file '%s': %s\n",
-                       progname, f1, strerror(errno));
+       if (fstat(fd, &stbuf1) < 0)
                return -errno;
-       }
 
-       if (stat(f2, &stbuf2) < 0) {
-               fprintf(stderr, "%s: cannot stat file '%s': %s\n",
-                       progname, f2, strerror(errno));
-               return -errno;
-       }
+       if (stat(f2, &stbuf2) < 0)
+               return 1;
 
        if (stbuf1.st_rdev == stbuf2.st_rdev &&
            stbuf1.st_ino == stbuf2.st_ino)
@@ -9512,19 +9533,6 @@ static inline int lfs_mirror_read(int argc, char **argv)
 
        /* open mirror file */
        fname = argv[optind];
-
-       if (outfile) {
-               rc = check_same_file(fname, outfile);
-               if (rc == 0) {
-                       fprintf(stderr,
-                       "%s %s: output file cannot be the mirrored file\n",
-                               progname, argv[0]);
-                       return -EINVAL;
-               }
-               if (rc < 0)
-                       return rc;
-       }
-
        fd = open(fname, O_DIRECT | O_RDONLY);
        if (fd < 0) {
                fprintf(stderr, "%s %s: cannot open '%s': %s\n",
@@ -9541,7 +9549,7 @@ static inline int lfs_mirror_read(int argc, char **argv)
                goto close_fd;
        }
 
-       /* open output file */
+       /* open output file - O_EXCL ensures output is not the same as input */
        if (outfile) {
                outfd = open(outfile, O_EXCL | O_WRONLY | O_CREAT, 0644);
                if (outfd < 0) {
@@ -9681,19 +9689,6 @@ static inline int lfs_mirror_write(int argc, char **argv)
 
        /* open mirror file */
        fname = argv[optind];
-
-       if (inputfile) {
-               rc = check_same_file(fname, inputfile);
-               if (rc == 0) {
-                       fprintf(stderr,
-                       "%s %s: input file cannot be the mirrored file\n",
-                               progname, argv[0]);
-                       return -EINVAL;
-               }
-               if (rc < 0)
-                       return rc;
-       }
-
        fd = open(fname, O_DIRECT | O_WRONLY);
        if (fd < 0) {
                fprintf(stderr, "%s %s: cannot open '%s': %s\n",
@@ -9712,6 +9707,16 @@ static inline int lfs_mirror_write(int argc, char **argv)
 
        /* open input file */
        if (inputfile) {
+               rc = check_same_file(fd, inputfile);
+               if (rc == 0) {
+                       fprintf(stderr,
+                       "%s %s: input file cannot be the mirrored file\n",
+                               progname, argv[0]);
+                       goto close_fd;
+               }
+               if (rc < 0)
+                       goto close_fd;
+
                inputfd = open(inputfile, O_RDONLY, 0644);
                if (inputfd < 0) {
                        fprintf(stderr, "%s %s: cannot open file '%s': %s\n",