lustre/utils/lhsmtool_posix.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.htm
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * (C) Copyright 2012 Commissariat a l'energie atomique et aux energies
  24  *     alternatives
  25  *
  26  * Copyright (c) 2013, 2016, Intel Corporation.
  27  */
  28 /* HSM copytool program for POSIX filesystem-based HSM's.
  29  *
  30  * An HSM copytool daemon acts on action requests from Lustre to copy files
  31  * to and from an HSM archive system. This one in particular makes regular
  32  * POSIX filesystem calls to a given path, where an HSM is presumably mounted.
  33  *
  34  * This particular tool can also import an existing HSM archive.
  35  */
  36
  37 #ifndef _GNU_SOURCE
  38 #define _GNU_SOURCE
  39 #endif
  40 #include <ctype.h>
  41 #include <fcntl.h>
  42 #include <stdio.h>
  43 #include <stdint.h>
  44 #include <stdlib.h>
  45 #include <string.h>
  46 #include <dirent.h>
  47 #include <errno.h>
  48 #include <getopt.h>
  49 #include <pthread.h>
  50 #include <time.h>
  51 #include <unistd.h>
  52 #include <utime.h>
  53 #include <signal.h>
  54 #include <sys/time.h>
  55 #include <sys/xattr.h>
  56 #include <sys/syscall.h>
  57 #include <sys/types.h>
  58
  59 #include <libcfs/util/string.h>
  60 #include <linux/lustre/lustre_fid.h>
  61 #include <lnetconfig/cyaml.h>
  62 #include <lustre/lustreapi.h>
  63 #include "lstddef.h"
  64 #include "pid_file.h"
  65
  66 /* Progress reporting period */
  67 #define REPORT_INTERVAL_DEFAULT 30
  68 /* HSM hash subdir permissions */
  69 #define DIR_PERM S_IRWXU
  70 /* HSM hash file permissions */
  71 #define FILE_PERM (S_IRUSR | S_IWUSR)
  72
  73 #define ONE_MB 0x100000
  74
  75 #ifndef NSEC_PER_SEC
  76 # define NSEC_PER_SEC 1000000000UL
  77 #endif
  78
  79 enum ct_action {
  80         CA_IMPORT = 1,
  81         CA_REBIND,
  82         CA_MAXSEQ,
  83         CA_ARCHIVE_UPGRADE,
  84 };
  85
  86 enum ct_archive_format {
  87         /* v1 (original) using 6 directories (oid & 0xffff)/-/-/-/-/-/FID.
  88          * Places only one FID per directory. See ct_path_archive() below. */
  89         CT_ARCHIVE_FORMAT_V1 = 1,
  90         /* v2 using 1 directory (oid & 0xffff)/FID. */
  91         CT_ARCHIVE_FORMAT_V2 = 2,
  92 };
  93
  94 static const char *ct_archive_format_name[] = {
  95         [CT_ARCHIVE_FORMAT_V1] = "v1",
  96         [CT_ARCHIVE_FORMAT_V2] = "v2",
  97 };
  98
  99 static int
 100 ct_str_to_archive_format(const char *str, enum ct_archive_format *pctaf)
 101 {
 102         enum ct_archive_format ctaf;
 103
 104         for (ctaf = 0; ctaf < ARRAY_SIZE(ct_archive_format_name); ctaf++) {
 105                 if (ct_archive_format_name[ctaf] != NULL &&
 106                     strcmp(ct_archive_format_name[ctaf], str) == 0) {
 107                         *pctaf = ctaf;
 108                         return 0;
 109                 }
 110         }
 111
 112         return -EINVAL;
 113 }
 114
 115 static const char *ct_archive_format_to_str(enum ct_archive_format ctaf)
 116 {
 117         if (0 <= ctaf &&
 118             ctaf < ARRAY_SIZE(ct_archive_format_name) &&
 119             ct_archive_format_name[ctaf] != NULL)
 120                 return ct_archive_format_name[ctaf];
 121
 122         return "null";
 123 }
 124
 125 struct options {
 126         int                      o_copy_attrs;
 127         int                      o_daemonize;
 128         int                      o_dry_run;
 129         int                      o_abort_on_error;
 130         int                      o_shadow_tree;
 131         int                      o_verbose;
 132         int                      o_copy_xattrs;
 133         const char              *o_config_path;
 134         enum ct_archive_format   o_archive_format;
 135         int                      o_archive_id_used;
 136         int                      o_archive_id_cnt;
 137         int                     *o_archive_id;
 138         int                      o_report_int;
 139         unsigned long long       o_bandwidth;
 140         size_t                   o_chunk_size;
 141         enum ct_action           o_action;
 142         char                    *o_event_fifo;
 143         char                    *o_mnt;
 144         int                      o_mnt_fd;
 145         char                    *o_hsm_root;
 146         char                    *o_src; /* for import, or rebind */
 147         char                    *o_dst; /* for import, or rebind */
 148         char                    *o_pid_file;
 149 };
 150
 151 /* everything else is zeroed */
 152 struct options opt = {
 153         .o_copy_attrs = 1,
 154         .o_shadow_tree = 1,
 155         .o_verbose = LLAPI_MSG_INFO,
 156         .o_copy_xattrs = 1,
 157         .o_archive_format = CT_ARCHIVE_FORMAT_V1,
 158         .o_report_int = REPORT_INTERVAL_DEFAULT,
 159         .o_chunk_size = ONE_MB,
 160 };
 161
 162 /* hsm_copytool_private will hold an open FD on the lustre mount point
 163  * for us. Additionally open one on the archive FS root to make sure
 164  * it doesn't drop out from under us (and remind the admin to shutdown
 165  * the copytool before unmounting). */
 166
 167 static int arc_fd = -1;
 168
 169 static int err_major;
 170 static int err_minor;
 171
 172 static char cmd_name[PATH_MAX];
 173 static char fs_name[MAX_OBD_NAME + 1];
 174 static int pid_file_fd = -1;
 175
 176 static struct hsm_copytool_private *ctdata;
 177
 178 static inline double ct_now(void)
 179 {
 180         struct timeval tv;
 181
 182         gettimeofday(&tv, NULL);
 183
 184         return tv.tv_sec + 0.000001 * tv.tv_usec;
 185 }
 186
 187 #define CT_ERROR(_rc, _format, ...)                                     \
 188         llapi_error(LLAPI_MSG_ERROR, _rc,                               \
 189                     "%f %s[%ld]: "_format,                              \
 190                     ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__)
 191
 192 #define CT_DEBUG(_format, ...)                                          \
 193         llapi_error(LLAPI_MSG_DEBUG | LLAPI_MSG_NO_ERRNO, 0,            \
 194                     "%f %s[%ld]: "_format,                              \
 195                     ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__)
 196
 197 #define CT_WARN(_format, ...) \
 198         llapi_error(LLAPI_MSG_WARN | LLAPI_MSG_NO_ERRNO, 0,             \
 199                     "%f %s[%ld]: "_format,                              \
 200                     ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__)
 201
 202 #define CT_TRACE(_format, ...)                                          \
 203         llapi_error(LLAPI_MSG_INFO | LLAPI_MSG_NO_ERRNO, 0,             \
 204                     "%f %s[%ld]: "_format,                              \
 205                     ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__)
 206
 207 static void usage(const char *name, int rc)
 208 {
 209         fprintf(stdout,
 210         " Usage: %s [options]... <mode> <lustre_mount_point>\n"
 211         "The Lustre HSM Posix copy tool can be used as a daemon or "
 212         "as a command line tool\n"
 213         "The Lustre HSM daemon acts on action requests from Lustre\n"
 214         "to copy files to and from an HSM archive system.\n"
 215         "This POSIX-flavored daemon makes regular POSIX filesystem calls\n"
 216         "to an HSM mounted at a given hsm_root.\n"
 217         "   --daemon            Daemon mode, run in background\n"
 218         " Options:\n"
 219         "   --no-attr           Don't copy file attributes\n"
 220         "   --no-shadow         Don't create shadow namespace in archive\n"
 221         "   --no-xattr          Don't copy file extended attributes\n"
 222         "The Lustre HSM tool performs administrator-type actions\n"
 223         "on a Lustre HSM archive.\n"
 224         "This POSIX-flavored tool can link an existing HSM namespace\n"
 225         "into a Lustre filesystem.\n"
 226         " Usage:\n"
 227         "   %s [options] --import <src> <dst> <lustre_mount_point>\n"
 228         "      import an archived subtree from\n"
 229         "       <src> (FID or relative path to hsm_root) into the Lustre\n"
 230         "             filesystem at\n"
 231         "       <dst> (absolute path)\n"
 232         "   %s [options] --rebind <old_FID> <new_FID> <lustre_mount_point>\n"
 233         "      rebind an entry in the HSM to a new FID\n"
 234         "       <old_FID> old FID the HSM entry is bound to\n"
 235         "       <new_FID> new FID to bind the HSM entry to\n"
 236         "   %s [options] --rebind <list_file> <lustre_mount_point>\n"
 237         "      perform the rebind operation for all FID in the list file\n"
 238         "       each line of <list_file> consists of <old_FID> <new_FID>\n"
 239         "   %s [options] --max-sequence <fsname>\n"
 240         "       return the max fid sequence of archived files\n"
 241         "   %s [options] --archive-upgrade=VER\n"
 242         "      Upgrade or downgrade the archive to version VER\n"
 243         "Options:\n"
 244         "   --abort-on-error          Abort operation on major error\n"
 245         "   -A, --archive <#>         Archive number (repeatable)\n"
 246         "   -C, --config=PATH         Read config from PATH\n"
 247         "   -F, --archive-format=VER  Use archive format VER\n"
 248         "   -b, --bandwidth <bw>      Limit I/O bandwidth (unit can be used\n,"
 249         "                             default is MB)\n"
 250         "   --dry-run                 Don't run, just show what would be done\n"
 251         "   -c, --chunk-size <sz>     I/O size used during data copy\n"
 252         "                             (unit can be used, default is MB)\n"
 253         "   -f, --event-fifo <path>   Write events stream to fifo\n"
 254         "   -P, --pid-file=PATH       Lock and write PID to PATH\n"
 255         "   -p, --hsm-root <path>     Target HSM mount point\n"
 256         "   -q, --quiet               Produce less verbose output\n"
 257         "   -u, --update-interval <s> Interval between progress reports sent\n"
 258         "                             to Coordinator\n"
 259         "   -v, --verbose             Produce more verbose output\n",
 260         cmd_name, cmd_name, cmd_name, cmd_name, cmd_name, cmd_name);
 261
 262         exit(rc);
 263 }
 264
 265 static int ct_parseopts(int argc, char * const *argv)
 266 {
 267         struct option long_opts[] = {
 268         { .val = 1,     .name = "abort-on-error",
 269           .flag = &opt.o_abort_on_error,        .has_arg = no_argument },
 270         { .val = 1,     .name = "abort_on_error",
 271           .flag = &opt.o_abort_on_error,        .has_arg = no_argument },
 272         { .val = 'A',   .name = "archive",      .has_arg = required_argument },
 273         { .val = 'b',   .name = "bandwidth",    .has_arg = required_argument },
 274         { .val = 'C',   .name = "config",       .has_arg = required_argument },
 275         { .val = 'c',   .name = "chunk-size",   .has_arg = required_argument },
 276         { .val = 'c',   .name = "chunk_size",   .has_arg = required_argument },
 277         { .val = 1,     .name = "daemon",       .has_arg = no_argument,
 278           .flag = &opt.o_daemonize },
 279         { .val = 'F',   .name = "archive-format", .has_arg = required_argument },
 280         { .val = 'U',   .name = "archive-upgrade", .has_arg = required_argument },
 281         { .val = 'f',   .name = "event-fifo",   .has_arg = required_argument },
 282         { .val = 'f',   .name = "event_fifo",   .has_arg = required_argument },
 283         { .val = 1,     .name = "dry-run",      .has_arg = no_argument,
 284           .flag = &opt.o_dry_run },
 285         { .val = 'h',   .name = "help",         .has_arg = no_argument },
 286         { .val = 'i',   .name = "import",       .has_arg = no_argument },
 287         { .val = 'M',   .name = "max-sequence", .has_arg = no_argument },
 288         { .val = 'M',   .name = "max_sequence", .has_arg = no_argument },
 289         { .val = 0,     .name = "no-attr",      .has_arg = no_argument,
 290           .flag = &opt.o_copy_attrs },
 291         { .val = 0,     .name = "no_attr",      .has_arg = no_argument,
 292           .flag = &opt.o_copy_attrs },
 293         { .val = 0,     .name = "no-shadow",    .has_arg = no_argument,
 294           .flag = &opt.o_shadow_tree },
 295         { .val = 0,     .name = "no_shadow",    .has_arg = no_argument,
 296           .flag = &opt.o_shadow_tree },
 297         { .val = 0,     .name = "no-xattr",     .has_arg = no_argument,
 298           .flag = &opt.o_copy_xattrs },
 299         { .val = 0,     .name = "no_xattr",     .has_arg = no_argument,
 300           .flag = &opt.o_copy_xattrs },
 301         { .val = 'P',   .name = "pid-file",     .has_arg = required_argument },
 302         { .val = 'p',   .name = "hsm-root",     .has_arg = required_argument },
 303         { .val = 'p',   .name = "hsm_root",     .has_arg = required_argument },
 304         { .val = 'q',   .name = "quiet",        .has_arg = no_argument },
 305         { .val = 'r',   .name = "rebind",       .has_arg = no_argument },
 306         { .val = 'u',   .name = "update-interval",
 307                                                 .has_arg = required_argument },
 308         { .val = 'u',   .name = "update_interval",
 309                                                 .has_arg = required_argument },
 310         { .val = 'v',   .name = "verbose",      .has_arg = no_argument },
 311         { .name = NULL } };
 312
 313         unsigned long long value;
 314         unsigned long long unit;
 315         bool all_id = false;
 316         int c, rc;
 317         int i;
 318
 319         optind = 0;
 320
 321         opt.o_archive_id_cnt = LL_HSM_ORIGIN_MAX_ARCHIVE;
 322         opt.o_archive_id = malloc(opt.o_archive_id_cnt *
 323                                   sizeof(*opt.o_archive_id));
 324         if (opt.o_archive_id == NULL)
 325                 return -ENOMEM;
 326 repeat:
 327         while ((c = getopt_long(argc, argv, "A:b:C:c:F:f:hiMp:P:qrU:u:v",
 328                                 long_opts, NULL)) != -1) {
 329                 switch (c) {
 330                 case 'A': {
 331                         char *end = NULL;
 332                         int val = strtol(optarg, &end, 10);
 333
 334                         if (*end != '\0') {
 335                                 rc = -EINVAL;
 336                                 CT_ERROR(rc, "invalid archive-id: '%s'",
 337                                          optarg);
 338                                 return rc;
 339                         }
 340                         /* if archiveID is zero, any archiveID is accepted */
 341                         if (all_id == true)
 342                                 goto repeat;
 343
 344                         if (val == 0) {
 345                                 free(opt.o_archive_id);
 346                                 opt.o_archive_id = NULL;
 347                                 opt.o_archive_id_cnt = 0;
 348                                 opt.o_archive_id_used = 0;
 349                                 all_id = true;
 350                                 CT_WARN("archive-id = 0 is found, any backend will be served\n");
 351                                 goto repeat;
 352                         }
 353
 354                         /* skip the duplicated id */
 355                         for (i = 0; i < opt.o_archive_id_used; i++) {
 356                                 if (opt.o_archive_id[i] == val)
 357                                         goto repeat;
 358                         }
 359                         /* extend the space */
 360                         if (opt.o_archive_id_used >= opt.o_archive_id_cnt) {
 361                                 int *tmp;
 362
 363                                 opt.o_archive_id_cnt *= 2;
 364                                 tmp = realloc(opt.o_archive_id,
 365                                               sizeof(*opt.o_archive_id) *
 366                                               opt.o_archive_id_cnt);
 367                                 if (tmp == NULL)
 368                                         return -ENOMEM;
 369
 370                                 opt.o_archive_id = tmp;
 371                         }
 372
 373                         opt.o_archive_id[opt.o_archive_id_used++] = val;
 374                         break;
 375                 }
 376                 case 'b': /* -b and -c have both a number with unit as arg */
 377                 case 'c':
 378                         unit = ONE_MB;
 379                         if (llapi_parse_size(optarg, &value, &unit, 0) < 0) {
 380                                 rc = -EINVAL;
 381                                 CT_ERROR(rc, "bad value for -%c '%s'", c,
 382                                          optarg);
 383                                 return rc;
 384                         }
 385                         if (c == 'c')
 386                                 opt.o_chunk_size = value;
 387                         else
 388                                 opt.o_bandwidth = value;
 389                         break;
 390                 case 'C':
 391                         opt.o_config_path = optarg;
 392                         break;
 393                 case 'F':
 394                         rc = ct_str_to_archive_format(optarg, &opt.o_archive_format);
 395                         if (rc < 0) {
 396                                 CT_ERROR(rc, "invalid archive format '%s'", optarg);
 397                                 return -EINVAL;
 398                         }
 399                         break;
 400                 case 'U':
 401                         rc = ct_str_to_archive_format(optarg, &opt.o_archive_format);
 402                         if (rc < 0) {
 403                                 CT_ERROR(rc, "invalid archive format '%s'", optarg);
 404                                 return -EINVAL;
 405                         }
 406                         opt.o_action = CA_ARCHIVE_UPGRADE;
 407                         break;
 408
 409                 case 'f':
 410                         opt.o_event_fifo = optarg;
 411                         break;
 412                 case 'h':
 413                         usage(argv[0], 0);
 414                 case 'i':
 415                         opt.o_action = CA_IMPORT;
 416                         break;
 417                 case 'M':
 418                         opt.o_action = CA_MAXSEQ;
 419                         break;
 420                 case 'P':
 421                         opt.o_pid_file = optarg;
 422                         break;
 423                 case 'p':
 424                         opt.o_hsm_root = optarg;
 425                         break;
 426                 case 'q':
 427                         opt.o_verbose--;
 428                         break;
 429                 case 'r':
 430                         opt.o_action = CA_REBIND;
 431                         break;
 432                 case 'u':
 433                         opt.o_report_int = atoi(optarg);
 434                         if (opt.o_report_int < 0) {
 435                                 rc = -EINVAL;
 436                                 CT_ERROR(rc, "bad value for -%c '%s'", c,
 437                                          optarg);
 438                                 return rc;
 439                         }
 440                         break;
 441                 case 'v':
 442                         opt.o_verbose++;
 443                         break;
 444                 case 0:
 445                         break;
 446                 default:
 447                         return -EINVAL;
 448                 }
 449         }
 450
 451         switch (opt.o_action) {
 452         case CA_IMPORT:
 453                 /* src dst mount_point */
 454                 if (argc != optind + 3) {
 455                         rc = -EINVAL;
 456                         CT_ERROR(rc, "--import requires 2 arguments");
 457                         return rc;
 458                 }
 459                 opt.o_src = argv[optind++];
 460                 opt.o_dst = argv[optind++];
 461                 break;
 462         case CA_REBIND:
 463                 /* FID1 FID2 mount_point or FILE mount_point */
 464                 if (argc == optind + 2) {
 465                         opt.o_src = argv[optind++];
 466                         opt.o_dst = NULL;
 467                 } else if (argc == optind + 3) {
 468                         opt.o_src = argv[optind++];
 469                         opt.o_dst = argv[optind++];
 470                 } else {
 471                         rc = -EINVAL;
 472                         CT_ERROR(rc, "--rebind requires 1 or 2 arguments");
 473                         return rc;
 474                 }
 475                 break;
 476         case CA_MAXSEQ:
 477         default:
 478                 /* just mount point */
 479                 break;
 480         }
 481
 482         if (opt.o_action == CA_ARCHIVE_UPGRADE)
 483                 return 0;
 484
 485         if (argc != optind + 1) {
 486                 rc = -EINVAL;
 487                 CT_ERROR(rc, "no mount point specified");
 488                 return rc;
 489         }
 490
 491         opt.o_mnt = argv[optind];
 492         opt.o_mnt_fd = -1;
 493
 494         CT_TRACE("action=%d src=%s dst=%s mount_point=%s",
 495                  opt.o_action, opt.o_src, opt.o_dst, opt.o_mnt);
 496
 497         if (opt.o_action == CA_IMPORT) {
 498                 if (opt.o_src && opt.o_src[0] == '/') {
 499                         rc = -EINVAL;
 500                         CT_ERROR(rc,
 501                                  "source path must be relative to HSM root");
 502                         return rc;
 503                 }
 504
 505                 if (opt.o_dst && opt.o_dst[0] != '/') {
 506                         rc = -EINVAL;
 507                         CT_ERROR(rc, "destination path must be absolute");
 508                         return rc;
 509                 }
 510         }
 511
 512         return 0;
 513 }
 514
 515 static int ct_mkdirat_p(int fd, char *path, mode_t mode)
 516 {
 517         char *split;
 518         int rc;
 519
 520         if (strlen(path) == 0 || strcmp(path, ".") == 0)
 521                 return 0;
 522
 523         rc = mkdirat(fd, path, mode);
 524         if (rc == 0 || errno == EEXIST)
 525                 return 0;
 526
 527         if (errno != ENOENT)
 528                 return -errno;
 529
 530         split = strrchr(path, '/');
 531         if (split == NULL)
 532                 return -ENOENT;
 533
 534         *split = '\0';
 535         rc = ct_mkdirat_p(fd, path, mode);
 536         *split = '/';
 537         if (rc < 0)
 538                 return rc;
 539
 540         rc = mkdirat(fd, path, mode);
 541         if (rc == 0 || errno == EEXIST)
 542                 return 0;
 543
 544         return -errno;
 545 }
 546
 547 /* XXX Despite the name, this is 'mkdir -p $(dirname path)' */
 548 static int ct_mkdir_p(const char *path)
 549 {
 550         char *path2;
 551         char *split;
 552         int rc;
 553
 554         path2 = strdup(path);
 555         if (path2 == NULL)
 556                 return -ENOMEM;
 557
 558         split = strrchr(path2, '/');
 559         if (split == NULL) {
 560                 rc = 0;
 561                 goto out;
 562         }
 563
 564         *split = '\0';
 565
 566         rc = ct_mkdirat_p(AT_FDCWD, path2, DIR_PERM);
 567 out:
 568         free(path2);
 569
 570         return rc;
 571 }
 572
 573 static int ct_save_stripe(int src_fd, const char *src, const char *dst)
 574 {
 575         char                     lov_file[PATH_MAX + 8];
 576         char                     lov_buf[XATTR_SIZE_MAX];
 577         struct lov_user_md      *lum;
 578         int                      rc;
 579         int                      fd;
 580         ssize_t                  xattr_size;
 581
 582         snprintf(lov_file, sizeof(lov_file), "%s.lov", dst);
 583         CT_TRACE("saving stripe info of '%s' in %s", src, lov_file);
 584
 585         xattr_size = fgetxattr(src_fd, XATTR_LUSTRE_LOV, lov_buf,
 586                                sizeof(lov_buf));
 587         if (xattr_size < 0) {
 588                 rc = -errno;
 589                 CT_ERROR(rc, "cannot get stripe info on '%s'", src);
 590                 return rc;
 591         }
 592
 593         lum = (struct lov_user_md *)lov_buf;
 594
 595         if (lum->lmm_magic == LOV_USER_MAGIC_V1 ||
 596             lum->lmm_magic == LOV_USER_MAGIC_V3) {
 597                 /* Set stripe_offset to -1 so that it is not interpreted as a
 598                  * hint on restore. */
 599                 lum->lmm_stripe_offset = -1;
 600         }
 601
 602         fd = open(lov_file, O_TRUNC | O_CREAT | O_WRONLY, FILE_PERM);
 603         if (fd < 0) {
 604                 rc = -errno;
 605                 CT_ERROR(rc, "cannot open '%s'", lov_file);
 606                 goto err_cleanup;
 607         }
 608
 609         rc = write(fd, lum, xattr_size);
 610         if (rc < 0) {
 611                 rc = -errno;
 612                 CT_ERROR(rc, "cannot write %zd bytes to '%s'",
 613                          xattr_size, lov_file);
 614                 close(fd);
 615                 goto err_cleanup;
 616         }
 617
 618         rc = close(fd);
 619         if (rc < 0) {
 620                 rc = -errno;
 621                 CT_ERROR(rc, "cannot close '%s'", lov_file);
 622                 goto err_cleanup;
 623         }
 624
 625         return 0;
 626
 627 err_cleanup:
 628         unlink(lov_file);
 629
 630         return rc;
 631 }
 632
 633 static int ct_load_stripe(const char *src, void *lovea, size_t *lovea_size)
 634 {
 635         char     lov_file[PATH_MAX + 4];
 636         int      rc;
 637         int      fd;
 638
 639         snprintf(lov_file, sizeof(lov_file), "%s.lov", src);
 640         CT_TRACE("reading stripe rules from '%s' for '%s'", lov_file, src);
 641
 642         fd = open(lov_file, O_RDONLY);
 643         if (fd < 0) {
 644                 CT_ERROR(errno, "cannot open '%s'", lov_file);
 645                 return -ENODATA;
 646         }
 647
 648         rc = read(fd, lovea, *lovea_size);
 649         if (rc < 0) {
 650                 CT_ERROR(errno, "cannot read %zu bytes from '%s'",
 651                          *lovea_size, lov_file);
 652                 close(fd);
 653                 return -ENODATA;
 654         }
 655
 656         *lovea_size = rc;
 657         close(fd);
 658
 659         return 0;
 660 }
 661
 662 static int ct_restore_stripe(const char *src, const char *dst, int dst_fd,
 663                              const void *lovea, size_t lovea_size)
 664 {
 665         int     rc;
 666
 667         rc = fsetxattr(dst_fd, XATTR_LUSTRE_LOV, lovea, lovea_size,
 668                        XATTR_CREATE);
 669         if (rc < 0) {
 670                 rc = -errno;
 671                 CT_ERROR(rc, "cannot set lov EA on '%s'", dst);
 672         }
 673
 674         return rc;
 675 }
 676
 677 static int ct_copy_data(struct hsm_copyaction_private *hcp, const char *src,
 678                         const char *dst, int src_fd, int dst_fd,
 679                         const struct hsm_action_item *hai, long hal_flags)
 680 {
 681         struct hsm_extent        he;
 682         __u64                    offset = hai->hai_extent.offset;
 683         struct stat              src_st;
 684         struct stat              dst_st;
 685         char                    *buf = NULL;
 686         __u64                    write_total = 0;
 687         __u64                    length = hai->hai_extent.length;
 688         time_t                   last_report_time;
 689         int                      rc = 0;
 690         double                   start_ct_now = ct_now();
 691         /* Bandwidth Control */
 692         time_t                  start_time;
 693         time_t                  now;
 694         time_t                  last_bw_print;
 695
 696         if (fstat(src_fd, &src_st) < 0) {
 697                 rc = -errno;
 698                 CT_ERROR(rc, "cannot stat '%s'", src);
 699                 return rc;
 700         }
 701
 702         if (!S_ISREG(src_st.st_mode)) {
 703                 rc = -EINVAL;
 704                 CT_ERROR(rc, "'%s' is not a regular file", src);
 705                 return rc;
 706         }
 707
 708         if (hai->hai_extent.offset > (__u64)src_st.st_size) {
 709                 rc = -EINVAL;
 710                 CT_ERROR(rc, "Trying to start reading past end (%ju > "
 711                          "%jd) of '%s' source file",
 712                          (uintmax_t)hai->hai_extent.offset,
 713                          (intmax_t)src_st.st_size, src);
 714                 return rc;
 715         }
 716
 717         if (fstat(dst_fd, &dst_st) < 0) {
 718                 rc = -errno;
 719                 CT_ERROR(rc, "cannot stat '%s'", dst);
 720                 return rc;
 721         }
 722
 723         if (!S_ISREG(dst_st.st_mode)) {
 724                 rc = -EINVAL;
 725                 CT_ERROR(rc, "'%s' is not a regular file", dst);
 726                 return rc;
 727         }
 728
 729         /* Don't read beyond a given extent */
 730         if (length > src_st.st_size - hai->hai_extent.offset)
 731                 length = src_st.st_size - hai->hai_extent.offset;
 732
 733         start_time = last_bw_print = last_report_time = time(NULL);
 734
 735         he.offset = offset;
 736         he.length = 0;
 737         rc = llapi_hsm_action_progress(hcp, &he, length, 0);
 738         if (rc < 0) {
 739                 /* Action has been canceled or something wrong
 740                  * is happening. Stop copying data. */
 741                 CT_ERROR(rc, "progress ioctl for copy '%s'->'%s' failed",
 742                          src, dst);
 743                 goto out;
 744         }
 745
 746         errno = 0;
 747
 748         buf = malloc(opt.o_chunk_size);
 749         if (buf == NULL) {
 750                 rc = -ENOMEM;
 751                 goto out;
 752         }
 753
 754         CT_TRACE("start copy of %ju bytes from '%s' to '%s'",
 755                  (uintmax_t)length, src, dst);
 756
 757         while (write_total < length) {
 758                 ssize_t rsize;
 759                 ssize_t wsize;
 760                 int     chunk = (length - write_total > opt.o_chunk_size) ?
 761                                  opt.o_chunk_size : length - write_total;
 762
 763                 rsize = pread(src_fd, buf, chunk, offset);
 764                 if (rsize == 0)
 765                         /* EOF */
 766                         break;
 767
 768                 if (rsize < 0) {
 769                         rc = -errno;
 770                         CT_ERROR(rc, "cannot read from '%s'", src);
 771                         break;
 772                 }
 773
 774                 wsize = pwrite(dst_fd, buf, rsize, offset);
 775                 if (wsize < 0) {
 776                         rc = -errno;
 777                         CT_ERROR(rc, "cannot write to '%s'", dst);
 778                         break;
 779                 }
 780
 781                 write_total += wsize;
 782                 offset += wsize;
 783
 784                 now = time(NULL);
 785                 /* sleep if needed, to honor bandwidth limits */
 786                 if (opt.o_bandwidth != 0) {
 787                         unsigned long long write_theory;
 788
 789                         write_theory = (now - start_time) * opt.o_bandwidth;
 790
 791                         if (write_theory < write_total) {
 792                                 unsigned long long      excess;
 793                                 struct timespec         delay;
 794
 795                                 excess = write_total - write_theory;
 796
 797                                 delay.tv_sec = excess / opt.o_bandwidth;
 798                                 delay.tv_nsec = (excess % opt.o_bandwidth) *
 799                                         NSEC_PER_SEC / opt.o_bandwidth;
 800
 801                                 if (now >= last_bw_print + opt.o_report_int) {
 802                                         CT_TRACE("bandwith control: %lluB/s "
 803                                                  "excess=%llu sleep for "
 804                                                  "%lld.%09lds",
 805                                                  (unsigned long long)opt.o_bandwidth,
 806                                                  (unsigned long long)excess,
 807                                                  (long long)delay.tv_sec,
 808                                                  delay.tv_nsec);
 809                                         last_bw_print = now;
 810                                 }
 811
 812                                 do {
 813                                         rc = nanosleep(&delay, &delay);
 814                                 } while (rc < 0 && errno == EINTR);
 815                                 if (rc < 0) {
 816                                         CT_ERROR(errno, "delay for bandwidth "
 817                                                  "control failed to sleep: "
 818                                                  "residual=%lld.%09lds",
 819                                                  (long long)delay.tv_sec,
 820                                                  delay.tv_nsec);
 821                                         rc = 0;
 822                                 }
 823                         }
 824                 }
 825
 826                 now = time(NULL);
 827                 if (now >= last_report_time + opt.o_report_int) {
 828                         last_report_time = now;
 829                         CT_TRACE("%%%ju ", (uintmax_t)(100 * write_total / length));
 830                         /* only give the length of the write since the last
 831                          * progress report */
 832                         he.length = offset - he.offset;
 833                         rc = llapi_hsm_action_progress(hcp, &he, length, 0);
 834                         if (rc < 0) {
 835                                 /* Action has been canceled or something wrong
 836                                  * is happening. Stop copying data. */
 837                                 CT_ERROR(rc, "progress ioctl for copy"
 838                                          " '%s'->'%s' failed", src, dst);
 839                                 goto out;
 840                         }
 841                         he.offset = offset;
 842                 }
 843                 rc = 0;
 844         }
 845
 846 out:
 847         /*
 848          * truncate restored file
 849          * size is taken from the archive this is done to support
 850          * restore after a force release which leaves the file with the
 851          * wrong size (can big bigger than the new size)
 852          */
 853         if ((hai->hai_action == HSMA_RESTORE) &&
 854             (src_st.st_size < dst_st.st_size)) {
 855                 /*
 856                  * make sure the file is on disk before reporting success.
 857                  */
 858                 rc = ftruncate(dst_fd, src_st.st_size);
 859                 if (rc < 0) {
 860                         rc = -errno;
 861                         CT_ERROR(rc, "cannot truncate '%s' to size %jd",
 862                                  dst, (intmax_t)src_st.st_size);
 863                         err_major++;
 864                 }
 865         }
 866
 867         if (buf != NULL)
 868                 free(buf);
 869
 870         CT_TRACE("copied %ju bytes in %f seconds",
 871                  (uintmax_t)length, ct_now() - start_ct_now);
 872
 873         return rc;
 874 }
 875
 876 /* Copy file attributes from file src to file dest */
 877 static int ct_copy_attr(const char *src, const char *dst, int src_fd,
 878                         int dst_fd)
 879 {
 880         struct stat     st;
 881         struct timeval  times[2];
 882         int             rc;
 883
 884         if (fstat(src_fd, &st) < 0) {
 885                 rc = -errno;
 886                 CT_ERROR(rc, "cannot stat '%s'", src);
 887                 return rc;
 888         }
 889
 890         times[0].tv_sec = st.st_atime;
 891         times[0].tv_usec = 0;
 892         times[1].tv_sec = st.st_mtime;
 893         times[1].tv_usec = 0;
 894         if (fchmod(dst_fd, st.st_mode) < 0 ||
 895             fchown(dst_fd, st.st_uid, st.st_gid) < 0 ||
 896             futimes(dst_fd, times) < 0) {
 897                 rc = -errno;
 898                 CT_ERROR(rc, "cannot set attributes of '%s'", src);
 899                 return rc;
 900         }
 901
 902         return 0;
 903 }
 904
 905 static int ct_copy_xattr(const char *src, const char *dst, int src_fd,
 906                          int dst_fd, bool is_restore)
 907 {
 908         char     list[XATTR_LIST_MAX];
 909         char     value[XATTR_SIZE_MAX];
 910         char    *name;
 911         ssize_t  list_len;
 912         int      rc;
 913
 914         list_len = flistxattr(src_fd, list, sizeof(list));
 915         if (list_len < 0)
 916                 return -errno;
 917
 918         name = list;
 919         while (name < list + list_len) {
 920                 rc = fgetxattr(src_fd, name, value, sizeof(value));
 921                 if (rc < 0)
 922                         return -errno;
 923
 924                 /* when we restore, we do not restore lustre xattr */
 925                 if (!is_restore ||
 926                     (strncmp(XATTR_TRUSTED_PREFIX, name,
 927                              sizeof(XATTR_TRUSTED_PREFIX) - 1) != 0)) {
 928                         rc = fsetxattr(dst_fd, name, value, rc, 0);
 929                         CT_TRACE("fsetxattr of '%s' on '%s' rc=%d (%s)",
 930                                  name, dst, rc, strerror(errno));
 931                         /* lustre.* attrs aren't supported on other FS's */
 932                         if (rc < 0 && errno != EOPNOTSUPP) {
 933                                 rc = -errno;
 934                                 CT_ERROR(rc, "cannot set extended attribute"
 935                                          " '%s' of '%s'",
 936                                          name, dst);
 937                                 return rc;
 938                         }
 939                 }
 940                 name += strlen(name) + 1;
 941         }
 942
 943         return 0;
 944 }
 945
 946 static int ct_path_lustre(char *buf, int sz, const char *mnt,
 947                           const struct lu_fid *fid)
 948 {
 949         return scnprintf(buf, sz, "%s/%s/fid/"DFID_NOBRACE, mnt,
 950                          dot_lustre_name, PFID(fid));
 951 }
 952
 953 static int ct_path_archive_v(char *buf, size_t buf_size,
 954                              enum ct_archive_format ctaf,
 955                              const char *archive_path,
 956                              const struct lu_fid *fid,
 957                              const char *suffix)
 958 {
 959         switch (ctaf) {
 960         case CT_ARCHIVE_FORMAT_V1:
 961                 return scnprintf(buf, buf_size,
 962                                  "%s/%04x/%04x/%04x/%04x/%04x/%04x/"DFID_NOBRACE"%s",
 963                                  archive_path,
 964                                  (fid)->f_oid       & 0xFFFF,
 965                                  (fid)->f_oid >> 16 & 0xFFFF,
 966                                  (unsigned int)((fid)->f_seq       & 0xFFFF),
 967                                  (unsigned int)((fid)->f_seq >> 16 & 0xFFFF),
 968                                  (unsigned int)((fid)->f_seq >> 32 & 0xFFFF),
 969                                  (unsigned int)((fid)->f_seq >> 48 & 0xFFFF),
 970                                  PFID(fid),
 971                                  suffix);
 972         case CT_ARCHIVE_FORMAT_V2:
 973                 return scnprintf(buf, buf_size, "%s/%04x/"DFID_NOBRACE"%s",
 974                                  archive_path,
 975                                  (unsigned int)((fid)->f_oid ^ (fid)->f_seq) & 0xFFFF,
 976                                  PFID(fid),
 977                                  suffix);
 978         default:
 979                 return -EINVAL;
 980         }
 981 }
 982
 983 static int ct_path_archive(char *buf, size_t buf_size,
 984                            const char *archive_path, const struct lu_fid *fid)
 985 {
 986         return ct_path_archive_v(buf, buf_size, opt.o_archive_format,
 987                                  archive_path, fid, "");
 988 }
 989
 990 static bool ct_is_retryable(int err)
 991 {
 992         return err == -ETIMEDOUT;
 993 }
 994
 995 static int ct_begin_restore(struct hsm_copyaction_private **phcp,
 996                             const struct hsm_action_item *hai,
 997                             int mdt_index, int open_flags)
 998 {
 999         char     src[PATH_MAX];
1000         int      rc;
1001
1002         rc = llapi_hsm_action_begin(phcp, ctdata, hai, mdt_index, open_flags,
1003                                     false);
1004         if (rc < 0) {
1005                 ct_path_lustre(src, sizeof(src), opt.o_mnt, &hai->hai_fid);
1006                 CT_ERROR(rc, "llapi_hsm_action_begin() on '%s' failed", src);
1007         }
1008
1009         return rc;
1010 }
1011
1012 static int ct_begin(struct hsm_copyaction_private **phcp,
1013                     const struct hsm_action_item *hai)
1014 {
1015         /* Restore takes specific parameters. Call the same function w/ default
1016          * values for all other operations. */
1017         return ct_begin_restore(phcp, hai, -1, 0);
1018 }
1019
1020 static int ct_fini(struct hsm_copyaction_private **phcp,
1021                    const struct hsm_action_item *hai, int hp_flags, int ct_rc)
1022 {
1023         struct hsm_copyaction_private   *hcp;
1024         char                             lstr[PATH_MAX];
1025         int                              rc;
1026
1027         CT_TRACE("Action completed, notifying coordinator "
1028                  "cookie=%#jx, FID="DFID", hp_flags=%d err=%d",
1029                  (uintmax_t)hai->hai_cookie, PFID(&hai->hai_fid),
1030                  hp_flags, -ct_rc);
1031
1032         ct_path_lustre(lstr, sizeof(lstr), opt.o_mnt, &hai->hai_fid);
1033
1034         if (phcp == NULL || *phcp == NULL) {
1035                 rc = llapi_hsm_action_begin(&hcp, ctdata, hai, -1, 0, true);
1036                 if (rc < 0) {
1037                         CT_ERROR(rc, "llapi_hsm_action_begin() on '%s' failed",
1038                                  lstr);
1039                         return rc;
1040                 }
1041                 phcp = &hcp;
1042         }
1043
1044         rc = llapi_hsm_action_end(phcp, &hai->hai_extent, hp_flags, abs(ct_rc));
1045         if (rc == -ECANCELED)
1046                 CT_ERROR(rc, "completed action on '%s' has been canceled: "
1047                          "cookie=%#jx, FID="DFID, lstr,
1048                          (uintmax_t)hai->hai_cookie, PFID(&hai->hai_fid));
1049         else if (rc < 0)
1050                 CT_ERROR(rc, "llapi_hsm_action_end() on '%s' failed", lstr);
1051         else
1052                 CT_TRACE("llapi_hsm_action_end() on '%s' ok (rc=%d)",
1053                          lstr, rc);
1054
1055         return rc;
1056 }
1057
1058 static int ct_archive(const struct hsm_action_item *hai, const long hal_flags)
1059 {
1060         struct hsm_copyaction_private   *hcp = NULL;
1061         char                             src[PATH_MAX];
1062         char                             dst[PATH_MAX + 4] = "";
1063         char                             root[PATH_MAX] = "";
1064         int                              rc;
1065         int                              rcf = 0;
1066         bool                             rename_needed = false;
1067         int                              hp_flags = 0;
1068         int                              open_flags;
1069         int                              src_fd = -1;
1070         int                              dst_fd = -1;
1071
1072         rc = ct_begin(&hcp, hai);
1073         if (rc < 0)
1074                 goto fini_major;
1075
1076         /* we fill archive so:
1077          * source = data FID
1078          * destination = lustre FID
1079          */
1080         ct_path_lustre(src, sizeof(src), opt.o_mnt, &hai->hai_dfid);
1081         ct_path_archive(root, sizeof(root), opt.o_hsm_root, &hai->hai_fid);
1082         if (hai->hai_extent.length == -1) {
1083                 /* whole file, write it to tmp location and atomically
1084                  * replace old archived file */
1085                 snprintf(dst, sizeof(dst), "%s_tmp", root);
1086                 /* we cannot rely on the same test because ct_copy_data()
1087                  * updates hai_extent.length */
1088                 rename_needed = true;
1089         } else {
1090                 snprintf(dst, sizeof(dst), "%s", root);
1091         }
1092
1093         CT_TRACE("archiving '%s' to '%s'", src, dst);
1094
1095         if (opt.o_dry_run) {
1096                 rc = 0;
1097                 goto fini_major;
1098         }
1099
1100         rc = ct_mkdir_p(dst);
1101         if (rc < 0) {
1102                 CT_ERROR(rc, "mkdir_p '%s' failed", dst);
1103                 goto fini_major;
1104         }
1105
1106         src_fd = llapi_hsm_action_get_fd(hcp);
1107         if (src_fd < 0) {
1108                 rc = src_fd;
1109                 CT_ERROR(rc, "cannot open '%s' for read", src);
1110                 goto fini_major;
1111         }
1112
1113         open_flags = O_WRONLY | O_NOFOLLOW;
1114         /* If extent is specified, don't truncate an old archived copy */
1115         open_flags |= ((hai->hai_extent.length == -1) ? O_TRUNC : 0) | O_CREAT;
1116
1117         dst_fd = open(dst, open_flags, FILE_PERM);
1118         if (dst_fd == -1) {
1119                 rc = -errno;
1120                 CT_ERROR(rc, "cannot open '%s' for write", dst);
1121                 goto fini_major;
1122         }
1123
1124         /* saving stripe is not critical */
1125         rc = ct_save_stripe(src_fd, src, dst);
1126         if (rc < 0)
1127                 CT_ERROR(rc, "cannot save file striping info of '%s' in '%s'",
1128                          src, dst);
1129
1130         rc = ct_copy_data(hcp, src, dst, src_fd, dst_fd, hai, hal_flags);
1131         if (rc < 0) {
1132                 CT_ERROR(rc, "data copy failed from '%s' to '%s'", src, dst);
1133                 goto fini_major;
1134         }
1135
1136         rc = fsync(dst_fd);
1137         if (rc < 0) {
1138                 rc = -errno;
1139                 CT_ERROR(rc, "cannot flush '%s' archive file '%s'", src, dst);
1140                 goto fini_major;
1141         }
1142
1143         CT_TRACE("data archiving for '%s' to '%s' done", src, dst);
1144
1145         /* attrs will remain on the MDS; no need to copy them, except possibly
1146           for disaster recovery */
1147         if (opt.o_copy_attrs) {
1148                 rc = ct_copy_attr(src, dst, src_fd, dst_fd);
1149                 if (rc < 0) {
1150                         CT_ERROR(rc, "cannot copy attr of '%s' to '%s'",
1151                                  src, dst);
1152                         rcf = rc;
1153                 }
1154                 CT_TRACE("attr file for '%s' saved to archive '%s'",
1155                          src, dst);
1156         }
1157
1158         /* xattrs will remain on the MDS; no need to copy them, except possibly
1159          for disaster recovery */
1160         if (opt.o_copy_xattrs) {
1161                 rc = ct_copy_xattr(src, dst, src_fd, dst_fd, false);
1162                 if (rc < 0) {
1163                         CT_ERROR(rc, "cannot copy xattr of '%s' to '%s'",
1164                                  src, dst);
1165                         rcf = rcf ? rcf : rc;
1166                 }
1167                 CT_TRACE("xattr file for '%s' saved to archive '%s'",
1168                          src, dst);
1169         }
1170
1171         if (rename_needed == true) {
1172                 char     tmp_src[PATH_MAX + 8];
1173                 char     tmp_dst[PATH_MAX + 8];
1174
1175                 /* atomically replace old archived file */
1176                 ct_path_archive(src, sizeof(src), opt.o_hsm_root,
1177                                 &hai->hai_fid);
1178                 rc = rename(dst, src);
1179                 if (rc < 0) {
1180                         rc = -errno;
1181                         CT_ERROR(rc, "cannot rename '%s' to '%s'", dst, src);
1182                         goto fini_major;
1183                 }
1184                 /* rename lov file */
1185                 snprintf(tmp_src, sizeof(tmp_src), "%s.lov", src);
1186                 snprintf(tmp_dst, sizeof(tmp_dst), "%s.lov", dst);
1187                 rc = rename(tmp_dst, tmp_src);
1188                 if (rc < 0)
1189                         CT_ERROR(errno, "cannot rename '%s' to '%s'",
1190                                  tmp_dst, tmp_src);
1191         }
1192
1193         if (opt.o_shadow_tree) {
1194                 /* Create a namespace of softlinks that shadows the original
1195                  * Lustre namespace.  This will only be current at
1196                  * time-of-archive (won't follow renames).
1197                  * WARNING: release won't kill these links; a manual
1198                  * cleanup of dead links would be required.
1199                  */
1200                 char             buf[PATH_MAX];
1201                 long long        recno = -1;
1202                 int              linkno = 0;
1203                 char            *ptr;
1204                 int              depth = 0;
1205                 ssize_t          sz;
1206
1207                 sprintf(src, "%s/shadow/", opt.o_hsm_root);
1208
1209                 ptr = opt.o_hsm_root;
1210                 while (*ptr)
1211                         (*ptr++ == '/') ? depth-- : 0;
1212
1213                 rc = llapi_fid2path_at(opt.o_mnt_fd, &hai->hai_fid,
1214                                        src + strlen(src),
1215                                        sizeof(src) - strlen(src),
1216                                        &recno, &linkno);
1217                 if (rc < 0) {
1218                         CT_ERROR(rc, "cannot get FID of "DFID,
1219                                  PFID(&hai->hai_fid));
1220                         rcf = rcf ? rcf : rc;
1221                         goto fini_minor;
1222                 }
1223
1224                 /* Figure out how many parent dirs to symlink back */
1225                 ptr = src;
1226                 while (*ptr)
1227                         (*ptr++ == '/') ? depth++ : 0;
1228                 sprintf(buf, "..");
1229                 while (--depth > 1)
1230                         strcat(buf, "/..");
1231
1232                 ct_path_archive(dst, sizeof(dst), buf, &hai->hai_fid);
1233
1234                 if (ct_mkdir_p(src)) {
1235                         CT_ERROR(errno, "mkdir_p '%s' failed", src);
1236                         rcf = rcf ? rcf : -errno;
1237                         goto fini_minor;
1238                 }
1239                 /* symlink already exists ? */
1240                 sz = readlink(src, buf, sizeof(buf));
1241                 /* detect truncation */
1242                 if (sz == sizeof(buf)) {
1243                         rcf = rcf ? rcf : -E2BIG;
1244                         CT_ERROR(rcf, "readlink '%s' truncated", src);
1245                         goto fini_minor;
1246                 }
1247                 if (sz >= 0) {
1248                         buf[sz] = '\0';
1249                         if (sz == 0 || strncmp(buf, dst, sz) != 0) {
1250                                 if (unlink(src) && errno != ENOENT) {
1251                                         CT_ERROR(errno,
1252                                                  "cannot unlink symlink '%s'",
1253                                                  src);
1254                                         rcf = rcf ? rcf : -errno;
1255                                         goto fini_minor;
1256                                 }
1257                                 /* unlink old symlink done */
1258                                 CT_TRACE("remove old symlink '%s' pointing"
1259                                          " to '%s'", src, buf);
1260                         } else {
1261                                 /* symlink already ok */
1262                                 CT_TRACE("symlink '%s' already pointing"
1263                                          " to '%s'", src, dst);
1264                                 rcf = 0;
1265                                 goto fini_minor;
1266                         }
1267                 }
1268                 if (symlink(dst, src)) {
1269                         CT_ERROR(errno, "cannot symlink '%s' to '%s'",
1270                                  src, dst);
1271                         rcf = rcf ? rcf : -errno;
1272                         goto fini_minor;
1273                 }
1274                 CT_TRACE("symlink '%s' to '%s' done", src, dst);
1275         }
1276 fini_minor:
1277         if (rcf)
1278                 err_minor++;
1279         goto out;
1280
1281
1282 fini_major:
1283         err_major++;
1284
1285         unlink(dst);
1286         if (ct_is_retryable(rc))
1287                 hp_flags |= HP_FLAG_RETRY;
1288
1289         rcf = rc;
1290
1291 out:
1292         if (!(src_fd < 0))
1293                 close(src_fd);
1294
1295         if (!(dst_fd < 0))
1296                 close(dst_fd);
1297
1298         rc = ct_fini(&hcp, hai, hp_flags, rcf);
1299
1300         return rc;
1301 }
1302
1303 static int ct_restore(const struct hsm_action_item *hai, const long hal_flags)
1304 {
1305         struct hsm_copyaction_private   *hcp = NULL;
1306         char                             src[PATH_MAX];
1307         char                             dst[PATH_MAX];
1308         char                             lov_buf[XATTR_SIZE_MAX];
1309         size_t                           lov_size = sizeof(lov_buf);
1310         int                              rc;
1311         int                              hp_flags = 0;
1312         int                              src_fd = -1;
1313         int                              dst_fd = -1;
1314         int                              mdt_index = -1;
1315         int                              open_flags = 0;
1316         bool                             set_lovea;
1317         struct lu_fid                    dfid;
1318         /* we fill lustre so:
1319          * source = lustre FID in the backend
1320          * destination = data FID = volatile file
1321          */
1322
1323         /* build backend file name from released file FID */
1324         ct_path_archive(src, sizeof(src), opt.o_hsm_root, &hai->hai_fid);
1325
1326         rc = llapi_get_mdt_index_by_fid(opt.o_mnt_fd, &hai->hai_fid,
1327                                         &mdt_index);
1328         if (rc < 0) {
1329                 CT_ERROR(rc, "cannot get mdt index "DFID"",
1330                          PFID(&hai->hai_fid));
1331                 return rc;
1332         }
1333         /* restore loads and sets the LOVEA w/o interpreting it to avoid
1334          * dependency on the structure format. */
1335         rc = ct_load_stripe(src, lov_buf, &lov_size);
1336         if (rc < 0) {
1337                 CT_WARN("cannot get stripe rules for '%s' (%s), use default",
1338                         src, strerror(-rc));
1339                 set_lovea = false;
1340         } else {
1341                 open_flags |= O_LOV_DELAY_CREATE;
1342                 set_lovea = true;
1343         }
1344
1345         rc = ct_begin_restore(&hcp, hai, mdt_index, open_flags);
1346         if (rc < 0)
1347                 goto fini;
1348
1349         /* get the FID of the volatile file */
1350         rc = llapi_hsm_action_get_dfid(hcp, &dfid);
1351         if (rc < 0) {
1352                 CT_ERROR(rc, "restoring "DFID
1353                          ", cannot get FID of created volatile file",
1354                          PFID(&hai->hai_fid));
1355                 goto fini;
1356         }
1357
1358         /* build volatile "file name", for messages */
1359         snprintf(dst, sizeof(dst), "{VOLATILE}="DFID, PFID(&dfid));
1360
1361         CT_TRACE("restoring data from '%s' to '%s'", src, dst);
1362
1363         if (opt.o_dry_run) {
1364                 rc = 0;
1365                 goto fini;
1366         }
1367
1368         src_fd = open(src, O_RDONLY | O_NOATIME | O_NOFOLLOW);
1369         if (src_fd < 0) {
1370                 rc = -errno;
1371                 CT_ERROR(rc, "cannot open '%s' for read", src);
1372                 goto fini;
1373         }
1374
1375         dst_fd = llapi_hsm_action_get_fd(hcp);
1376         if (dst_fd < 0) {
1377                 rc = dst_fd;
1378                 CT_ERROR(rc, "cannot open '%s' for write", dst);
1379                 goto fini;
1380         }
1381
1382         if (set_lovea) {
1383                 /* the layout cannot be allocated through .fid so we have to
1384                  * restore a layout */
1385                 rc = ct_restore_stripe(src, dst, dst_fd, lov_buf, lov_size);
1386                 if (rc < 0) {
1387                         CT_ERROR(rc, "cannot restore file striping info"
1388                                  " for '%s' from '%s'", dst, src);
1389                         err_major++;
1390                         goto fini;
1391                 }
1392         }
1393
1394         rc = ct_copy_data(hcp, src, dst, src_fd, dst_fd, hai, hal_flags);
1395         if (rc < 0) {
1396                 CT_ERROR(rc, "cannot copy data from '%s' to '%s'",
1397                          src, dst);
1398                 err_major++;
1399                 if (ct_is_retryable(rc))
1400                         hp_flags |= HP_FLAG_RETRY;
1401                 goto fini;
1402         }
1403
1404         CT_TRACE("data restore from '%s' to '%s' done", src, dst);
1405
1406 fini:
1407         rc = ct_fini(&hcp, hai, hp_flags, rc);
1408
1409         /* object swaping is done by cdt at copy end, so close of volatile file
1410          * cannot be done before */
1411         if (!(src_fd < 0))
1412                 close(src_fd);
1413
1414         if (!(dst_fd < 0))
1415                 close(dst_fd);
1416
1417         return rc;
1418 }
1419
1420 static int ct_remove(const struct hsm_action_item *hai, const long hal_flags)
1421 {
1422         struct hsm_copyaction_private   *hcp = NULL;
1423         char                             dst[PATH_MAX], attr[PATH_MAX + 4];
1424         int                              rc;
1425
1426         rc = ct_begin(&hcp, hai);
1427         if (rc < 0)
1428                 goto fini;
1429
1430         ct_path_archive(dst, sizeof(dst), opt.o_hsm_root, &hai->hai_fid);
1431
1432         CT_TRACE("removing file '%s'", dst);
1433
1434         if (opt.o_dry_run) {
1435                 rc = 0;
1436                 goto fini;
1437         }
1438
1439         rc = unlink(dst);
1440         if (rc < 0) {
1441                 rc = -errno;
1442                 CT_ERROR(rc, "cannot unlink '%s'", dst);
1443                 err_minor++;
1444                 goto fini;
1445         }
1446
1447         snprintf(attr, sizeof(attr), "%s.lov", dst);
1448         rc = unlink(attr);
1449         if (rc < 0) {
1450                 rc = -errno;
1451                 CT_ERROR(rc, "cannot unlink '%s'", attr);
1452                 err_minor++;
1453
1454                 /* ignore the error when lov file does not exist. */
1455                 if (rc == -ENOENT)
1456                         rc = 0;
1457                 else
1458                         goto fini;
1459         }
1460
1461 fini:
1462         rc = ct_fini(&hcp, hai, 0, rc);
1463
1464         return rc;
1465 }
1466
1467 static int ct_process_item(struct hsm_action_item *hai, const long hal_flags)
1468 {
1469         int     rc = 0;
1470
1471         if (opt.o_verbose >= LLAPI_MSG_INFO || opt.o_dry_run) {
1472                 /* Print the original path */
1473                 char            path[PATH_MAX];
1474                 long long       recno = -1;
1475                 int             linkno = 0;
1476
1477                 CT_TRACE(DFID" action %s reclen %d, cookie=%#jx",
1478                          PFID(&hai->hai_fid),
1479                          hsm_copytool_action2name(hai->hai_action),
1480                          hai->hai_len, (uintmax_t)hai->hai_cookie);
1481                 rc = llapi_fid2path_at(opt.o_mnt_fd, &hai->hai_fid, path,
1482                                        sizeof(path), &recno, &linkno);
1483                 if (rc < 0)
1484                         CT_ERROR(rc, "cannot get path of "DFID,
1485                                  PFID(&hai->hai_fid));
1486                 else
1487                         CT_TRACE("processing file '%s'", path);
1488         }
1489
1490         switch (hai->hai_action) {
1491         /* set err_major, minor inside these functions */
1492         case HSMA_ARCHIVE:
1493                 rc = ct_archive(hai, hal_flags);
1494                 break;
1495         case HSMA_RESTORE:
1496                 rc = ct_restore(hai, hal_flags);
1497                 break;
1498         case HSMA_REMOVE:
1499                 rc = ct_remove(hai, hal_flags);
1500                 break;
1501         case HSMA_CANCEL:
1502                 CT_TRACE("cancel not implemented for file system '%s'",
1503                          opt.o_mnt);
1504                 /* Don't report progress to coordinator for this cookie:
1505                  * the copy function will get ECANCELED when reporting
1506                  * progress. */
1507                 err_minor++;
1508                 return 0;
1509                 break;
1510         default:
1511                 rc = -EINVAL;
1512                 CT_ERROR(rc, "unknown action %d, on '%s'", hai->hai_action,
1513                          opt.o_mnt);
1514                 err_minor++;
1515                 ct_fini(NULL, hai, 0, rc);
1516         }
1517
1518         return 0;
1519 }
1520
1521 struct ct_th_data {
1522         long                     hal_flags;
1523         struct hsm_action_item  *hai;
1524 };
1525
1526 static void *ct_thread(void *data)
1527 {
1528         struct ct_th_data *cttd = data;
1529         int rc;
1530
1531         rc = ct_process_item(cttd->hai, cttd->hal_flags);
1532
1533         free(cttd->hai);
1534         free(cttd);
1535         pthread_exit((void *)(intptr_t)rc);
1536 }
1537
1538 static int ct_process_item_async(const struct hsm_action_item *hai,
1539                                  long hal_flags)
1540 {
1541         pthread_attr_t           attr;
1542         pthread_t                thread;
1543         struct ct_th_data       *data;
1544         int                      rc;
1545
1546         data = malloc(sizeof(*data));
1547         if (data == NULL)
1548                 return -ENOMEM;
1549
1550         data->hai = malloc(hai->hai_len);
1551         if (data->hai == NULL) {
1552                 free(data);
1553                 return -ENOMEM;
1554         }
1555
1556         memcpy(data->hai, hai, hai->hai_len);
1557         data->hal_flags = hal_flags;
1558
1559         rc = pthread_attr_init(&attr);
1560         if (rc != 0) {
1561                 CT_ERROR(rc, "pthread_attr_init failed for '%s' service",
1562                          opt.o_mnt);
1563                 free(data->hai);
1564                 free(data);
1565                 return -rc;
1566         }
1567
1568         pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
1569
1570         rc = pthread_create(&thread, &attr, ct_thread, data);
1571         if (rc != 0)
1572                 CT_ERROR(rc, "cannot create thread for '%s' service",
1573                          opt.o_mnt);
1574
1575         pthread_attr_destroy(&attr);
1576         return 0;
1577 }
1578
1579 static int ct_import_one(const char *src, const char *dst)
1580 {
1581         char            newarc[PATH_MAX];
1582         struct lu_fid   fid;
1583         struct stat     st;
1584         int             rc;
1585
1586         CT_TRACE("importing '%s' from '%s'", dst, src);
1587
1588         if (stat(src, &st) < 0) {
1589                 rc = -errno;
1590                 CT_ERROR(rc, "cannot stat '%s'", src);
1591                 return rc;
1592         }
1593
1594         if (opt.o_dry_run)
1595                 return 0;
1596
1597         rc = llapi_hsm_import(dst,
1598                               opt.o_archive_id_used ? opt.o_archive_id[0] : 0,
1599                               &st,
1600                               0 /* default stripe_size */,
1601                               -1 /* default stripe offset */,
1602                               0 /* default stripe count */,
1603                               0 /* stripe pattern (will be RAID0+RELEASED) */,
1604                               NULL /* pool_name */,
1605                               &fid);
1606         if (rc < 0) {
1607                 CT_ERROR(rc, "cannot import '%s' from '%s'", dst, src);
1608                 return rc;
1609         }
1610
1611         ct_path_archive(newarc, sizeof(newarc), opt.o_hsm_root, &fid);
1612
1613         rc = ct_mkdir_p(newarc);
1614         if (rc < 0) {
1615                 CT_ERROR(rc, "mkdir_p '%s' failed", newarc);
1616                 err_major++;
1617                 return rc;
1618
1619         }
1620
1621         /* Lots of choices now: mv, ln, ln -s ? */
1622         rc = link(src, newarc); /* hardlink */
1623         if (rc < 0) {
1624                 rc = -errno;
1625                 CT_ERROR(rc, "cannot link '%s' to '%s'", newarc, src);
1626                 err_major++;
1627                 return rc;
1628         }
1629         CT_TRACE("imported '%s' from '%s'=='%s'", dst, newarc, src);
1630
1631         return 0;
1632 }
1633
1634 static char *path_concat(const char *dirname, const char *basename)
1635 {
1636         char    *result;
1637         int      rc;
1638
1639         rc = asprintf(&result, "%s/%s", dirname, basename);
1640         if (rc < 0)
1641                 return NULL;
1642
1643         return result;
1644 }
1645
1646 static int ct_import_fid(const struct lu_fid *import_fid)
1647 {
1648         char    fid_path[PATH_MAX];
1649         int     rc;
1650
1651         ct_path_lustre(fid_path, sizeof(fid_path), opt.o_mnt, import_fid);
1652         rc = access(fid_path, F_OK);
1653         if (rc == 0 || errno != ENOENT) {
1654                 rc = (errno == 0) ? -EEXIST : -errno;
1655                 CT_ERROR(rc, "cannot import '"DFID"'", PFID(import_fid));
1656                 return rc;
1657         }
1658
1659         ct_path_archive(fid_path, sizeof(fid_path), opt.o_hsm_root,
1660                         import_fid);
1661
1662         CT_TRACE("Resolving "DFID" to %s", PFID(import_fid), fid_path);
1663
1664         return ct_import_one(fid_path, opt.o_dst);
1665 }
1666
1667 static int ct_import_recurse(const char *relpath)
1668 {
1669         DIR             *dir;
1670         struct dirent   *ent;
1671         char            *srcpath, *newpath;
1672         struct lu_fid    import_fid;
1673         int              rc;
1674
1675         if (relpath == NULL)
1676                 return -EINVAL;
1677
1678         /* Is relpath a FID? */
1679         rc = llapi_fid_parse(relpath, &import_fid, NULL);
1680         if (!rc)
1681                 return ct_import_fid(&import_fid);
1682
1683         srcpath = path_concat(opt.o_hsm_root, relpath);
1684         if (srcpath == NULL) {
1685                 err_major++;
1686                 return -ENOMEM;
1687         }
1688
1689         dir = opendir(srcpath);
1690         if (dir == NULL) {
1691                 /* Not a dir, or error */
1692                 if (errno == ENOTDIR) {
1693                         /* Single regular file case, treat o_dst as absolute
1694                            final location. */
1695                         rc = ct_import_one(srcpath, opt.o_dst);
1696                 } else {
1697                         rc = -errno;
1698                         CT_ERROR(rc, "cannot opendir '%s'", srcpath);
1699                         err_major++;
1700                 }
1701                 free(srcpath);
1702                 return rc;
1703         }
1704         free(srcpath);
1705
1706         while ((ent = readdir(dir)) != NULL) {
1707                 if (!strcmp(ent->d_name, ".") ||
1708                     !strcmp(ent->d_name, ".."))
1709                         continue;
1710
1711                 /* New relative path */
1712                 newpath = path_concat(relpath, ent->d_name);
1713                 if (newpath == NULL) {
1714                         err_major++;
1715                         rc = -ENOMEM;
1716                         goto out;
1717                 }
1718
1719                 if (ent->d_type == DT_DIR) {
1720                         rc = ct_import_recurse(newpath);
1721                 } else {
1722                         char src[PATH_MAX];
1723                         char dst[PATH_MAX];
1724
1725                         sprintf(src, "%s/%s", opt.o_hsm_root, newpath);
1726                         sprintf(dst, "%s/%s", opt.o_dst, newpath);
1727                         /* Make the target dir in the Lustre fs */
1728                         rc = ct_mkdir_p(dst);
1729                         if (rc == 0) {
1730                                 /* Import the file */
1731                                 rc = ct_import_one(src, dst);
1732                         } else {
1733                                 CT_ERROR(rc, "ct_mkdir_p '%s' failed", dst);
1734                                 err_major++;
1735                         }
1736                 }
1737
1738                 if (rc != 0) {
1739                         CT_ERROR(rc, "cannot import '%s'", newpath);
1740                         if (err_major && opt.o_abort_on_error) {
1741                                 free(newpath);
1742                                 goto out;
1743                         }
1744                 }
1745                 free(newpath);
1746         }
1747
1748         rc = 0;
1749 out:
1750         closedir(dir);
1751         return rc;
1752 }
1753
1754 static int ct_rebind_one(const struct lu_fid *old_fid,
1755                          const struct lu_fid *new_fid)
1756 {
1757         char    src[PATH_MAX];
1758         char    dst[PATH_MAX];
1759         int     rc;
1760
1761         CT_TRACE("rebind "DFID" to "DFID, PFID(old_fid), PFID(new_fid));
1762
1763         ct_path_archive(src, sizeof(src), opt.o_hsm_root, old_fid);
1764         ct_path_archive(dst, sizeof(dst), opt.o_hsm_root, new_fid);
1765
1766         if (!opt.o_dry_run) {
1767                 char src_attr[PATH_MAX + 4];
1768                 char dst_attr[PATH_MAX + 4];
1769
1770                 ct_mkdir_p(dst);
1771                 if (rename(src, dst)) {
1772                         rc = -errno;
1773                         CT_ERROR(rc, "cannot rename '%s' to '%s'", src, dst);
1774                         return -errno;
1775                 }
1776                 /* rename lov file */
1777                 snprintf(src_attr, sizeof(src_attr), "%s.lov", src);
1778                 snprintf(dst_attr, sizeof(dst_attr), "%s.lov", dst);
1779                 if (rename(src_attr, dst_attr))
1780                         CT_ERROR(errno, "cannot rename '%s' to '%s'",
1781                                  src_attr, dst_attr);
1782
1783         }
1784         return 0;
1785 }
1786
1787 static bool fid_is_file(struct lu_fid *fid)
1788 {
1789         return fid_is_norm(fid) || fid_is_igif(fid);
1790 }
1791
1792 static bool should_ignore_line(const char *line)
1793 {
1794         int     i;
1795
1796         for (i = 0; line[i] != '\0'; i++) {
1797                 if (isspace(line[i]))
1798                         continue;
1799                 else if (line[i] == '#')
1800                         return true;
1801                 else
1802                         return false;
1803         }
1804
1805         return true;
1806 }
1807
1808 static int ct_rebind_list(const char *list)
1809 {
1810         int              rc;
1811         FILE            *filp;
1812         ssize_t          r;
1813         char            *line = NULL;
1814         size_t           line_size = 0;
1815         unsigned int     nl = 0;
1816         unsigned int     ok = 0;
1817
1818         filp = fopen(list, "r");
1819         if (filp == NULL) {
1820                 rc = -errno;
1821                 CT_ERROR(rc, "cannot open '%s'", list);
1822                 return rc;
1823         }
1824
1825         /* each line consists of 2 FID */
1826         while ((r = getline(&line, &line_size, filp)) != -1) {
1827                 struct lu_fid old_fid;
1828                 struct lu_fid new_fid;
1829                 char *next_fid;
1830
1831                 /* Ignore empty and commented out ('#...') lines. */
1832                 if (should_ignore_line(line))
1833                         continue;
1834
1835                 nl++;
1836
1837                 rc = llapi_fid_parse(line, &old_fid, &next_fid);
1838                 if (rc)
1839                         goto error;
1840                 rc = llapi_fid_parse(next_fid, &new_fid, NULL);
1841                 if (rc)
1842                         goto error;
1843                 if (!fid_is_file(&old_fid) || !fid_is_file(&new_fid))
1844                         rc = -EINVAL;
1845                 if (rc) {
1846 error:                  CT_ERROR(rc, "%s:%u: two FIDs expected in '%s'",
1847                                  list, nl, line);
1848                         err_major++;
1849                         continue;
1850                 }
1851
1852                 if (ct_rebind_one(&old_fid, &new_fid))
1853                         err_major++;
1854                 else
1855                         ok++;
1856         }
1857
1858         fclose(filp);
1859
1860         if (line)
1861                 free(line);
1862
1863         /* return 0 if all rebinds were successful */
1864         CT_TRACE("%u lines read from '%s', %u rebind successful", nl, list, ok);
1865
1866         return ok == nl ? 0 : -1;
1867 }
1868
1869 static int ct_rebind(void)
1870 {
1871         int rc;
1872
1873         if (opt.o_dst) {
1874                 struct lu_fid old_fid;
1875                 struct lu_fid new_fid;
1876
1877                 rc = llapi_fid_parse(opt.o_src, &old_fid, NULL);
1878                 if (!rc && !fid_is_file(&old_fid))
1879                         rc = -EINVAL;
1880                 if (rc) {
1881                         CT_ERROR(rc, "invalid source FID '%s'", opt.o_src);
1882                         return rc;
1883                 }
1884
1885                 rc = llapi_fid_parse(opt.o_dst, &new_fid, NULL);
1886                 if (!rc && !fid_is_file(&new_fid))
1887                         rc = -EINVAL;
1888                 if (rc) {
1889                         CT_ERROR(rc, "invalid destination FID '%s'", opt.o_dst);
1890                         return rc;
1891                 }
1892
1893                 rc = ct_rebind_one(&old_fid, &new_fid);
1894
1895                 return rc;
1896         }
1897
1898         /* o_src is a list file */
1899         rc = ct_rebind_list(opt.o_src);
1900
1901         return rc;
1902 }
1903
1904 static int ct_opendirat(int parent_fd, const char *name, DIR **pdir)
1905 {
1906         DIR *dir = NULL;
1907         int fd = -1;
1908         int rc;
1909
1910         fd = openat(parent_fd, name, O_RDONLY|O_DIRECTORY);
1911         if (fd < 0)
1912                 return -errno;
1913
1914         dir = fdopendir(fd);
1915         if (dir == NULL) {
1916                 rc = -errno;
1917                 goto out;
1918         }
1919
1920         *pdir = dir;
1921         fd = -1;
1922         rc = 0;
1923 out:
1924         if (!(fd < 0))
1925                 close(fd);
1926
1927         return rc;
1928 }
1929
1930 static int ct_archive_upgrade_reg(int arc_fd, enum ct_archive_format ctaf,
1931                                   int old_fd, const char *name)
1932 {
1933         char new_path[PATH_MAX];
1934         struct lu_fid fid;
1935         char *split;
1936         int scan_count;
1937         int suffix_offset = -1;
1938         int rc;
1939
1940         /* Formatted fix with optional suffix. We do not inspect
1941          * suffixes. */
1942         scan_count = sscanf(name, SFID"%n", RFID(&fid), &suffix_offset);
1943         if (scan_count != 3 || suffix_offset < 0) {
1944                 rc = 0;
1945                 CT_TRACE("ignoring unexpected file '%s' in archive", name);
1946                 goto out;
1947         }
1948
1949         ct_path_archive_v(new_path, sizeof(new_path),
1950                           ctaf, ".", &fid, name + suffix_offset);
1951
1952         rc = renameat(old_fd, name, arc_fd, new_path);
1953         if (rc == 0)
1954                 goto out;
1955
1956         if (errno != ENOENT) {
1957                 rc = -errno;
1958                 goto out;
1959         }
1960
1961         /* Create parent directory and try again. */
1962         split = strrchr(new_path, '/');
1963
1964         *split = '\0';
1965         rc = ct_mkdirat_p(arc_fd, new_path, DIR_PERM);
1966         *split = '/';
1967         if (rc < 0)
1968                 goto out;
1969
1970         rc = renameat(old_fd, name, arc_fd, new_path);
1971         if (rc < 0)
1972                 rc = -errno;
1973 out:
1974         if (rc < 0)
1975                 CT_ERROR(rc, "cannot rename '%s' to '%s'", name, new_path);
1976         else
1977                 CT_TRACE("renamed '%s' to '%s'", name, new_path);
1978
1979         return rc;
1980 }
1981
1982 static const char *d_type_name(unsigned int type)
1983 {
1984         static const char *name[] = {
1985                 [DT_UNKNOWN] = "unknown",
1986                 [DT_FIFO] = "fifo",
1987                 [DT_CHR] = "chr",
1988                 [DT_DIR] = "dir",
1989                 [DT_BLK] = "blk",
1990                 [DT_REG] = "reg",
1991                 [DT_LNK] = "lnk",
1992                 [DT_SOCK] = "sock",
1993                 [DT_WHT] = "wht",
1994         };
1995
1996         if (type < ARRAY_SIZE(name) && name[type] != NULL)
1997                 return name[type];
1998
1999         return name[DT_UNKNOWN];
2000 }
2001
2002 static int ct_archive_upgrade_dir(int arc_fd, enum ct_archive_format ctaf,
2003                                   int parent_fd, const char *dir_name)
2004 {
2005         DIR *dir = NULL;
2006         struct dirent *d;
2007         int rc = 0;
2008         int rc2;
2009
2010         rc = ct_opendirat(parent_fd, dir_name, &dir);
2011         if (rc < 0) {
2012                 CT_ERROR(rc, "cannot open archive dir '%s'", dir_name);
2013                 goto out;
2014         }
2015
2016         while ((d = readdir(dir)) != NULL) {
2017                 CT_TRACE("archive upgrade found %s '%s' (%ld)\n",
2018                          d_type_name(d->d_type), d->d_name, d->d_ino);
2019
2020                 switch (d->d_type) {
2021                 case DT_DIR:
2022                         if (strcmp(d->d_name, ".") == 0 ||
2023                             strcmp(d->d_name, "..") == 0)
2024                                 continue;
2025
2026                         if (strlen(d->d_name) != 4 ||
2027                             strspn(d->d_name, "0123456789abcdef") != 4)
2028                                 goto ignore;
2029
2030                         rc2 = ct_archive_upgrade_dir(arc_fd, ctaf,
2031                                                      dirfd(dir), d->d_name);
2032                         if (rc2 < 0) {
2033                                 rc = rc2;
2034                                 CT_ERROR(rc, "cannot upgrade dir '%s' (%ld)",
2035                                          d->d_name, d->d_ino);
2036                         }
2037
2038                         rc2 = unlinkat(dirfd(dir), d->d_name, AT_REMOVEDIR);
2039                         CT_TRACE("unlink dir '%s' (%ld): %s",
2040                                  d->d_name, d->d_ino, strerror(rc2 < 0 ? errno: 0));
2041                         if (rc2 < 0 && errno != ENOTEMPTY)
2042                                 rc = -errno;
2043
2044                         break;
2045                 case DT_REG:
2046                         rc2 = ct_archive_upgrade_reg(arc_fd, ctaf,
2047                                                      dirfd(dir), d->d_name);
2048                         if (rc2 < 0)
2049                                 rc = rc2;
2050
2051                         break;
2052                 default:
2053 ignore:
2054                         CT_TRACE("ignoring unexpected %s '%s' (%ld) in archive",
2055                                  d_type_name(d->d_type), d->d_name, d->d_ino);
2056                         break;
2057                 }
2058         }
2059 out:
2060         if (dir != NULL)
2061                 closedir(dir);
2062
2063         return rc;
2064 }
2065
2066 /* Recursive inplace upgrade (or downgrade) of archive to format
2067  * ctaf. Prunes empty archive subdirectories. Idempotent. */
2068 static int ct_archive_upgrade(int arc_fd, enum ct_archive_format ctaf)
2069 {
2070         /* FIXME Handle shadow tree. */
2071         CT_TRACE("upgrade archive to format %s", ct_archive_format_to_str(ctaf));
2072
2073         return ct_archive_upgrade_dir(arc_fd, ctaf, arc_fd, ".");
2074 }
2075
2076 static int ct_dir_level_max(const char *dirpath, __u16 *sub_seqmax)
2077 {
2078         DIR             *dir;
2079         int              rc;
2080         __u16            sub_seq;
2081         struct dirent *ent;
2082
2083         *sub_seqmax = 0;
2084
2085         dir = opendir(dirpath);
2086         if (dir == NULL) {
2087                 rc = -errno;
2088                 CT_ERROR(rc, "cannot open directory '%s'", opt.o_hsm_root);
2089                 return rc;
2090         }
2091
2092         do {
2093                 errno = 0;
2094                 ent = readdir(dir);
2095                 if (ent == NULL) {
2096                         /* end of directory.
2097                          * rc is 0 and seqmax contains the max value. */
2098                         rc = -errno;
2099                         if (rc)
2100                                 CT_ERROR(rc, "cannot readdir '%s'", dirpath);
2101                         goto out;
2102                 }
2103
2104                 if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
2105                         continue;
2106
2107                 if (sscanf(ent->d_name, "%hx", &sub_seq) != 1) {
2108                         CT_TRACE("'%s' has an unexpected dirname format, "
2109                                  "skip entry", ent->d_name);
2110                         continue;
2111                 }
2112                 if (sub_seq > *sub_seqmax)
2113                         *sub_seqmax = sub_seq;
2114         } while (1);
2115 out:
2116         closedir(dir);
2117         return rc;
2118 }
2119
2120 static int ct_max_sequence(void)
2121 {
2122         int     rc, i;
2123         char    path[PATH_MAX];
2124         __u64   seq = 0;
2125         __u16   subseq;
2126
2127         snprintf(path, sizeof(path), "%s", opt.o_hsm_root);
2128         /* FID sequence is stored in top-level directory names:
2129          * hsm_root/16bits (high weight)/16 bits/16 bits/16 bits (low weight).
2130          */
2131         for (i = 0; i < 4; i++) {
2132                 size_t path_len;
2133
2134                 rc = ct_dir_level_max(path, &subseq);
2135                 if (rc != 0)
2136                         return rc;
2137                 seq |= ((__u64)subseq << ((3 - i) * 16));
2138                 path_len = strlen(path);
2139                 rc = snprintf(path + path_len, sizeof(path) - path_len,
2140                               "/%04x", subseq);
2141                 if (rc >= (sizeof(path) - path_len))
2142                         return -E2BIG;
2143                 path[sizeof(path) - 1] = '\0';
2144         }
2145
2146         printf("max_sequence: %#jx\n", (uintmax_t)seq);
2147
2148         return 0;
2149 }
2150
2151 static void handler(int signal)
2152 {
2153         psignal(signal, "exiting");
2154         /* If we don't clean up upon interrupt, umount thinks there's a ref
2155          * and doesn't remove us from mtab (EINPROGRESS). The lustre client
2156          * does successfully unmount and the mount is actually gone, but the
2157          * mtab entry remains. So this just makes mtab happier. */
2158         llapi_hsm_copytool_unregister(&ctdata);
2159
2160         /* Also remove fifo upon signal as during normal/error exit */
2161         if (opt.o_event_fifo != NULL)
2162                 llapi_hsm_unregister_event_fifo(opt.o_event_fifo);
2163         _exit(1);
2164 }
2165
2166 /* Daemon waits for messages from the kernel; run it in the background. */
2167 static int ct_run(void)
2168 {
2169         struct sigaction cleanup_sigaction;
2170         int rc;
2171
2172         if (opt.o_daemonize) {
2173                 rc = daemon(1, 1);
2174                 if (rc < 0) {
2175                         rc = -errno;
2176                         CT_ERROR(rc, "cannot daemonize");
2177                         return rc;
2178                 }
2179         }
2180
2181         if (opt.o_pid_file != NULL) {
2182                 pid_file_fd = create_pid_file(opt.o_pid_file);
2183                 if (pid_file_fd < 0) {
2184                         rc = -errno;
2185                         CT_ERROR(rc, "cannot create PID file");
2186                         return rc;
2187                 }
2188         }
2189
2190         setbuf(stdout, NULL);
2191
2192         if (opt.o_event_fifo != NULL) {
2193                 rc = llapi_hsm_register_event_fifo(opt.o_event_fifo);
2194                 if (rc < 0) {
2195                         CT_ERROR(rc, "failed to register event fifo");
2196                         return rc;
2197                 }
2198                 llapi_error_callback_set(llapi_hsm_log_error);
2199         }
2200
2201         rc = llapi_hsm_copytool_register(&ctdata, opt.o_mnt,
2202                                          opt.o_archive_id_used,
2203                                          opt.o_archive_id, 0);
2204         if (rc < 0) {
2205                 CT_ERROR(rc, "cannot start copytool interface");
2206                 return rc;
2207         }
2208
2209         memset(&cleanup_sigaction, 0, sizeof(cleanup_sigaction));
2210         cleanup_sigaction.sa_handler = handler;
2211         sigemptyset(&cleanup_sigaction.sa_mask);
2212         sigaction(SIGINT, &cleanup_sigaction, NULL);
2213         sigaction(SIGTERM, &cleanup_sigaction, NULL);
2214
2215         while (1) {
2216                 struct hsm_action_list *hal;
2217                 struct hsm_action_item *hai;
2218                 int msgsize;
2219                 int i = 0;
2220
2221                 CT_TRACE("waiting for message from kernel");
2222
2223                 rc = llapi_hsm_copytool_recv(ctdata, &hal, &msgsize);
2224                 if (rc == -ESHUTDOWN) {
2225                         CT_TRACE("shutting down");
2226                         break;
2227                 } else if (rc < 0) {
2228                         CT_WARN("cannot receive action list: %s",
2229                                 strerror(-rc));
2230                         err_major++;
2231                         if (opt.o_abort_on_error)
2232                                 break;
2233                         else
2234                                 continue;
2235                 }
2236
2237                 CT_TRACE("copytool fs=%s archive#=%d item_count=%d",
2238                          hal->hal_fsname, hal->hal_archive_id, hal->hal_count);
2239
2240                 if (strcmp(hal->hal_fsname, fs_name) != 0) {
2241                         rc = -EINVAL;
2242                         CT_ERROR(rc, "'%s' invalid fs name, expecting: %s",
2243                                  hal->hal_fsname, fs_name);
2244                         err_major++;
2245                         if (opt.o_abort_on_error)
2246                                 break;
2247                         else
2248                                 continue;
2249                 }
2250
2251                 hai = hai_first(hal);
2252                 while (++i <= hal->hal_count) {
2253                         if ((char *)hai - (char *)hal > msgsize) {
2254                                 rc = -EPROTO;
2255                                 CT_ERROR(rc,
2256                                          "'%s' item %d past end of message!",
2257                                          opt.o_mnt, i);
2258                                 err_major++;
2259                                 break;
2260                         }
2261                         rc = ct_process_item_async(hai, hal->hal_flags);
2262                         if (rc < 0)
2263                                 CT_ERROR(rc, "'%s' item %d process",
2264                                          opt.o_mnt, i);
2265                         if (opt.o_abort_on_error && err_major)
2266                                 break;
2267                         hai = hai_next(hai);
2268                 }
2269
2270                 if (opt.o_abort_on_error && err_major)
2271                         break;
2272         }
2273
2274         llapi_hsm_copytool_unregister(&ctdata);
2275         if (opt.o_event_fifo != NULL)
2276                 llapi_hsm_unregister_event_fifo(opt.o_event_fifo);
2277
2278         return rc;
2279 }
2280
2281 static int ct_config_get_str(struct cYAML *obj, const char *key, char **pvalue)
2282 {
2283         struct cYAML *cy;
2284         char *value;
2285
2286         if (obj->cy_type != CYAML_TYPE_OBJECT)
2287                 return -EINVAL;
2288
2289         for (cy = obj->cy_child; cy != NULL; cy = cy->cy_next) {
2290                 if (cy->cy_string != NULL && strcmp(cy->cy_string, key) == 0) {
2291                         if (cy->cy_type != CYAML_TYPE_STRING)
2292                                 return -EINVAL;
2293
2294                         if (cy->cy_valuestring == NULL)
2295                                 return -EINVAL;
2296
2297                         value = strdup(cy->cy_valuestring);
2298                         if (value == NULL)
2299                                 return -ENOMEM;
2300
2301                         *pvalue = value;
2302
2303                         return 0;
2304                 }
2305         }
2306
2307         return -ENOENT;
2308 }
2309
2310 static int ct_config_archive_format(struct cYAML *config)
2311 {
2312         char *value = NULL;
2313         int rc;
2314
2315         rc = ct_config_get_str(config, "archive_format", &value);
2316         if (rc < 0)
2317                 return (rc == -ENOENT) ? 0 : rc;
2318
2319         rc = ct_str_to_archive_format(value, &opt.o_archive_format);
2320         if (rc < 0)
2321                 goto out;
2322
2323         CT_TRACE("setting archive format to %s",
2324                  ct_archive_format_to_str(opt.o_archive_format));
2325
2326 out:
2327         free(value);
2328
2329         return 0;
2330 }
2331
2332 static int ct_config_archive_path(struct cYAML *config)
2333 {
2334         int rc;
2335
2336         rc = ct_config_get_str(config, "archive_path", &opt.o_hsm_root);
2337         if (rc < 0)
2338                 return (rc == -ENOENT) ? 0 : rc;
2339
2340         CT_TRACE("setting archive path to '%s'", opt.o_hsm_root);
2341
2342         return 0;
2343 }
2344
2345 static int ct_config(const char *path)
2346 {
2347         FILE *file = NULL;
2348         struct cYAML *config = NULL;
2349         int rc;
2350
2351         if (path == NULL)
2352                 return 0;
2353
2354         file = fopen(path, "r");
2355         if (file == NULL) {
2356                 rc = -errno;
2357                 CT_ERROR(rc, "cannot open '%s'", path);
2358                 goto out;
2359         }
2360
2361         config = cYAML_load(file, NULL, false);
2362         if (config == NULL) {
2363                 rc = -EINVAL;
2364                 CT_ERROR(rc, "cannot load archive config from '%s'", path);
2365                 goto out;
2366         }
2367
2368         rc = ct_config_archive_format(config);
2369         if (rc < 0) {
2370                 CT_ERROR(rc, "cannot load archive format from '%s'", path);
2371                 goto out;
2372         }
2373
2374         rc = ct_config_archive_path(config);
2375         if (rc < 0) {
2376                 CT_ERROR(rc, "cannot load archive path from '%s'", path);
2377                 goto out;
2378         }
2379 out:
2380         if (config != NULL)
2381                 cYAML_free_tree(config);
2382
2383         if (file != NULL)
2384                 fclose(file);
2385
2386         return rc;
2387 }
2388
2389 static int ct_setup(void)
2390 {
2391         int     rc;
2392
2393         if (opt.o_action == CA_ARCHIVE_UPGRADE)
2394                 return 0;
2395
2396         rc = llapi_search_fsname(opt.o_mnt, fs_name);
2397         if (rc < 0) {
2398                 CT_ERROR(rc, "cannot find a Lustre filesystem mounted at '%s'",
2399                          opt.o_mnt);
2400                 return rc;
2401         }
2402
2403         opt.o_mnt_fd = open(opt.o_mnt, O_RDONLY);
2404         if (opt.o_mnt_fd < 0) {
2405                 rc = -errno;
2406                 CT_ERROR(rc, "cannot open mount point at '%s'",
2407                          opt.o_mnt);
2408                 return rc;
2409         }
2410
2411         return rc;
2412 }
2413
2414 static int ct_cleanup(void)
2415 {
2416         int     rc;
2417
2418         if (opt.o_mnt_fd >= 0) {
2419                 rc = close(opt.o_mnt_fd);
2420                 if (rc < 0) {
2421                         rc = -errno;
2422                         CT_ERROR(rc, "cannot close mount point");
2423                         return rc;
2424                 }
2425         }
2426
2427         if (arc_fd >= 0) {
2428                 rc = close(arc_fd);
2429                 if (rc < 0) {
2430                         rc = -errno;
2431                         CT_ERROR(rc, "cannot close archive root directory");
2432                         return rc;
2433                 }
2434         }
2435
2436         if (opt.o_archive_id_cnt > 0) {
2437                 free(opt.o_archive_id);
2438                 opt.o_archive_id = NULL;
2439                 opt.o_archive_id_cnt = 0;
2440         }
2441
2442         return 0;
2443 }
2444
2445 int main(int argc, char **argv)
2446 {
2447         int     rc;
2448
2449         snprintf(cmd_name, sizeof(cmd_name), "%s", basename(argv[0]));
2450         rc = ct_parseopts(argc, argv);
2451         if (rc < 0) {
2452                 CT_WARN("try '%s --help' for more information", cmd_name);
2453                 return -rc;
2454         }
2455
2456         llapi_msg_set_level(opt.o_verbose);
2457
2458         rc = ct_config(opt.o_config_path);
2459         if (rc < 0)
2460                 return -rc;
2461
2462         if (opt.o_hsm_root == NULL) {
2463                 rc = -EINVAL;
2464                 CT_ERROR(rc, "must specify a root directory for the backend");
2465                 return -rc;
2466         }
2467
2468         arc_fd = open(opt.o_hsm_root, O_RDONLY);
2469         if (arc_fd < 0) {
2470                 rc = -errno;
2471                 CT_ERROR(rc, "cannot open archive at '%s'", opt.o_hsm_root);
2472                 return rc;
2473         }
2474
2475         rc = ct_setup();
2476         if (rc < 0)
2477                 goto error_cleanup;
2478
2479         switch (opt.o_action) {
2480         case CA_IMPORT:
2481                 rc = ct_import_recurse(opt.o_src);
2482                 break;
2483         case CA_REBIND:
2484                 rc = ct_rebind();
2485                 break;
2486         case CA_MAXSEQ:
2487                 rc = ct_max_sequence();
2488                 break;
2489         case CA_ARCHIVE_UPGRADE:
2490                 rc = ct_archive_upgrade(arc_fd, opt.o_archive_format);
2491                 break;
2492         default:
2493                 rc = ct_run();
2494                 break;
2495         }
2496
2497         if (opt.o_action != CA_MAXSEQ)
2498                 CT_TRACE("process finished, errs: %d major, %d minor,"
2499                          " rc=%d (%s)", err_major, err_minor, rc,
2500                          strerror(-rc));
2501
2502 error_cleanup:
2503         ct_cleanup();
2504
2505         return -rc;
2506 }