lustre/utils/ll_recover_lost_found_objs.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2013, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lustre/utils/ll_recover_lost_found_objs.c
  37  *
  38  * Tool for recovering objects from lost+found that might result from a
  39  * Lustre OST with a corrupted directory. Running e2fsck will fix the
  40  * directory, but puts all of the objects into lost+found, where they are
  41  * inaccessible to Lustre.
  42  *
  43  * Author: Kalpak Shah <kalpak.shah@sun.com>
  44  */
  45
  46 #ifndef _GNU_SOURCE
  47 #define _GNU_SOURCE
  48 #endif
  49
  50 #include <stdio.h>
  51 #include <stdlib.h>
  52 #include <unistd.h>
  53 #include <string.h>
  54 #include <errno.h>
  55 #include <dirent.h>
  56 #include <sys/types.h>
  57 #include <sys/xattr.h>
  58 #include <sys/stat.h>
  59
  60 #include <liblustre.h>
  61 #include <libcfs/list.h>
  62
  63 #define MAX_GROUPS 64
  64
  65 int verbose;
  66 bool dry_run;
  67
  68 struct obd_group_info {
  69         __u64           grp_last_id;
  70         __u64           grp_seq;
  71         cfs_list_t      grp_list;
  72 };
  73
  74 cfs_list_t grp_info_list;
  75
  76 static void grp_info_list_destroy(cfs_list_t *list)
  77 {
  78         struct obd_group_info *grp, *tmp;
  79
  80         cfs_list_for_each_entry_safe(grp, tmp, list, grp_list) {
  81                 cfs_list_del_init(&grp->grp_list);
  82                 free(grp);
  83         }
  84 }
  85
  86 static void usage(char *progname)
  87 {
  88         fprintf(stderr, "Usage: %s [-hnv] -d directory\n"
  89                 "Recover Lustre OST objects put in lost+found by e2fsck.\n"
  90                 "\t-d: Check directory, usually '/lost+found' (required).\n"
  91                 "\t    Alternately, verify object directories under '/O'.\n"
  92                 "\t-n: Do not modify filesystem, only report changes.\n"
  93                 "\t-h: Print help message.\n"
  94                 "\t-v: Print changes verbosely.\n",
  95                 progname);
  96         exit(1);
  97 }
  98
  99 static int _ll_sprintf(char *buf, size_t size, const char *func, int line,
 100                       const char *format, ...)
 101 {
 102         int rc;
 103         va_list ap;
 104
 105         va_start(ap, format);
 106         rc = vsnprintf(buf, size, format, ap);
 107         if (!(rc > -1 && rc < size)) {
 108                 fprintf(stderr,
 109                         "error: %s:%d: path \"", func, line);
 110                 vfprintf(stderr, format, ap);
 111                 va_end(ap);
 112                 fprintf(stderr, "\" is too long\n");
 113                 return 1;
 114         }
 115         va_end(ap);
 116         return 0;
 117 }
 118
 119 #define ll_sprintf(buf, size, format, ...) \
 120         _ll_sprintf(buf, size, __FUNCTION__, __LINE__, format, ## __VA_ARGS__)
 121
 122 static int mkdir_p(const char *dest_path, mode_t mode)
 123 {
 124         struct stat stat_buf;
 125         int rc;
 126
 127         rc = stat(dest_path, &stat_buf);
 128         if (rc == 0) {
 129                 if (S_ISDIR(stat_buf.st_mode))
 130                         goto out;
 131                 if (!S_ISDIR(stat_buf.st_mode)) {
 132                         fprintf(stderr, "error: '%s' is not a directory (%o)\n",
 133                                 dest_path, stat_buf.st_mode);
 134                         rc = -ENOTDIR;
 135                         goto out;
 136                 }
 137         } else if (errno != ENOENT) {
 138                 rc = -errno;
 139                 fprintf(stderr, "error: error checking directory '%s': %s\n",
 140                         dest_path, strerror(errno));
 141                 goto out;
 142         }
 143
 144         if (dry_run) {
 145                 fprintf(stderr, "dry_run: not creating directory '%s'\n",
 146                         dest_path);
 147                 rc = 0;
 148                 goto out;
 149         }
 150
 151         rc = mkdir(dest_path, mode);
 152         if (rc != 0)
 153                 fprintf(stderr, "error: creating directory '%s': %s\n",
 154                         dest_path, strerror(errno));
 155 out:
 156         return rc;
 157 }
 158
 159 /* This is returning 0 for an error */
 160 static __u64 read_last_id(char *file_path)
 161 {
 162         __u64 last_id;
 163         int fd;
 164         int count;
 165
 166         fd = open(file_path, O_RDONLY);
 167         if (fd < 0) {
 168                 if (errno != ENOENT)
 169                         fprintf(stderr, "error: opening '%s': %s\n",
 170                                 file_path, strerror(errno));
 171                 return 0;
 172         }
 173
 174         count = read(fd, &last_id, sizeof(last_id));
 175         if (count < 0) {
 176                 fprintf(stderr, "error: reading file '%s': %s\n",
 177                         file_path, strerror(errno));
 178                 close(fd);
 179                 return 0;
 180         }
 181         if (count != sizeof(last_id)) {
 182                 fprintf(stderr, "error: only read %d bytes from '%s'\n",
 183                         count, file_path);
 184                 close(fd);
 185                 return 0;
 186         }
 187
 188         close(fd);
 189         return le64_to_cpu(last_id);
 190 }
 191
 192 struct obd_group_info *find_or_create_grp(cfs_list_t *list, __u64 seq,
 193                                           const char *mount)
 194 {
 195         struct obd_group_info   *grp;
 196         cfs_list_t              *entry;
 197         char                    tmp_path[PATH_MAX];
 198         char                    seq_name[32];
 199         int                     retval;
 200         __u64                   tmp_last_id;
 201
 202         cfs_list_for_each(entry, list) {
 203                 grp = (struct obd_group_info *)cfs_list_entry(entry,
 204                                                 struct obd_group_info,
 205                                                 grp_list);
 206                 if (grp->grp_seq == seq)
 207                         return grp;
 208         }
 209
 210         grp = malloc(sizeof(struct obd_group_info));
 211         if (grp == NULL)
 212                 return NULL;
 213
 214         sprintf(seq_name, (fid_seq_is_rsvd(seq) ||
 215                            fid_seq_is_mdt0(seq)) ? LPU64 : LPX64i,
 216                            fid_seq_is_idif(seq) ? 0 : seq);
 217
 218         /* Check whether the obj dir has been created */
 219         if (ll_sprintf(tmp_path, PATH_MAX, "%s/O/%s", mount, seq_name)) {
 220                 free(grp);
 221                 return NULL;
 222         }
 223
 224         retval = mkdir_p(tmp_path, 0700);
 225         if (retval < 0) {
 226                 free(grp);
 227                 fprintf(stderr, "error: creating directory %s: %s\n",
 228                         tmp_path, strerror(errno));
 229                 return NULL;
 230         }
 231
 232         if (ll_sprintf(tmp_path, PATH_MAX, "%s/O/%s/LAST_ID",
 233                        mount, seq_name)) {
 234                 free(grp);
 235                 return NULL;
 236         }
 237
 238         /*
 239          * Object ID needs to be verified against last_id.
 240          * LAST_ID file may not be present in the group directory
 241          * due to corruption. In case of any error try to recover
 242          * as many objects as possible by setting last_id to ~0ULL.
 243          */
 244         tmp_last_id = read_last_id(tmp_path);
 245         if (tmp_last_id == 0)
 246                 tmp_last_id = ~0ULL;
 247         grp->grp_last_id = tmp_last_id;
 248         grp->grp_seq = seq;
 249
 250         cfs_list_add(&grp->grp_list, list);
 251         return grp;
 252 }
 253
 254 static unsigned filetype_dir_table[] = {
 255         [0]= DT_UNKNOWN,
 256         [S_IFIFO]= DT_FIFO,
 257         [S_IFCHR] = DT_CHR,
 258         [S_IFDIR] = DT_DIR,
 259         [S_IFBLK] = DT_BLK,
 260         [S_IFREG] = DT_REG,
 261         [S_IFLNK] = DT_LNK,
 262         [S_IFSOCK]= DT_SOCK,
 263 #if defined(DT_DOOR) && defined(S_IFDOOR)
 264         [S_IFDOOR]= DT_DOOR,
 265 #endif
 266 };
 267
 268 static int traverse_lost_found(char *src_dir, const char *mount_path)
 269 {
 270         DIR *dir_ptr;
 271         struct lustre_mdt_attrs lma;
 272         struct dirent64 *dirent;
 273         __u64 ff_seq, ff_objid;
 274         char *file_path;
 275         char dest_path[PATH_MAX];
 276         struct stat st;
 277         int obj_exists, xattr_len;
 278         int len, ret = 0, error = 0;
 279         char seq_name[32];
 280         char obj_name[32];
 281         struct obd_group_info *grp_info;
 282
 283         len = strlen(src_dir);
 284
 285         dir_ptr = opendir(src_dir);
 286         if (!dir_ptr) {
 287                 fprintf(stderr, "error: opening directory: %s\n",
 288                         strerror(errno));
 289                 return 1;
 290         }
 291
 292         while ((dirent = readdir64(dir_ptr)) != NULL) {
 293                 if (!strcmp(dirent->d_name, ".") ||
 294                     !strcmp(dirent->d_name, ".."))
 295                         continue;
 296
 297                 src_dir[len] = 0;
 298                 if ((len + strlen(dirent->d_name) + 2) > PATH_MAX) {
 299                         fprintf(stderr, "error: %s/%s: path too long\n",
 300                                 src_dir, dirent->d_name);
 301                         break;
 302                 }
 303                 strcat(src_dir, "/");
 304                 strcat(src_dir, dirent->d_name);
 305
 306                 if (dirent->d_type == DT_UNKNOWN) {
 307                         ret = stat(src_dir, &st);
 308                         if (ret == -1) {
 309                                 fprintf(stderr,
 310                                         "error: stating %s: %s\n",
 311                                         src_dir, strerror(errno));
 312                                 continue;
 313                         }
 314                         dirent->d_type = filetype_dir_table[st.st_mode &
 315                                                             S_IFMT];
 316                         if (dirent->d_type == DT_UNKNOWN) {
 317                                 fprintf(stderr,
 318                                         "error: %s of unknown type 0%o\n",
 319                                         src_dir, st.st_mode);
 320                                 continue;
 321                         }
 322                 }
 323
 324                 switch(dirent->d_type) {
 325                 case DT_DIR:
 326                 ret = traverse_lost_found(src_dir, mount_path);
 327                 if (ret) {
 328                         closedir(dir_ptr);
 329                         return ret;
 330                 }
 331                 break;
 332
 333                 case DT_REG:
 334                 file_path = src_dir;
 335                 xattr_len = getxattr(file_path, "trusted.lma",
 336                                      (void *)&lma, sizeof(lma));
 337                 if (xattr_len == -1 || xattr_len < sizeof(lma)) {
 338                         struct filter_fid_old   ff;
 339
 340                         /* try old filter_fid EA */
 341                         xattr_len = getxattr(file_path, "trusted.fid",
 342                                              (void *)&ff, sizeof(ff));
 343                         /* It's very much possible that we don't find any
 344                          * FID on precreated or unused objects or LAST_ID.
 345                          * The xattr needs to hold the full filter_fid_old
 346                          * with the OID/parent to be useful. */
 347                         if (xattr_len == -1 || xattr_len < sizeof(ff))
 348                                 continue;
 349
 350                         ff_seq = le64_to_cpu(ff.ff_seq);
 351                         ff_objid = le64_to_cpu(ff.ff_objid);
 352                         if (verbose)
 353                                 printf(DOSTID": ", ff_seq, ff_objid);
 354                 } else {
 355                         if (verbose)
 356                                 printf(DFID": ", PFID(&lma.lma_self_fid));
 357                         ff_seq = le64_to_cpu(lma.lma_self_fid.f_seq);
 358                         ff_objid = le32_to_cpu(lma.lma_self_fid.f_oid);
 359                 }
 360
 361                 sprintf(seq_name, (fid_seq_is_rsvd(ff_seq) ||
 362                                    fid_seq_is_mdt0(ff_seq)) ? LPU64 : LPX64i,
 363                         fid_seq_is_idif(ff_seq) ? 0 : ff_seq);
 364
 365                 /* LAST_ID uses OID = 0.  It will be regenerated later. */
 366                 if (ff_objid == 0) {
 367                         if (verbose)
 368                                 printf("'%s': LAST_ID\n", file_path);
 369                         continue;
 370                 }
 371
 372                 sprintf(obj_name, (fid_seq_is_rsvd(ff_seq) ||
 373                                    fid_seq_is_mdt0(ff_seq) ||
 374                                    fid_seq_is_idif(ff_seq)) ?
 375                                    LPU64 : LPX64i, ff_objid);
 376
 377                 grp_info = find_or_create_grp(&grp_info_list, ff_seq,
 378                                               mount_path);
 379                 if (grp_info == NULL) {
 380                         closedir(dir_ptr);
 381                         return 1;
 382                 }
 383
 384                 /* Might need to create the parent directory for this object */
 385                 if (ll_sprintf(dest_path, PATH_MAX, "%s/O/%s/d"LPU64,
 386                                 mount_path, seq_name, ff_objid % 32)) {
 387                         closedir(dir_ptr);
 388                         return 1;
 389                 }
 390
 391                 /* The O/{seq} directory was created in find_or_create_grp() */
 392                 ret = mkdir_p(dest_path, 0700);
 393                 if (ret < 0) {
 394                         closedir(dir_ptr);
 395                         return ret;
 396                 }
 397
 398                 if (ff_objid > grp_info->grp_last_id) {
 399                         fprintf(stderr, "error: file skipped because object ID "
 400                                 "greater than LAST_ID\nFilename: %s\n"
 401                                 "Group: "LPU64"\nObjectid: "LPU64"\n"
 402                                 "LAST_ID: "LPU64, file_path, ff_seq, ff_objid,
 403                                 grp_info->grp_last_id);
 404                         continue;
 405                 }
 406
 407                 /* move file from lost+found to proper object directory */
 408                 if (ll_sprintf(dest_path, PATH_MAX,
 409                                 "%s/O/%s/d"LPU64"/%s", mount_path,
 410                                 seq_name, ff_objid % 32, obj_name)) {
 411                         closedir(dir_ptr);
 412                         return 1;
 413                 }
 414
 415                 /* Source and destination are the same file, do nothing. */
 416                 if (strcmp(file_path, dest_path) == 0) {
 417                         if (verbose)
 418                                 printf("'%s': OK\n", file_path);
 419                         continue;
 420                 }
 421
 422                 obj_exists = 1;
 423                 ret = stat(dest_path, &st);
 424                 if (ret == 0) {
 425                         if (st.st_size == 0)
 426                                 obj_exists = 0;
 427                 } else {
 428                         if (errno != ENOENT)
 429                                 fprintf(stderr, "warning: stat for %s: %s\n",
 430                                         dest_path, strerror(errno));
 431                         obj_exists = 0;
 432                 }
 433
 434                 if (obj_exists) {
 435                         fprintf(dry_run ? stdout : stderr,
 436                                 "%s: '%s' exists, will not replace with '%s'\n",
 437                                 dry_run ? "dry_run" : "error",
 438                                 dest_path, file_path);
 439                         continue;
 440                 }
 441                 if (dry_run) {
 442                         printf("dry_run: not renaming '%s' to '%s'\n",
 443                                file_path, dest_path);
 444                         continue;
 445                 }
 446                 if (rename(file_path, dest_path) < 0) {
 447                         fprintf(stderr, "error: rename failed for '%s': %s\n",
 448                                 file_path, strerror(errno));
 449                         error++;
 450                         continue;
 451                 }
 452
 453                 printf("object '%s' restored.\n", dest_path);
 454                 break;
 455                 }
 456         }
 457
 458         closedir(dir_ptr);
 459
 460         return error;
 461 }
 462
 463 /*
 464  * If LAST_ID file is not present in some group then restore it with the highest
 465  * object ID found in that group. By the time we come here all possible objects
 466  * have been restored.
 467  */
 468 static int check_last_id(const char *mount_path)
 469 {
 470         char lastid_path[PATH_MAX];
 471         char dirname[PATH_MAX], subdirname[PATH_MAX];
 472         DIR *groupdir, *subdir;
 473         struct stat st;
 474         struct dirent *dirent;
 475         __u64 group;
 476         __u64 max_objid;
 477         int fd;
 478         int ret;
 479
 480         for (group = 0; group < MAX_GROUPS; group++) {
 481                 max_objid = 0;
 482
 483                 if (ll_sprintf(dirname, PATH_MAX, "%s/O/"LPU64,
 484                                mount_path, group))
 485                         return 1;
 486                 if (ll_sprintf(lastid_path, PATH_MAX, "%s/LAST_ID", dirname))
 487                         return 1;
 488
 489                 if (stat(lastid_path, &st) == 0)
 490                         continue;
 491
 492                 groupdir = opendir(dirname);
 493                 if (groupdir == NULL) {
 494                         if (errno != ENOENT)
 495                                 fprintf(stderr, "error: opening %s: %s\n",
 496                                         dirname, strerror(errno));
 497                         continue;
 498                 }
 499
 500                 while ((dirent = readdir(groupdir)) != NULL) {
 501                         if (!strcmp(dirent->d_name, ".") ||
 502                             !strcmp(dirent->d_name, ".."))
 503                                 continue;
 504
 505                         if (ll_sprintf(subdirname, PATH_MAX, "%s/%s",
 506                                        dirname, dirent->d_name)) {
 507                                 closedir(groupdir);
 508                                 return 1;
 509                         }
 510                         subdir = opendir(subdirname);
 511                         if (subdir == NULL) {
 512                                 fprintf(stderr, "error: opening %s: %s\n",
 513                                         subdirname, strerror(errno));
 514                                 continue;
 515                         }
 516
 517                         while ((dirent = readdir(subdir)) != NULL) {
 518                                 __u64 objid;
 519                                 char *end;
 520
 521                                 if (!strcmp(dirent->d_name, ".") ||
 522                                     !strcmp(dirent->d_name, ".."))
 523                                         continue;
 524
 525                                 objid = strtoull(dirent->d_name, &end, 0);
 526                                 if (end == dirent->d_name || *end != 0) {
 527                                         fprintf(stderr, "error: unknown object"
 528                                                 "ID %s/%s\n", subdirname,
 529                                                 dirent->d_name);
 530                                         continue;
 531                                 }
 532                                 if (objid > max_objid)
 533                                        max_objid = objid;
 534                         }
 535                         closedir(subdir);
 536                 }
 537                 closedir(groupdir);
 538
 539                 if (dry_run) {
 540                         fprintf(stderr, "dry_run: not updating '%s' to "
 541                                 LPU64"\n", lastid_path, max_objid);
 542                         return 0;
 543                 }
 544                 fd = open(lastid_path, O_RDWR | O_CREAT, 0700);
 545                 if (fd < 0) {
 546                         fprintf(stderr, "error: open '%s' failed: %s\n",
 547                                 lastid_path, strerror(errno));
 548                         return 1;
 549                 }
 550
 551                 max_objid = cpu_to_le64(max_objid);
 552                 ret = write(fd, &max_objid, sizeof(__u64));
 553                 if (ret < sizeof(__u64)) {
 554                         fprintf(stderr, "error: write '%s' failed: %s\n",
 555                                 lastid_path, strerror(errno));
 556                         close(fd);
 557                         return 1;
 558                 }
 559
 560                 close(fd);
 561         }
 562
 563         return 0;
 564 }
 565
 566 int main(int argc, char **argv)
 567 {
 568         char *progname;
 569         char src_dir[PATH_MAX] = "";
 570         char mount_path[PATH_MAX];
 571         char tmp_path[PATH_MAX];
 572         int rc;
 573         int c;
 574
 575         progname = strrchr(argv[0], '/');
 576         if (progname++ == NULL)
 577                 progname = argv[0];
 578
 579         while ((c = getopt(argc, argv, "d:hnv")) != EOF) {
 580                 switch (c) {
 581                 case 'd':
 582                         if (chdir(optarg)) {
 583                                 fprintf(stderr, "error: chdir to %s: %s\n",
 584                                         optarg, strerror(errno));
 585                                 return 1;
 586                         }
 587                         if (getcwd(src_dir, PATH_MAX) == NULL) {
 588                                 fprintf(stderr,
 589                                         "error: getcwd of lost+found: %s\n",
 590                                         strerror(errno));
 591                                 return 1;
 592                         }
 593                         if (chdir("..")) {
 594                                 fprintf(stderr, "error: chdir to \"..\": %s\n",
 595                                         strerror(errno));
 596                                 return 1;
 597                         }
 598                         if (getcwd(mount_path, PATH_MAX) == NULL) {
 599                                 fprintf(stderr,
 600                                         "error: getcwd of mount point: %s\n",
 601                                         strerror(errno));
 602                                 return 1;
 603                         }
 604                         if (!strcmp(src_dir, mount_path)) {
 605                                 fprintf(stderr,
 606                                         "error: root directory is detected\n");
 607                                 return 1;
 608                         }
 609                         fprintf(stdout, "%s: %sscan directory path: %s\n",
 610                                 progname, dry_run ? "read_only " : "", src_dir);
 611                         break;
 612                 case 'n':
 613                         dry_run = true;
 614                         break;
 615                 case 'v':
 616                         verbose = true;
 617                         break;
 618                 case 'h':
 619                         usage(progname);
 620                 default:
 621                         fprintf(stderr, "%s: bad option '%c'\n",
 622                                 progname, c);
 623                         usage(progname);
 624                 }
 625         }
 626
 627         if (src_dir[0] == 0)
 628                 usage(progname);
 629
 630         /* Check if 'O' directory exists and create it if needed */
 631         if (ll_sprintf(tmp_path, PATH_MAX, "%s/O",  mount_path))
 632                 return 1;
 633
 634         rc = mkdir_p(tmp_path, 0700);
 635         if (rc == -1) {
 636                 fprintf(stderr, "error: creating objects directory %s:"
 637                         " %s\n", tmp_path, strerror(errno));
 638                 return 1;
 639         }
 640
 641         CFS_INIT_LIST_HEAD(&grp_info_list);
 642         rc = traverse_lost_found(src_dir, mount_path);
 643         if (rc) {
 644                 fprintf(stderr, "error: traversing lost+found looking for "
 645                         "orphan objects.\n");
 646                 goto grp_destory;
 647         }
 648
 649         rc = check_last_id(mount_path);
 650         if (rc)
 651                 fprintf(stderr, "error: while checking/restoring LAST_ID.\n");
 652
 653 grp_destory:
 654         grp_info_list_destroy(&grp_info_list);
 655
 656         printf("%s: scan finished: rc = %d\n", progname, rc);
 657         return rc;
 658 }