From f7b481d8194811b7137c7f702ddeeca76cb36b5f Mon Sep 17 00:00:00 2001 From: kalpak Date: Tue, 1 Jul 2008 05:36:42 +0000 Subject: [PATCH] b=15284 i=adilger i=johann OST crashes and subsequent e2fsck can lead to objects being moved to lost+found directory. Using the "ll_recover_lost_found_objs" tool, these objects can be retrieved and data can be salvaged by using the object ID saved in the fid. --- lustre/ChangeLog | 7 + lustre/include/lustre/lustre_idl.h | 8 - lustre/include/lustre/lustre_user.h | 13 + lustre/obdfilter/filter_internal.h | 6 - lustre/utils/Makefile.am | 7 +- lustre/utils/ll_recover_lost_found_objs.c | 451 ++++++++++++++++++++++++++++++ 6 files changed, 477 insertions(+), 15 deletions(-) create mode 100644 lustre/utils/ll_recover_lost_found_objs.c diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 25b64dc..0d142b5 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -213,6 +213,13 @@ Severity : enhancement Bugzilla : 13128 Description: add -gid, -group, -uid, -user options to lfs find +Severity : enhancement +Bugzilla : 15284 +Description: ll_recover_lost_found_objs - rename objects in lost+found to object ID +Details : OST crashes and subsequent e2fsck can lead to objects being moved + to lost+found directory. Using the "ll_recover_lost_found_objs" + tool, these objects can be retrieved and data can be salvaged + by using the object ID saved in the fid. ------------------------------------------------------------------------------- diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index a0e1a6b..84fe7d7 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -667,14 +667,6 @@ typedef enum { /* This FULL lock is useful to take on unlink sort of operations */ #define MDS_INODELOCK_FULL ((1<<(MDS_INODELOCK_MAXSHIFT+1))-1) -struct ll_fid { - __u64 id; /* holds object id */ - __u32 generation; /* holds object generation */ - - __u32 f_type; /* holds object type or stripe idx when passing it to - * OST for saving into EA. */ -}; - extern void lustre_swab_ll_fid (struct ll_fid *fid); #define MDS_STATUS_CONN 1 diff --git a/lustre/include/lustre/lustre_user.h b/lustre/include/lustre/lustre_user.h index 354127c..8a883e8 100644 --- a/lustre/include/lustre/lustre_user.h +++ b/lustre/include/lustre/lustre_user.h @@ -129,6 +129,19 @@ struct ll_recreate_obj { __u32 lrc_ost_idx; }; +struct ll_fid { + __u64 id; /* holds object id */ + __u32 generation; /* holds object generation */ + __u32 f_type; /* holds object type or stripe idx when passing it to + * OST for saving into EA. */ +}; + +struct filter_fid { + struct ll_fid ff_fid; /* ff_fid.f_type == file stripe number */ + __u64 ff_objid; + __u64 ff_group; +}; + struct obd_uuid { char uuid[40]; }; diff --git a/lustre/obdfilter/filter_internal.h b/lustre/obdfilter/filter_internal.h index 615d6ce..4620d0d 100644 --- a/lustre/obdfilter/filter_internal.h +++ b/lustre/obdfilter/filter_internal.h @@ -37,12 +37,6 @@ extern struct file_operations filter_per_nid_stats_fops; OBD_MD_FLSIZE | OBD_MD_FLBLOCKS | OBD_MD_FLBLKSZ|\ OBD_MD_FLATIME | OBD_MD_FLMTIME | OBD_MD_FLCTIME) -struct filter_fid { - struct ll_fid ff_fid; /* ff_fid.f_type == file stripe number */ - __u64 ff_objid; - __u64 ff_group; -}; - /* per-client-per-object persistent state (LRU) */ struct filter_mod_data { struct list_head fmd_list; /* linked to fed_mod_list */ diff --git a/lustre/utils/Makefile.am b/lustre/utils/Makefile.am index 068894e..99d893c 100644 --- a/lustre/utils/Makefile.am +++ b/lustre/utils/Makefile.am @@ -15,7 +15,8 @@ EXTRA_PROGRAMS = wirecheck # mount only finds helpers in /sbin rootsbin_PROGRAMS = mount.lustre sbin_PROGRAMS = mkfs.lustre tunefs.lustre lctl wiretest \ - l_getgroups llverfs llverdev llog_reader lr_reader ltrack_stats + l_getgroups llverfs llverdev llog_reader ll_recover_lost_found_objs \ + lr_reader ltrack_stats if LIBPTHREAD sbin_PROGRAMS += loadgen endif @@ -70,6 +71,10 @@ llog_reader_SOURCES = llog_reader.c llog_reader_LDADD := $(LIBPTLCTL) llog_reader_DEPENDENCIES := $(LIBPTLCTL) +ll_recover_lost_found_objs_SOURCES = ll_recover_lost_found_objs.c +ll_recover_lost_found_objs_LDADD := $(LIBPTLCTL) +ll_recover_lost_found_objs_DEPENDENCIES := $(LIBPTLCTL) + lr_reader_SOURCES = lr_reader.c mount_lustre_SOURCES = mount_lustre.c diff --git a/lustre/utils/ll_recover_lost_found_objs.c b/lustre/utils/ll_recover_lost_found_objs.c new file mode 100644 index 0000000..c42413d --- /dev/null +++ b/lustre/utils/ll_recover_lost_found_objs.c @@ -0,0 +1,451 @@ +/* + * Copyright (C) 2008 Sun Microssystems, Inc. + * Author: Rupesh Thakare + * Author: Kalpak Shah + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +/* + * Tool for recovering objects from lost+found that might result from a + * Lustre OST with a corrupted directory. Running e2fsck will fix the + * directory, but puts all of the objects into lost+found, where they are + * inaccessible to Lustre. + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define MAX_GROUPS 64 + +int verbose = 0; + +struct obd_group_info { + int dir_exists; +}; +struct obd_group_info grp_info[MAX_GROUPS]; + +void usage(char *progname) +{ + fprintf(stderr, "Usage: %s [-hv] -d lost+found_directory\n", progname); + fprintf(stderr, "You need to mount the corrupted OST filesystem and" + "provide the path for the lost+found directory as the -d " + "option, for example:\n" + "ll_recover_lost_found_objs -d /mnt/ost/lost+found\n"); + exit(1); +} + +int mkdir_p(char *dest_path, char *mount, __u64 ff_group) +{ + struct stat stat_buf; + char tmp_path[PATH_MAX]; + int retval; + mode_t mode = 0700; + + if (stat(dest_path, &stat_buf) == 0) + return 0; + + if (grp_info[ff_group].dir_exists == 0) { + sprintf(tmp_path, "%s/O/"LPU64, mount, ff_group); + if (stat(tmp_path, &stat_buf) != 0) { + retval = mkdir(tmp_path, 0700); + if (retval < 0) { + fprintf(stderr, "error: creating directory %s: " + "%s\n", tmp_path, strerror(errno)); + return 1; + } + grp_info[ff_group].dir_exists = 1; + } + } + + retval = mkdir(dest_path, mode); + if (retval < 0) + return 1; + + return 0; +} + +/* This is returning 0 for an error */ +__u64 read_last_id(char *file_path) +{ + __u64 last_id; + int fd; + int count; + + fd = open(file_path, O_RDONLY); + if (fd < 0) + return 0; + + count = read(fd, &last_id, sizeof(last_id)); + if (count < 0) { + fprintf(stderr, "error: reading file %s: %s\n", file_path, + strerror(errno)); + close(fd); + return 0; + } + if (count != sizeof(last_id)) { + fprintf(stderr, "error: Could not read full last_id from %s\n", + file_path); + close(fd); + return 0; + } + + close(fd); + return le64_to_cpu(last_id); +} + +static unsigned filetype_dir_table[] = { + [0]= DT_UNKNOWN, + [S_IFIFO]= DT_FIFO, + [S_IFCHR] = DT_CHR, + [S_IFDIR] = DT_DIR, + [S_IFBLK] = DT_BLK, + [S_IFREG] = DT_REG, + [S_IFLNK] = DT_LNK, + [S_IFSOCK]= DT_SOCK, +#if defined(DT_DOOR) && defined(S_IFDOOR) + [S_IFDOOR]= DT_DOOR, +#endif +}; + +static int traverse_lost_found(char *src_dir, char *mount_path) +{ + DIR *dir_ptr; + struct filter_fid trusted_fid; + struct dirent64 *dirent; + __u64 ff_group, ff_objid; + char file_path[PATH_MAX]; + char dest_path[PATH_MAX]; + char last_id_file[PATH_MAX]; + __u64 last_id[MAX_GROUPS] = {0}; + __u64 tmp_last_id; + struct stat st; + int obj_exists, xattr_len; + int len, ret = 0, error = 0; + + len = strlen(src_dir); + + dir_ptr = opendir(src_dir); + if (!dir_ptr) { + fprintf(stderr, "error: opening directory: %s\n", + strerror(errno)); + return errno; + } + + while ((dirent = readdir64(dir_ptr)) != NULL) { + if (!strcmp(dirent->d_name, ".") || + !strcmp(dirent->d_name, "..")) + continue; + + src_dir[len] = 0; + if ((len + dirent->d_reclen + 2) > PATH_MAX) { + fprintf(stderr, "error: %s: string buffer is too small", + __FUNCTION__); + break; + } + strcat(src_dir, "/"); + strcat(src_dir, dirent->d_name); + + if (dirent->d_type == DT_UNKNOWN) { + struct stat st; + + ret = stat(src_dir, &st); + if (ret == 0) + dirent->d_type = filetype_dir_table[st.st_mode & + S_IFMT]; + } + + switch(dirent->d_type) { + case DT_DIR: + ret = traverse_lost_found(src_dir, mount_path); + if (ret) + goto out; + break; + + case DT_REG: + sprintf(file_path, "%s", src_dir); + xattr_len = getxattr(file_path, "trusted.fid", (void *)&trusted_fid, + sizeof(trusted_fid)); + + if (xattr_len < 0 || xattr_len < sizeof(trusted_fid)) { + /* + * Its very much possible that we dont find fid + * on precreated files, LAST_ID + */ + continue; + } + + ff_group = le64_to_cpu(trusted_fid.ff_group); + ff_objid = le64_to_cpu(trusted_fid.ff_objid); + + if (ff_group >= MAX_GROUPS) { + fprintf(stderr, "error: invalid group "LPU64" likely" + "indicates a corrupt xattr for file %s.\n", + ff_group, file_path); + continue; + } + + /* might need to create the parent directories for this object */ + sprintf(dest_path, "%s/O/"LPU64"/d"LPU64, mount_path, ff_group, + ff_objid % 32); + + ret = mkdir_p(dest_path, mount_path, ff_group); + if (ret) { + fprintf(stderr, "error: creating directory %s : %s\n", + dest_path, strerror(errno)); + goto out; + } + + /* + * Object ID needs to be verified against last_id. + * LAST_ID file may not be present in the group directory + * due to corruption. In case of any error try to recover + * as many objects as possible by setting last_id to ~0ULL. + */ + if (last_id[ff_group] == 0) { + sprintf(last_id_file, "%s/O/"LPU64"/LAST_ID", + mount_path, ff_group); + tmp_last_id = read_last_id(last_id_file); + + if (tmp_last_id == 0) + tmp_last_id = ~0ULL; + last_id[ff_group] = tmp_last_id; + } + + if (ff_objid > last_id[ff_group]) { + fprintf(stderr, "error: file skipped because object ID " + "greater than LAST_ID\nFilename: %s\n" + "Group: "LPU64"\nObjectid: "LPU64"\n" + "LAST_ID: "LPU64, file_path, ff_group, ff_objid, + last_id[ff_group]); + continue; + } + + /* move file from lost+found to proper object directory */ + sprintf(dest_path, "%s/O/"LPU64"/d"LPU64"/"LPU64, mount_path, + ff_group, ff_objid % 32, ff_objid); + + obj_exists = 1; + ret = stat(dest_path, &st); + if (ret == 0) { + if (st.st_size == 0) + obj_exists = 0; + } else if (ret < 0 && errno == ENOENT) { + obj_exists = 0; + } + + if (obj_exists) { + fprintf(stderr, "error: target object %s already " + "exists and will not be replaced.\n",dest_path); + continue; + } + + if (rename(file_path, dest_path) < 0) { + fprintf(stderr, "error: rename failed for file %s: %s\n", + file_path, strerror(errno)); + error++; + continue; + } + + printf("Object %s restored.\n", dest_path); + break; + + case DT_UNKNOWN: + continue; + } + } +out: + if (dir_ptr) + closedir(dir_ptr); + + return error; +} + +/* + * If LAST_ID file is not present in some group then restore it with the highest + * object ID found in that group. By the time we come here all possible objects + * have been restored. + */ +int check_last_id(char *mount_path) +{ + char lastid_path[PATH_MAX]; + char dirname[PATH_MAX], subdirname[PATH_MAX]; + DIR *groupdir, *subdir; + struct stat st; + struct dirent *dirent; + unsigned long long group; + __u64 max_objid; + int fd; + int ret; + + for (group = 0; group < MAX_GROUPS; group++) { + max_objid = 0; + sprintf(dirname, "%s/O/"LPU64, mount_path, group); + + strcpy(lastid_path, dirname); + strcat(lastid_path, "/LAST_ID"); + if (stat(lastid_path, &st) == 0) + continue; + + groupdir = opendir(dirname); + if (groupdir == NULL) { + if (errno != ENOENT) + fprintf(stderr, "error: opening %s: %s\n", + dirname, strerror(errno)); + continue; + } + + while ((dirent = readdir(groupdir)) != NULL) { + if (!strcmp(dirent->d_name, ".") || + !strcmp(dirent->d_name, "..")) + continue; + + sprintf(subdirname, "%s/%s", dirname, dirent->d_name); + + subdir = opendir(subdirname); + if (subdir == NULL) { + fprintf(stderr, "error: opening %s: %s\n", + subdirname, strerror(errno)); + continue; + } + + while ((dirent = readdir(subdir)) != NULL) { + __u64 objid; + char *end; + + if (!strcmp(dirent->d_name, ".") || + !strcmp(dirent->d_name, "..")) + continue; + + objid = strtoull(dirent->d_name, &end, 0); + if (end == dirent->d_name || *end != 0) { + fprintf(stderr, "error: unknown object" + "ID %s/%s\n", subdirname, + dirent->d_name); + continue; + } + if (objid > max_objid) + max_objid = objid; + } + closedir(subdir); + } + closedir(groupdir); + + fd = open(lastid_path, O_RDWR | O_CREAT, 0700); + if (fd < 0) { + fprintf(stderr, "error: open \"%s\" failed: %s\n", + lastid_path, strerror(errno)); + close(fd); + return -errno; + } + + ret = write(fd, &max_objid, sizeof(__u64)); + if (ret < sizeof(__u64)) { + close(fd); + return errno; + } + + close(fd); + } + + return 0; +} + +int main(int argc, char **argv) +{ + char *progname; + char *src_dir = NULL, *last_dir = NULL; + struct stat stat_buf; + char tmp_path[PATH_MAX]; + char mount_path[PATH_MAX] = {0}; + char c; + int retval; + + progname = argv[0]; + + while ((c = getopt(argc, argv, "d:hv")) != EOF) { + switch (c) { + case 'd': + src_dir = optarg; + /* Trim last '/' */ + last_dir = strrchr(src_dir, '/'); + if (last_dir != strchr(src_dir, '/')) { + if (last_dir != NULL && (*(last_dir + 1) == '\0')) + *(last_dir) = '\0'; + } + fprintf(stdout, "\"lost+found\" directory path: %s\n", + src_dir); + break; + case 'v': + verbose = 1; + break; + case 'h': + usage(progname); + default: + fprintf(stderr, "%s: bad option '%c'\n", + progname, c); + usage(progname); + } + } + + if (src_dir == NULL) + usage(progname); + + last_dir = strrchr(src_dir, '/'); + if (last_dir == NULL) { + /* Current directory */ + strcpy(mount_path, src_dir); + strcat(mount_path, "/.."); + } else { + strncpy(mount_path, src_dir, (int)(last_dir - src_dir)); + } + + /* Check if 'O' directory exists and create it if needed */ + sprintf(tmp_path, "%s/O", mount_path); + if (stat(tmp_path, &stat_buf) != 0) { + retval = mkdir(tmp_path, 0700); + if (retval < 0) + fprintf(stderr, "error: creating objects directory %s:" + " %s\n", tmp_path, strerror(errno)); + return errno; + } + + memset(grp_info, 0, MAX_GROUPS * sizeof(struct obd_group_info)); + + retval = traverse_lost_found(src_dir, mount_path); + if (retval) { + fprintf(stderr, "error: traversing lost+found looking for " + "orphan objects.\n"); + return retval; + } + + retval = check_last_id(mount_path); + if (retval) + fprintf(stderr, "error: while checking/restoring LAST_ID.\n"); + + return retval; +} -- 1.8.3.1