From 6282cca3cc3787ca61fe166c3ef9b1e768f7a3f4 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Thu, 12 Apr 2012 15:31:35 -0600 Subject: [PATCH] filefrag: Lustre changes to filefrag FIEMAP handling Add support for multiple-device filesystems by defining a new fe_device field in the fiemap_extent structure. This allows printing the filesystem-relative or linux block device number associated with each extent of a file. If a single filesystem extent is mirrored to multiple block devices, the fe_device field can be used to disambiguate the multiple copies. If the "-l" (device-logical) option is given to filefrag, then all extents for a particular device of a file are returned before returning extents for the next device. This makes it easier to see if extent allocation within a single device is contiguous, instead of returning all of the blocks of a file interleaved in file-logical-offset order. filefrag: support PFL and FLR file Add FIEMAP_EXTENT_DATA_MIRROR support FLR file, which is returned from Lustre indicating that a new mirror will be processed, so we'd reset the fm_start and fe_logical. Change-Id: Ifa68d951ac68d9f141eae915e58c13d73833c8c9 Signed-off-by: Bobi Jam Reviewed-on: https://review.whamcloud.com/40765 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Wang Shilong Tested-by: Andreas Dilger LU-11848 filefrag: add -V option to print version Add '-V' to filefrag to print the installed version of the tool. If '-V' is used twice, print out the list of supported FIEMAP flags. This can be used to check if filefrag understands a specific feature. Change-Id: Ib126bdd70efa1775aef6db761f54e27a593ebbe5 Signed-off-by: Andreas Dilger Reviewed-on: https://review.whamcloud.com/40873 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Wang Shilong LU-11848 filefrag: rework port PFL and FLR file Lustre uses the high 16bits of fiemap_extent:fe_device to record the absolute stripe number is under process, so that next fiemap call continues from that stripe. Fixes: 0b6718f5e22d ("filefrag: support PFL and FLR file") Change-Id: I111bcb100307aebacbffed99109057accdc27f5b Signed-off-by: Bobi Jam Reviewed-on: https://review.whamcloud.com/40889 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger LU-11848 filefrag: further PFL and FLR fixes Revert some of the changes from the earlier patches to bring the filefrag functionality closer to the original behavior. The kernel is instead changed to return the device number with every extent. This allows the previous filefrag to work for PFL and FLR files without any changes by just copying the fe_device from the last returned extent to the next call. The only gap in old filefrag is that it prints the stripe number with OST index. Fixes: d99d208e73cd ("LU-11848 filefrag: rework port PFL and FLR file") Fixes: 0b6718f5e22d ("filefrag: support PFL and FLR file") Signed-off-by: Andreas Dilger Change-Id: Ie6a8d93ebceb1e70894b21f0dadc9655083ebbe5 Reviewed-on: https://review.whamcloud.com/40932 Tested-by: jenkins Reviewed-by: Bobi Jam Change-Id: Ifb40cc159ddc61d2296c494f0e899ac11bf88b60 Signed-off-by: Andreas Dilger --- lib/ext2fs/fiemap.h | 7 ++- misc/filefrag.8.in | 9 +++- misc/filefrag.c | 143 ++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 118 insertions(+), 41 deletions(-) diff --git a/lib/ext2fs/fiemap.h b/lib/ext2fs/fiemap.h index 33ab8fb..a157c41 100644 --- a/lib/ext2fs/fiemap.h +++ b/lib/ext2fs/fiemap.h @@ -19,7 +19,9 @@ struct fiemap_extent { __u64 fe_length; /* length in bytes for this extent */ __u64 fe_reserved64[2]; __u32 fe_flags; /* FIEMAP_EXTENT_* flags for this extent */ - __u32 fe_reserved[3]; + __u32 fe_device; /* device number (fs-specific if FIEMAP_EXTENT_NET) + * low 16bits are used */ + __u32 fe_reserved[2]; }; struct fiemap { @@ -62,6 +64,7 @@ struct fiemap { #define FIEMAP_FLAG_SYNC 0x00000001 /* sync file data before map */ #define FIEMAP_FLAG_XATTR 0x00000002 /* map extended attribute tree */ #define FIEMAP_FLAG_CACHE 0x00000004 /* request caching of the extents */ +#define FIEMAP_FLAG_DEVICE_ORDER 0x40000000 /* return device ordered mapping */ #define FIEMAP_FLAGS_COMPAT (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR) @@ -89,5 +92,7 @@ struct fiemap { #define EXT4_FIEMAP_EXTENT_HOLE 0x08000000 /* Entry in extent status cache for a hole*/ +/* Network filesystem flags - use a high bit, don't conflict with upstream */ +#define FIEMAP_EXTENT_NET 0x80000000 /* Data stored remotely. */ #endif /* _LINUX_FIEMAP_H */ diff --git a/misc/filefrag.8.in b/misc/filefrag.8.in index d15d363..3268fd7 100644 --- a/misc/filefrag.8.in +++ b/misc/filefrag.8.in @@ -8,7 +8,7 @@ filefrag \- report on file fragmentation .BI \-b blocksize ] [ -.B \-BeksvxX +.B \-BeklsvxXV ] [ .I files... @@ -54,6 +54,9 @@ supported on all kernels, and is only supported on ext4 file systems. .B \-k Use 1024\-byte blocksize for output (identical to '\-b 1024'). .TP +.B \-l +Extents are displayed in device-logical offset order. +.TP .B \-s Sync the file before requesting the mapping. .TP @@ -65,6 +68,10 @@ Display mapping of extended attributes. .TP .B \-X Display extent block numbers in hexadecimal format. +.TP +.B \-V +Print version number of program and library. If given twice, also +print the FIEMAP flags that are understood by the current version. .SH AUTHOR .B filefrag was written by Theodore Ts'o . diff --git a/misc/filefrag.c b/misc/filefrag.c index 1e43131..5deb4c1 100644 --- a/misc/filefrag.c +++ b/misc/filefrag.c @@ -51,21 +51,24 @@ extern int optind; #include #include #include +#include "../version.h" int verbose = 0; unsigned int blocksize; /* Use specified blocksize (default 1kB) */ int sync_file = 0; /* fsync file before getting the mapping */ int precache_file = 0; /* precache the file before getting the mapping */ int xattr_map = 0; /* get xattr mapping */ -int force_bmap; /* force use of FIBMAP instead of FIEMAP */ +int force_bmap; /* force use of FIBMAP instead of FIEMAP */ int force_extent; /* print output in extent format always */ int use_extent_cache; /* Use extent cache */ +int device_offset; /* extents report device-relative offsets */ int logical_width = 8; int physical_width = 10; const char *ext_fmt = "%4d: %*llu..%*llu: %*llu..%*llu: %6llu: %s\n"; const char *hex_fmt = "%4d: %*llx..%*llx: %*llx..%*llx: %6llx: %s\n"; -#define FILEFRAG_FIEMAP_FLAGS_COMPAT (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR) +#define FILEFRAG_FIEMAP_FLAGS_COMPAT (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR |\ + FIEMAP_FLAG_DEVICE_ORDER) #define FIBMAP _IO(0x00, 1) /* bmap access */ #define FIGETBSZ _IO(0x00, 2) /* get the block size used for bmap */ @@ -124,21 +127,53 @@ static void print_extent_header(void) { printf(" ext: %*s %*s length: %*s flags:\n", logical_width * 2 + 3, - "logical_offset:", + device_offset ? "device_logical:" : "logical_offset:", physical_width * 2 + 3, "physical_offset:", - physical_width + 1, - "expected:"); + device_offset ? 5 : physical_width + 1, + device_offset ? " dev:" : "expected:"); } static void print_flag(__u32 *flags, __u32 mask, char *buf, const char *name) { + char hex[sizeof(mask) * 2 + 4]; /* 2 chars/byte + 0x, + NUL */ + if ((*flags & mask) == 0) return; + if (name == NULL) { + sprintf(hex, "%#04x,", mask); + name = hex; + } strcat(buf, name); *flags &= ~mask; } +static void print_flags(__u32 fe_flags, char *flags, int print_unknown) +{ + __u32 mask; + + print_flag(&fe_flags, FIEMAP_EXTENT_LAST, flags, "last,"); + print_flag(&fe_flags, FIEMAP_EXTENT_UNKNOWN, flags, "unknown_loc,"); + print_flag(&fe_flags, FIEMAP_EXTENT_DELALLOC, flags, "delalloc,"); + print_flag(&fe_flags, FIEMAP_EXTENT_ENCODED, flags, "encoded,"); + print_flag(&fe_flags, FIEMAP_EXTENT_DATA_ENCRYPTED, flags,"encrypted,"); + print_flag(&fe_flags, FIEMAP_EXTENT_NOT_ALIGNED, flags, "not_aligned,"); + print_flag(&fe_flags, FIEMAP_EXTENT_DATA_INLINE, flags, "inline,"); + print_flag(&fe_flags, FIEMAP_EXTENT_DATA_TAIL, flags, "tail_packed,"); + print_flag(&fe_flags, FIEMAP_EXTENT_UNWRITTEN, flags, "unwritten,"); + print_flag(&fe_flags, FIEMAP_EXTENT_MERGED, flags, "merged,"); + print_flag(&fe_flags, FIEMAP_EXTENT_SHARED, flags, "shared,"); + print_flag(&fe_flags, EXT4_FIEMAP_EXTENT_HOLE, flags, "hole,"); + print_flag(&fe_flags, FIEMAP_EXTENT_NET, flags, "net,"); + + if (!print_unknown) + return; + + /* print any unknown flags as hex values */ + for (mask = 1; fe_flags != 0 && mask != 0; mask <<= 1) + print_flag(&fe_flags, mask, flags, NULL); +} + static void print_extent_info(struct fiemap_extent *fm_extent, int cur_ex, unsigned long long expected, int blk_shift, ext2fs_struct_stat *st) @@ -148,7 +183,6 @@ static void print_extent_info(struct fiemap_extent *fm_extent, int cur_ex, unsigned long long ext_len; unsigned long long ext_blks; unsigned long long ext_blks_phys; - __u32 fe_flags, mask; char flags[256] = ""; /* For inline data all offsets should be in bytes, not blocks */ @@ -164,36 +198,15 @@ static void print_extent_info(struct fiemap_extent *fm_extent, int cur_ex, physical_blk = fm_extent->fe_physical >> blk_shift; } - fe_flags = fm_extent->fe_flags; - if (expected && - !(fe_flags & FIEMAP_EXTENT_UNKNOWN) && - !(fe_flags & EXT4_FIEMAP_EXTENT_HOLE)) + if (device_offset) + sprintf(flags, "%04x: ", fm_extent->fe_device & 0xffff); + else if (expected && + !(fm_extent->fe_flags & FIEMAP_EXTENT_UNKNOWN) && + !(fm_extent->fe_flags & EXT4_FIEMAP_EXTENT_HOLE)) sprintf(flags, ext_fmt == hex_fmt ? "%*llx: " : "%*llu: ", physical_width, expected >> blk_shift); - else - sprintf(flags, "%.*s ", physical_width, " "); - print_flag(&fe_flags, FIEMAP_EXTENT_LAST, flags, "last,"); - print_flag(&fe_flags, FIEMAP_EXTENT_UNKNOWN, flags, "unknown_loc,"); - print_flag(&fe_flags, FIEMAP_EXTENT_DELALLOC, flags, "delalloc,"); - print_flag(&fe_flags, FIEMAP_EXTENT_ENCODED, flags, "encoded,"); - print_flag(&fe_flags, FIEMAP_EXTENT_DATA_ENCRYPTED, flags,"encrypted,"); - print_flag(&fe_flags, FIEMAP_EXTENT_NOT_ALIGNED, flags, "not_aligned,"); - print_flag(&fe_flags, FIEMAP_EXTENT_DATA_INLINE, flags, "inline,"); - print_flag(&fe_flags, FIEMAP_EXTENT_DATA_TAIL, flags, "tail_packed,"); - print_flag(&fe_flags, FIEMAP_EXTENT_UNWRITTEN, flags, "unwritten,"); - print_flag(&fe_flags, FIEMAP_EXTENT_MERGED, flags, "merged,"); - print_flag(&fe_flags, FIEMAP_EXTENT_SHARED, flags, "shared,"); - print_flag(&fe_flags, EXT4_FIEMAP_EXTENT_HOLE, flags, "hole,"); - /* print any unknown flags as hex values */ - for (mask = 1; fe_flags != 0 && mask != 0; mask <<= 1) { - char hex[sizeof(mask) * 2 + 4]; /* 2 chars/byte + 0x, + NUL */ - - if ((fe_flags & mask) == 0) - continue; - sprintf(hex, "%#04x,", mask); - print_flag(&fe_flags, mask, flags, hex); - } + print_flags(fm_extent->fe_flags, flags, 1); if (fm_extent->fe_logical + fm_extent->fe_length >= (unsigned long long) st->st_size) @@ -233,6 +246,7 @@ static int filefrag_fiemap(int fd, int blk_shift, int *num_extents, unsigned long cmd = FS_IOC_FIEMAP; int fiemap_header_printed = 0; int tot_extents = 0, n = 0; + int previous_device = -1; int last = 0; int rc; @@ -251,6 +265,12 @@ static int filefrag_fiemap(int fd, int blk_shift, int *num_extents, if (use_extent_cache) cmd = EXT4_IOC_GET_ES_CACHE; + if (device_offset) { + flags |= FIEMAP_FLAG_DEVICE_ORDER; + memset(fm_ext, 0, sizeof(struct fiemap_extent)); + } + +retry_wo_device_order: do { fiemap->fm_length = ~0ULL; fiemap->fm_flags = flags; @@ -265,6 +285,10 @@ static int filefrag_fiemap(int fd, int blk_shift, int *num_extents, "flags %x\n", fiemap->fm_flags); fiemap_incompat_printed = 1; + } else if (rc == EBADR && (fiemap->fm_flags & + FIEMAP_FLAG_DEVICE_ORDER)) { + flags &= ~FIEMAP_FLAG_DEVICE_ORDER; + goto retry_wo_device_order; } return rc; } @@ -283,9 +307,10 @@ static int filefrag_fiemap(int fd, int blk_shift, int *num_extents, fm_last.fe_length; expected = fm_last.fe_physical + fm_ext[i].fe_logical - fm_last.fe_logical; - if (fm_ext[i].fe_logical != 0 && - fm_ext[i].fe_physical != expected && - fm_ext[i].fe_physical != expected_dense) { + if ((fm_ext[i].fe_logical != 0 && + fm_ext[i].fe_physical != expected && + fm_ext[i].fe_physical != expected_dense) || + ((fm_ext[i].fe_device & 0xffff) != previous_device)) { tot_extents++; } else { expected = 0; @@ -299,10 +324,23 @@ static int filefrag_fiemap(int fd, int blk_shift, int *num_extents, last = 1; fm_last = fm_ext[i]; n++; + previous_device = fm_ext[i].fe_device & 0xffff; } - fiemap->fm_start = (fm_ext[i - 1].fe_logical + - fm_ext[i - 1].fe_length); + /* For DEVICE_ORDER mappings, if EXTENT_LAST not yet found then + * fm_start needs to be the same as it was for earlier ioctl. + * The first extent is used to pass the end offset and device + * of the last FIEMAP call. Otherwise, we ask for extents + * starting from where the last mapping ended. */ + if (flags & FIEMAP_FLAG_DEVICE_ORDER) { + fm_ext[0].fe_logical = fm_ext[i - 1].fe_logical + + fm_ext[i - 1].fe_length; + fm_ext[0].fe_device = fm_ext[i - 1].fe_device; + fiemap->fm_start = 0; + } else { + fiemap->fm_start = fm_ext[i - 1].fe_logical + + fm_ext[i - 1].fe_length; + } } while (last == 0); *num_extents = tot_extents; @@ -326,6 +364,7 @@ static int filefrag_fibmap(int fd, int blk_shift, int *num_extents, memset(&fm_ext, 0, sizeof(fm_ext)); memset(&fm_last, 0, sizeof(fm_last)); if (force_extent) { + fm_ext.fe_device = st->st_dev; fm_ext.fe_flags = FIEMAP_EXTENT_MERGED; } @@ -460,6 +499,13 @@ static int frag_report(const char *filename) is_ext2 = 1; } + /* Check if filesystem is Lustre. Always print in extent format + * with 1kB blocks, using the device-relative logical offsets. */ + if (fsinfo.f_type == LUSTRE_SUPER_MAGIC) { + device_offset = 1; + blocksize = blocksize ?: 1024; + } + if (is_ext2) { long cylgroups = div_ceil(fsinfo.f_blocks, blksize * 8); @@ -567,11 +613,13 @@ int main(int argc, char**argv) { char **cpp; int rc = 0, c; + int version = 0; - while ((c = getopt(argc, argv, "Bb::eEkPsvxX")) != EOF) { + while ((c = getopt(argc, argv, "Bb::eEkPlsvxXV")) != EOF) { switch (c) { case 'B': force_bmap++; + force_extent = 0; break; case 'b': if (optarg) { @@ -636,12 +684,18 @@ int main(int argc, char**argv) case 'P': precache_file++; break; + case 'l': + device_offset++; + break; case 's': sync_file++; break; case 'v': verbose++; break; + case 'V': + version++; + break; case 'x': xattr_map++; break; @@ -653,6 +707,17 @@ int main(int argc, char**argv) break; } } + if (version) { + /* Print version number and exit */ + printf("filefrag %s (%s)\n", E2FSPROGS_VERSION, + E2FSPROGS_DATE); + if (version + verbose > 1) { + char flags[256] = ""; + print_flags(0xffffffff, flags, 0); + printf("supported: %s\n", flags); + } + exit(0); + } if (optind == argc) usage(argv[0]); -- 1.8.3.1