misc/mk_hugefiles.c

   1 /*
   2  * mk_hugefiles.c -- create huge files
   3  */
   4
   5 #include "config.h"
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <strings.h>
   9 #include <fcntl.h>
  10 #include <ctype.h>
  11 #include <time.h>
  12 #ifdef __linux__
  13 #include <sys/utsname.h>
  14 #endif
  15 #ifdef HAVE_GETOPT_H
  16 #include <getopt.h>
  17 #else
  18 extern char *optarg;
  19 extern int optind;
  20 #endif
  21 #ifdef HAVE_UNISTD_H
  22 #include <unistd.h>
  23 #endif
  24 #ifdef HAVE_STDLIB_H
  25 #include <stdlib.h>
  26 #endif
  27 #ifdef HAVE_ERRNO_H
  28 #include <errno.h>
  29 #endif
  30 #ifdef HAVE_SYS_IOCTL_H
  31 #include <sys/ioctl.h>
  32 #endif
  33 #include <sys/types.h>
  34 #include <sys/stat.h>
  35 #ifdef HAVE_SYS_SYSMACROS_H
  36 #include <sys/sysmacros.h>
  37 #endif
  38 #include <libgen.h>
  39 #include <limits.h>
  40 #include <blkid/blkid.h>
  41
  42 #include "ext2fs/ext2_fs.h"
  43 #include "ext2fs/ext2fsP.h"
  44 #include "et/com_err.h"
  45 #include "uuid/uuid.h"
  46 #include "e2p/e2p.h"
  47 #include "ext2fs/ext2fs.h"
  48 #include "util.h"
  49 #include "support/profile.h"
  50 #include "support/prof_err.h"
  51 #include "support/nls-enable.h"
  52 #include "mke2fs.h"
  53
  54 static int uid;
  55 static int gid;
  56 static blk64_t num_blocks;
  57 static blk64_t num_slack;
  58 static unsigned long num_files;
  59 static blk64_t goal;
  60 static char *fn_prefix;
  61 static int idx_digits;
  62 static char *fn_buf;
  63 static char *fn_numbuf;
  64 int zero_hugefile = 1;
  65
  66 static blk64_t
  67 get_partition_start(const char *device_name EXT2FS_ATTR((unused)))
  68 {
  69 #ifdef __linux__
  70         unsigned long long start;
  71         char            path[128];
  72         struct stat     st;
  73         FILE            *f;
  74         int             n;
  75
  76         if ((stat(device_name, &st) < 0) || !S_ISBLK(st.st_mode))
  77                 return 0;
  78
  79         sprintf(path, "/sys/dev/block/%d:%d/start",
  80                 major(st.st_rdev), minor(st.st_rdev));
  81         f = fopen(path, "r");
  82         if (!f)
  83                 return 0;
  84         n = fscanf(f, "%llu", &start);
  85         fclose(f);
  86         return (n == 1) ? start : 0;
  87 #else
  88         return 0;
  89 #endif
  90 }
  91
  92 static errcode_t create_directory(ext2_filsys fs, char *dir,
  93                                   ext2_ino_t *ret_ino)
  94
  95 {
  96         struct ext2_inode       inode;
  97         ext2_ino_t              ino = EXT2_ROOT_INO;
  98         ext2_ino_t              newdir;
  99         errcode_t               retval = 0;
 100         char                    *fn, *cp, *next;
 101
 102         fn = malloc(strlen(dir) + 1);
 103         if (fn == NULL)
 104                 return ENOMEM;
 105
 106         strcpy(fn, dir);
 107         cp = fn;
 108         while(1) {
 109                 next = strchr(cp, '/');
 110                 if (next)
 111                         *next++ = 0;
 112                 if (*cp) {
 113                         retval = ext2fs_new_inode(fs, ino, LINUX_S_IFDIR,
 114                                                   NULL, &newdir);
 115                         if (retval)
 116                                 goto errout;
 117
 118                         retval = ext2fs_mkdir(fs, ino, newdir, cp);
 119                         if (retval)
 120                                 goto errout;
 121
 122                         ino = newdir;
 123                         retval = ext2fs_read_inode(fs, ino, &inode);
 124                         if (retval)
 125                                 goto errout;
 126
 127                         inode.i_uid = uid & 0xFFFF;
 128                         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 129                         inode.i_gid = gid & 0xFFFF;
 130                         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 131                         retval = ext2fs_write_inode(fs, ino, &inode);
 132                         if (retval)
 133                                 goto errout;
 134                 }
 135                 if (next == NULL || *next == '\0')
 136                         break;
 137                 cp = next;
 138         }
 139 errout:
 140         free(fn);
 141         if (retval == 0)
 142                 *ret_ino = ino;
 143         return retval;
 144 }
 145
 146 static errcode_t mk_hugefile(ext2_filsys fs, blk64_t num,
 147                              ext2_ino_t dir, unsigned long idx, ext2_ino_t *ino)
 148
 149 {
 150         errcode_t               retval;
 151         blk64_t                 lblk, bend = 0;
 152         __u64                   size;
 153         blk64_t                 left;
 154         blk64_t                 count = 0;
 155         struct ext2_inode       inode;
 156         ext2_extent_handle_t    handle;
 157
 158         retval = ext2fs_new_inode(fs, 0, LINUX_S_IFREG, NULL, ino);
 159         if (retval)
 160                 return retval;
 161
 162         memset(&inode, 0, sizeof(struct ext2_inode));
 163         inode.i_mode = LINUX_S_IFREG | (0666 & ~fs->umask);
 164         inode.i_links_count = 1;
 165         inode.i_uid = uid & 0xFFFF;
 166         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 167         inode.i_gid = gid & 0xFFFF;
 168         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 169
 170         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 171         if (retval)
 172                 return retval;
 173
 174         ext2fs_inode_alloc_stats2(fs, *ino, +1, 0);
 175
 176         retval = ext2fs_extent_open2(fs, *ino, &inode, &handle);
 177         if (retval)
 178                 return retval;
 179
 180         /*
 181          * We don't use ext2fs_fallocate() here because hugefiles are
 182          * designed to be physically contiguous (if the block group
 183          * descriptors are configured to be in a single block at the
 184          * beginning of the file system, by using the
 185          * packed_meta_blocks layout), with the extent tree blocks
 186          * allocated near the beginning of the file system.
 187          */
 188         lblk = 0;
 189         left = num ? num : 1;
 190         while (left) {
 191                 blk64_t pblk, end;
 192                 blk64_t n = left;
 193
 194                 retval =  ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 195                         goal, ext2fs_blocks_count(fs->super) - 1, &end);
 196                 if (retval)
 197                         goto errout;
 198                 goal = end;
 199
 200                 retval =  ext2fs_find_first_set_block_bitmap2(fs->block_map, goal,
 201                                ext2fs_blocks_count(fs->super) - 1, &bend);
 202                 if (retval == ENOENT) {
 203                         bend = ext2fs_blocks_count(fs->super);
 204                         if (num == 0)
 205                                 left = 0;
 206                 }
 207                 if (!num || bend - goal < left)
 208                         n = bend - goal;
 209                 pblk = goal;
 210                 if (num)
 211                         left -= n;
 212                 goal += n;
 213                 count += n;
 214                 ext2fs_block_alloc_stats_range(fs, pblk, n, +1);
 215
 216                 if (zero_hugefile) {
 217                         blk64_t ret_blk;
 218                         retval = ext2fs_zero_blocks2(fs, pblk, n,
 219                                                      &ret_blk, NULL);
 220
 221                         if (retval)
 222                                 com_err(program_name, retval,
 223                                         _("while zeroing block %llu "
 224                                           "for hugefile"),
 225                                         (unsigned long long) ret_blk);
 226                 }
 227
 228                 while (n) {
 229                         blk64_t l = n;
 230                         struct ext2fs_extent newextent;
 231
 232                         if (l > EXT_INIT_MAX_LEN)
 233                                 l = EXT_INIT_MAX_LEN;
 234
 235                         newextent.e_len = l;
 236                         newextent.e_pblk = pblk;
 237                         newextent.e_lblk = lblk;
 238                         newextent.e_flags = 0;
 239
 240                         retval = ext2fs_extent_insert(handle,
 241                                         EXT2_EXTENT_INSERT_AFTER, &newextent);
 242                         if (retval)
 243                                 return retval;
 244                         pblk += l;
 245                         lblk += l;
 246                         n -= l;
 247                 }
 248         }
 249
 250         retval = ext2fs_read_inode(fs, *ino, &inode);
 251         if (retval)
 252                 goto errout;
 253
 254         retval = ext2fs_iblk_add_blocks(fs, &inode,
 255                                         count / EXT2FS_CLUSTER_RATIO(fs));
 256         if (retval)
 257                 goto errout;
 258         size = (__u64) count * fs->blocksize;
 259         retval = ext2fs_inode_size_set(fs, &inode, size);
 260         if (retval)
 261                 goto errout;
 262
 263         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 264         if (retval)
 265                 goto errout;
 266
 267         if (idx_digits)
 268                 sprintf(fn_numbuf, "%0*lu", idx_digits, idx);
 269         else if (num_files > 1)
 270                 sprintf(fn_numbuf, "%lu", idx);
 271
 272         retval = ext2fs_link(fs, dir, fn_buf, *ino,
 273                              EXT2_FT_REG_FILE | EXT2FS_LINK_EXPAND);
 274         if (retval)
 275                 goto errout;
 276
 277 errout:
 278         if (handle)
 279                 ext2fs_extent_free(handle);
 280
 281         return retval;
 282 }
 283
 284 static blk64_t calc_overhead(ext2_filsys fs, blk64_t num)
 285 {
 286         blk64_t e_blocks, e_blocks2, e_blocks3, e_blocks4;
 287         int extents_per_block;
 288         int extents = (num + EXT_INIT_MAX_LEN - 1) / EXT_INIT_MAX_LEN;
 289
 290         if (extents <= 4)
 291                 return 0;
 292
 293         /*
 294          * This calculation is due to the fact that we are inefficient
 295          * in how handle extent splits when appending to the end of
 296          * the extent tree.  Sigh.  We should fix this so that we can
 297          * actually store 340 extents per 4k block, instead of only 170.
 298          */
 299         extents_per_block = ((fs->blocksize -
 300                               sizeof(struct ext3_extent_header)) /
 301                              sizeof(struct ext3_extent));
 302         extents_per_block = (extents_per_block/ 2) - 1;
 303
 304         e_blocks = (extents + extents_per_block - 1) / extents_per_block;
 305         e_blocks2 = (e_blocks + extents_per_block - 1) / extents_per_block;
 306         e_blocks3 = (e_blocks2 + extents_per_block - 1) / extents_per_block;
 307         e_blocks4 = (e_blocks3 + extents_per_block - 1) / extents_per_block;
 308         return (e_blocks + e_blocks2 + e_blocks3 + e_blocks4) *
 309                 EXT2FS_CLUSTER_RATIO(fs);
 310 }
 311
 312 /*
 313  * Find the place where we should start allocating blocks for the huge
 314  * files.  Leave <slack> free blocks at the beginning of the file
 315  * system for things like metadata blocks.
 316  */
 317 static blk64_t get_start_block(ext2_filsys fs, blk64_t slack)
 318 {
 319         errcode_t retval;
 320         blk64_t blk = fs->super->s_first_data_block, next;
 321         blk64_t last_blk = ext2fs_blocks_count(fs->super) - 1;
 322
 323         while (slack) {
 324                 retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 325                                                 blk, last_blk, &blk);
 326                 if (retval)
 327                         break;
 328
 329                 retval = ext2fs_find_first_set_block_bitmap2(fs->block_map,
 330                                                 blk, last_blk, &next);
 331                 if (retval)
 332                         next = last_blk;
 333
 334                 if (next - blk > slack) {
 335                         blk += slack;
 336                         break;
 337                 }
 338
 339                 slack -= (next - blk);
 340                 blk = next;
 341         }
 342         return blk;
 343 }
 344
 345 static blk64_t round_up_align(blk64_t b, unsigned long align,
 346                               blk64_t part_offset)
 347 {
 348         unsigned long m;
 349
 350         if (align == 0)
 351                 return b;
 352         part_offset = part_offset % align;
 353         m = (b + part_offset) % align;
 354         if (m)
 355                 b += align - m;
 356         return b;
 357 }
 358
 359 errcode_t mk_hugefiles(ext2_filsys fs, const char *device_name)
 360 {
 361         unsigned long   i;
 362         ext2_ino_t      dir;
 363         errcode_t       retval;
 364         blk64_t         fs_blocks, part_offset = 0;
 365         unsigned long   align;
 366         int             d, dsize;
 367         char            *t;
 368
 369         if (!get_bool_from_profile(fs_types, "make_hugefiles", 0))
 370                 return 0;
 371
 372         if (!ext2fs_has_feature_extents(fs->super))
 373                 return EXT2_ET_EXTENT_NOT_SUPPORTED;
 374
 375         uid = get_int_from_profile(fs_types, "hugefiles_uid", 0);
 376         gid = get_int_from_profile(fs_types, "hugefiles_gid", 0);
 377         fs->umask = get_int_from_profile(fs_types, "hugefiles_umask", 077);
 378         num_files = get_int_from_profile(fs_types, "num_hugefiles", 0);
 379         t = get_string_from_profile(fs_types, "hugefiles_slack", "1M");
 380         num_slack = parse_num_blocks2(t, fs->super->s_log_block_size);
 381         free(t);
 382         t = get_string_from_profile(fs_types, "hugefiles_size", "0");
 383         num_blocks = parse_num_blocks2(t, fs->super->s_log_block_size);
 384         free(t);
 385         t = get_string_from_profile(fs_types, "hugefiles_align", "0");
 386         align = parse_num_blocks2(t, fs->super->s_log_block_size);
 387         free(t);
 388         if (get_bool_from_profile(fs_types, "hugefiles_align_disk", 0)) {
 389                 part_offset = get_partition_start(device_name) /
 390                         (fs->blocksize / 512);
 391                 if (part_offset % EXT2FS_CLUSTER_RATIO(fs)) {
 392                         fprintf(stderr,
 393                                 _("Partition offset of %llu (%uk) blocks "
 394                                   "not compatible with cluster size %u.\n"),
 395                                 (unsigned long long) part_offset, fs->blocksize,
 396                                 EXT2_CLUSTER_SIZE(fs->super));
 397                         exit(1);
 398                 }
 399         }
 400         num_blocks = round_up_align(num_blocks, align, 0);
 401         zero_hugefile = get_bool_from_profile(fs_types, "zero_hugefiles",
 402                                               zero_hugefile);
 403
 404         t = get_string_from_profile(fs_types, "hugefiles_dir", "/");
 405         retval = create_directory(fs, t, &dir);
 406         free(t);
 407         if (retval)
 408                 return retval;
 409
 410         fn_prefix = get_string_from_profile(fs_types, "hugefiles_name",
 411                                             "hugefile");
 412         idx_digits = get_int_from_profile(fs_types, "hugefiles_digits", 5);
 413         d = ext2fs_log10_u32(num_files) + 1;
 414         if (idx_digits > d)
 415                 d = idx_digits;
 416         dsize = strlen(fn_prefix) + d + 16;
 417         fn_buf = malloc(dsize);
 418         if (!fn_buf) {
 419                 free(fn_prefix);
 420                 return ENOMEM;
 421         }
 422         strcpy(fn_buf, fn_prefix);
 423         fn_numbuf = fn_buf + strlen(fn_prefix);
 424         free(fn_prefix);
 425
 426         fs_blocks = ext2fs_free_blocks_count(fs->super);
 427         if (fs_blocks < num_slack + align)
 428                 return ENOSPC;
 429         fs_blocks -= num_slack + align;
 430         if (num_blocks && num_blocks > fs_blocks)
 431                 return ENOSPC;
 432         if (num_blocks == 0 && num_files == 0)
 433                 num_files = 1;
 434
 435         if (num_files == 0 && num_blocks) {
 436                 num_files = fs_blocks / num_blocks;
 437                 fs_blocks -= (num_files / 16) + 1;
 438                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 439                 num_files = fs_blocks / num_blocks;
 440         }
 441
 442         if (num_blocks == 0 && num_files > 1) {
 443                 num_blocks = fs_blocks / num_files;
 444                 fs_blocks -= (num_files / 16) + 1;
 445                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 446                 num_blocks = fs_blocks / num_files;
 447         }
 448
 449         num_slack += (calc_overhead(fs, num_blocks ? num_blocks : fs_blocks) *
 450                       num_files);
 451         num_slack += (num_files / 16) + 1; /* space for dir entries */
 452         goal = get_start_block(fs, num_slack);
 453         goal = round_up_align(goal, align, part_offset);
 454
 455         if ((num_blocks ? num_blocks : fs_blocks) >
 456             (0x80000000UL / fs->blocksize))
 457                 ext2fs_set_feature_large_file(fs->super);
 458
 459         if (!quiet) {
 460                 if (zero_hugefile && verbose)
 461                         printf("%s", _("Huge files will be zero'ed\n"));
 462                 printf(_("Creating %lu huge file(s) "), num_files);
 463                 if (num_blocks)
 464                         printf(_("with %llu blocks each"),
 465                                (unsigned long long) num_blocks);
 466                 fputs(": ", stdout);
 467         }
 468         for (i=0; i < num_files; i++) {
 469                 ext2_ino_t ino;
 470
 471                 retval = mk_hugefile(fs, num_blocks, dir, i, &ino);
 472                 if (retval) {
 473                         com_err(program_name, retval,
 474                                 _("while creating huge file %lu"), i);
 475                         goto errout;
 476                 }
 477         }
 478         if (!quiet)
 479                 fputs(_("done\n"), stdout);
 480
 481 errout:
 482         free(fn_buf);
 483         return retval;
 484 }