misc/mk_hugefiles.c

   1 /*
   2  * mk_hugefiles.c -- create huge files
   3  */
   4
   5 #include "config.h"
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <strings.h>
   9 #include <fcntl.h>
  10 #include <ctype.h>
  11 #include <time.h>
  12 #ifdef __linux__
  13 #include <sys/utsname.h>
  14 #endif
  15 #ifdef HAVE_GETOPT_H
  16 #include <getopt.h>
  17 #else
  18 extern char *optarg;
  19 extern int optind;
  20 #endif
  21 #ifdef HAVE_UNISTD_H
  22 #include <unistd.h>
  23 #endif
  24 #ifdef HAVE_STDLIB_H
  25 #include <stdlib.h>
  26 #endif
  27 #ifdef HAVE_ERRNO_H
  28 #include <errno.h>
  29 #endif
  30 #ifdef HAVE_SYS_IOCTL_H
  31 #include <sys/ioctl.h>
  32 #endif
  33 #include <sys/types.h>
  34 #include <sys/stat.h>
  35 #ifdef HAVE_SYS_SYSMACROS_H
  36 #include <sys/sysmacros.h>
  37 #endif
  38 #include <libgen.h>
  39 #include <limits.h>
  40 #include <blkid/blkid.h>
  41
  42 #include "ext2fs/ext2_fs.h"
  43 #include "ext2fs/ext2fsP.h"
  44 #include "et/com_err.h"
  45 #include "uuid/uuid.h"
  46 #include "e2p/e2p.h"
  47 #include "ext2fs/ext2fs.h"
  48 #include "util.h"
  49 #include "support/profile.h"
  50 #include "support/prof_err.h"
  51 #include "support/nls-enable.h"
  52 #include "mke2fs.h"
  53
  54 static int uid;
  55 static int gid;
  56 static blk64_t num_blocks;
  57 static blk64_t num_slack;
  58 static unsigned long num_files;
  59 static blk64_t goal;
  60 static char *fn_prefix;
  61 static int idx_digits;
  62 static char *fn_buf;
  63 static char *fn_numbuf;
  64 int zero_hugefile = 1;
  65
  66 static blk64_t
  67 get_partition_start(const char *device_name EXT2FS_ATTR((unused)))
  68 {
  69 #ifdef __linux__
  70         unsigned long long start;
  71         char            path[128];
  72         struct stat     st;
  73         FILE            *f;
  74         int             n;
  75
  76         if ((stat(device_name, &st) < 0) || !S_ISBLK(st.st_mode))
  77                 return 0;
  78
  79         sprintf(path, "/sys/dev/block/%d:%d/start",
  80                 major(st.st_rdev), minor(st.st_rdev));
  81         f = fopen(path, "r");
  82         if (!f)
  83                 return 0;
  84         n = fscanf(f, "%llu", &start);
  85         fclose(f);
  86         return (n == 1) ? start : 0;
  87 #else
  88         return 0;
  89 #endif
  90 }
  91
  92 static errcode_t create_directory(ext2_filsys fs, char *dir,
  93                                   ext2_ino_t *ret_ino)
  94
  95 {
  96         struct ext2_inode       inode;
  97         ext2_ino_t              ino = EXT2_ROOT_INO;
  98         ext2_ino_t              newdir;
  99         errcode_t               retval = 0;
 100         char                    *fn, *cp, *next;
 101
 102         fn = malloc(strlen(dir) + 1);
 103         if (fn == NULL)
 104                 return ENOMEM;
 105
 106         strcpy(fn, dir);
 107         cp = fn;
 108         while(1) {
 109                 next = strchr(cp, '/');
 110                 if (next)
 111                         *next++ = 0;
 112                 if (*cp) {
 113                         retval = ext2fs_new_inode(fs, ino, LINUX_S_IFDIR,
 114                                                   NULL, &newdir);
 115                         if (retval)
 116                                 goto errout;
 117
 118                         retval = ext2fs_mkdir(fs, ino, newdir, cp);
 119                         if (retval)
 120                                 goto errout;
 121
 122                         ino = newdir;
 123                         retval = ext2fs_read_inode(fs, ino, &inode);
 124                         if (retval)
 125                                 goto errout;
 126
 127                         inode.i_uid = uid & 0xFFFF;
 128                         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 129                         inode.i_gid = gid & 0xFFFF;
 130                         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 131                         retval = ext2fs_write_inode(fs, ino, &inode);
 132                         if (retval)
 133                                 goto errout;
 134                 }
 135                 if (next == NULL || *next == '\0')
 136                         break;
 137                 cp = next;
 138         }
 139 errout:
 140         free(fn);
 141         if (retval == 0)
 142                 *ret_ino = ino;
 143         return retval;
 144 }
 145
 146 static errcode_t mk_hugefile(ext2_filsys fs, blk64_t num,
 147                              ext2_ino_t dir, unsigned long idx, ext2_ino_t *ino)
 148
 149 {
 150         errcode_t               retval;
 151         blk64_t                 lblk, bend = 0;
 152         __u64                   size;
 153         blk64_t                 left;
 154         blk64_t                 count = 0;
 155         struct ext2_inode       inode;
 156         ext2_extent_handle_t    handle;
 157
 158         retval = ext2fs_new_inode(fs, 0, LINUX_S_IFREG, NULL, ino);
 159         if (retval)
 160                 return retval;
 161
 162         memset(&inode, 0, sizeof(struct ext2_inode));
 163         inode.i_mode = LINUX_S_IFREG | (0666 & ~fs->umask);
 164         inode.i_links_count = 1;
 165         inode.i_uid = uid & 0xFFFF;
 166         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 167         inode.i_gid = gid & 0xFFFF;
 168         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 169
 170         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 171         if (retval)
 172                 return retval;
 173
 174         ext2fs_inode_alloc_stats2(fs, *ino, +1, 0);
 175
 176         retval = ext2fs_extent_open2(fs, *ino, &inode, &handle);
 177         if (retval)
 178                 return retval;
 179
 180         /*
 181          * We don't use ext2fs_fallocate() here because hugefiles are
 182          * designed to be physically contiguous (if the block group
 183          * descriptors are configured to be in a single block at the
 184          * beginning of the file system, by using the
 185          * packed_meta_blocks layout), with the extent tree blocks
 186          * allocated near the beginning of the file system.
 187          */
 188         lblk = 0;
 189         left = num ? num : 1;
 190         while (left) {
 191                 blk64_t pblk, end;
 192                 blk64_t n = left;
 193
 194                 retval =  ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 195                         goal, ext2fs_blocks_count(fs->super) - 1, &end);
 196                 if (retval)
 197                         goto errout;
 198                 goal = end;
 199
 200                 retval =  ext2fs_find_first_set_block_bitmap2(fs->block_map, goal,
 201                                ext2fs_blocks_count(fs->super) - 1, &bend);
 202                 if (retval == ENOENT) {
 203                         bend = ext2fs_blocks_count(fs->super);
 204                         if (num == 0)
 205                                 left = 0;
 206                 }
 207                 if (!num || bend - goal < left)
 208                         n = bend - goal;
 209                 pblk = goal;
 210                 if (num)
 211                         left -= n;
 212                 goal += n;
 213                 count += n;
 214                 ext2fs_block_alloc_stats_range(fs, pblk, n, +1);
 215
 216                 if (zero_hugefile) {
 217                         blk64_t ret_blk;
 218                         retval = ext2fs_zero_blocks2(fs, pblk, n,
 219                                                      &ret_blk, NULL);
 220
 221                         if (retval)
 222                                 com_err(program_name, retval,
 223                                         _("while zeroing block %llu "
 224                                           "for hugefile"),
 225                                         (unsigned long long) ret_blk);
 226                 }
 227
 228                 while (n) {
 229                         blk64_t l = n;
 230                         struct ext2fs_extent newextent;
 231
 232                         if (l > EXT_INIT_MAX_LEN)
 233                                 l = EXT_INIT_MAX_LEN;
 234
 235                         newextent.e_len = l;
 236                         newextent.e_pblk = pblk;
 237                         newextent.e_lblk = lblk;
 238                         newextent.e_flags = 0;
 239
 240                         retval = ext2fs_extent_insert(handle,
 241                                         EXT2_EXTENT_INSERT_AFTER, &newextent);
 242                         if (retval)
 243                                 return retval;
 244                         pblk += l;
 245                         lblk += l;
 246                         n -= l;
 247                 }
 248         }
 249
 250         retval = ext2fs_read_inode(fs, *ino, &inode);
 251         if (retval)
 252                 goto errout;
 253
 254         retval = ext2fs_iblk_add_blocks(fs, &inode,
 255                                         count / EXT2FS_CLUSTER_RATIO(fs));
 256         if (retval)
 257                 goto errout;
 258         size = (__u64) count * fs->blocksize;
 259         retval = ext2fs_inode_size_set(fs, &inode, size);
 260         if (retval)
 261                 goto errout;
 262
 263         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 264         if (retval)
 265                 goto errout;
 266
 267         if (idx_digits)
 268                 sprintf(fn_numbuf, "%0*lu", idx_digits, idx);
 269         else if (num_files > 1)
 270                 sprintf(fn_numbuf, "%lu", idx);
 271
 272 retry:
 273         retval = ext2fs_link(fs, dir, fn_buf, *ino, EXT2_FT_REG_FILE);
 274         if (retval == EXT2_ET_DIR_NO_SPACE) {
 275                 retval = ext2fs_expand_dir(fs, dir);
 276                 if (retval)
 277                         goto errout;
 278                 goto retry;
 279         }
 280
 281         if (retval)
 282                 goto errout;
 283
 284 errout:
 285         if (handle)
 286                 ext2fs_extent_free(handle);
 287
 288         return retval;
 289 }
 290
 291 static blk64_t calc_overhead(ext2_filsys fs, blk64_t num)
 292 {
 293         blk64_t e_blocks, e_blocks2, e_blocks3, e_blocks4;
 294         int extents_per_block;
 295         int extents = (num + EXT_INIT_MAX_LEN - 1) / EXT_INIT_MAX_LEN;
 296
 297         if (extents <= 4)
 298                 return 0;
 299
 300         /*
 301          * This calculation is due to the fact that we are inefficient
 302          * in how handle extent splits when appending to the end of
 303          * the extent tree.  Sigh.  We should fix this so that we can
 304          * actually store 340 extents per 4k block, instead of only 170.
 305          */
 306         extents_per_block = ((fs->blocksize -
 307                               sizeof(struct ext3_extent_header)) /
 308                              sizeof(struct ext3_extent));
 309         extents_per_block = (extents_per_block/ 2) - 1;
 310
 311         e_blocks = (extents + extents_per_block - 1) / extents_per_block;
 312         e_blocks2 = (e_blocks + extents_per_block - 1) / extents_per_block;
 313         e_blocks3 = (e_blocks2 + extents_per_block - 1) / extents_per_block;
 314         e_blocks4 = (e_blocks3 + extents_per_block - 1) / extents_per_block;
 315         return (e_blocks + e_blocks2 + e_blocks3 + e_blocks4) *
 316                 EXT2FS_CLUSTER_RATIO(fs);
 317 }
 318
 319 /*
 320  * Find the place where we should start allocating blocks for the huge
 321  * files.  Leave <slack> free blocks at the beginning of the file
 322  * system for things like metadata blocks.
 323  */
 324 static blk64_t get_start_block(ext2_filsys fs, blk64_t slack)
 325 {
 326         errcode_t retval;
 327         blk64_t blk = fs->super->s_first_data_block, next;
 328         blk64_t last_blk = ext2fs_blocks_count(fs->super) - 1;
 329
 330         while (slack) {
 331                 retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 332                                                 blk, last_blk, &blk);
 333                 if (retval)
 334                         break;
 335
 336                 retval = ext2fs_find_first_set_block_bitmap2(fs->block_map,
 337                                                 blk, last_blk, &next);
 338                 if (retval)
 339                         next = last_blk;
 340
 341                 if (next - blk > slack) {
 342                         blk += slack;
 343                         break;
 344                 }
 345
 346                 slack -= (next - blk);
 347                 blk = next;
 348         }
 349         return blk;
 350 }
 351
 352 static blk64_t round_up_align(blk64_t b, unsigned long align,
 353                               blk64_t part_offset)
 354 {
 355         unsigned long m;
 356
 357         if (align == 0)
 358                 return b;
 359         part_offset = part_offset % align;
 360         m = (b + part_offset) % align;
 361         if (m)
 362                 b += align - m;
 363         return b;
 364 }
 365
 366 errcode_t mk_hugefiles(ext2_filsys fs, const char *device_name)
 367 {
 368         unsigned long   i;
 369         ext2_ino_t      dir;
 370         errcode_t       retval;
 371         blk64_t         fs_blocks, part_offset = 0;
 372         unsigned long   align;
 373         int             d, dsize;
 374         char            *t;
 375
 376         if (!get_bool_from_profile(fs_types, "make_hugefiles", 0))
 377                 return 0;
 378
 379         if (!ext2fs_has_feature_extents(fs->super))
 380                 return EXT2_ET_EXTENT_NOT_SUPPORTED;
 381
 382         uid = get_int_from_profile(fs_types, "hugefiles_uid", 0);
 383         gid = get_int_from_profile(fs_types, "hugefiles_gid", 0);
 384         fs->umask = get_int_from_profile(fs_types, "hugefiles_umask", 077);
 385         num_files = get_int_from_profile(fs_types, "num_hugefiles", 0);
 386         t = get_string_from_profile(fs_types, "hugefiles_slack", "1M");
 387         num_slack = parse_num_blocks2(t, fs->super->s_log_block_size);
 388         free(t);
 389         t = get_string_from_profile(fs_types, "hugefiles_size", "0");
 390         num_blocks = parse_num_blocks2(t, fs->super->s_log_block_size);
 391         free(t);
 392         t = get_string_from_profile(fs_types, "hugefiles_align", "0");
 393         align = parse_num_blocks2(t, fs->super->s_log_block_size);
 394         free(t);
 395         if (get_bool_from_profile(fs_types, "hugefiles_align_disk", 0)) {
 396                 part_offset = get_partition_start(device_name) /
 397                         (fs->blocksize / 512);
 398                 if (part_offset % EXT2FS_CLUSTER_RATIO(fs)) {
 399                         fprintf(stderr,
 400                                 _("Partition offset of %llu (%uk) blocks "
 401                                   "not compatible with cluster size %u.\n"),
 402                                 (unsigned long long) part_offset, fs->blocksize,
 403                                 EXT2_CLUSTER_SIZE(fs->super));
 404                         exit(1);
 405                 }
 406         }
 407         num_blocks = round_up_align(num_blocks, align, 0);
 408         zero_hugefile = get_bool_from_profile(fs_types, "zero_hugefiles",
 409                                               zero_hugefile);
 410
 411         t = get_string_from_profile(fs_types, "hugefiles_dir", "/");
 412         retval = create_directory(fs, t, &dir);
 413         free(t);
 414         if (retval)
 415                 return retval;
 416
 417         fn_prefix = get_string_from_profile(fs_types, "hugefiles_name",
 418                                             "hugefile");
 419         idx_digits = get_int_from_profile(fs_types, "hugefiles_digits", 5);
 420         d = int_log10(num_files) + 1;
 421         if (idx_digits > d)
 422                 d = idx_digits;
 423         dsize = strlen(fn_prefix) + d + 16;
 424         fn_buf = malloc(dsize);
 425         if (!fn_buf) {
 426                 free(fn_prefix);
 427                 return ENOMEM;
 428         }
 429         strcpy(fn_buf, fn_prefix);
 430         fn_numbuf = fn_buf + strlen(fn_prefix);
 431         free(fn_prefix);
 432
 433         fs_blocks = ext2fs_free_blocks_count(fs->super);
 434         if (fs_blocks < num_slack + align)
 435                 return ENOSPC;
 436         fs_blocks -= num_slack + align;
 437         if (num_blocks && num_blocks > fs_blocks)
 438                 return ENOSPC;
 439         if (num_blocks == 0 && num_files == 0)
 440                 num_files = 1;
 441
 442         if (num_files == 0 && num_blocks) {
 443                 num_files = fs_blocks / num_blocks;
 444                 fs_blocks -= (num_files / 16) + 1;
 445                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 446                 num_files = fs_blocks / num_blocks;
 447         }
 448
 449         if (num_blocks == 0 && num_files > 1) {
 450                 num_blocks = fs_blocks / num_files;
 451                 fs_blocks -= (num_files / 16) + 1;
 452                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 453                 num_blocks = fs_blocks / num_files;
 454         }
 455
 456         num_slack += (calc_overhead(fs, num_blocks ? num_blocks : fs_blocks) *
 457                       num_files);
 458         num_slack += (num_files / 16) + 1; /* space for dir entries */
 459         goal = get_start_block(fs, num_slack);
 460         goal = round_up_align(goal, align, part_offset);
 461
 462         if ((num_blocks ? num_blocks : fs_blocks) >
 463             (0x80000000UL / fs->blocksize))
 464                 ext2fs_set_feature_large_file(fs->super);
 465
 466         if (!quiet) {
 467                 if (zero_hugefile && verbose)
 468                         printf("%s", _("Huge files will be zero'ed\n"));
 469                 printf(_("Creating %lu huge file(s) "), num_files);
 470                 if (num_blocks)
 471                         printf(_("with %llu blocks each"),
 472                                (unsigned long long) num_blocks);
 473                 fputs(": ", stdout);
 474         }
 475         for (i=0; i < num_files; i++) {
 476                 ext2_ino_t ino;
 477
 478                 retval = mk_hugefile(fs, num_blocks, dir, i, &ino);
 479                 if (retval) {
 480                         com_err(program_name, retval,
 481                                 _("while creating huge file %lu"), i);
 482                         goto errout;
 483                 }
 484         }
 485         if (!quiet)
 486                 fputs(_("done\n"), stdout);
 487
 488 errout:
 489         free(fn_buf);
 490         return retval;
 491 }