misc/mk_hugefiles.c

   1 /*
   2  * mk_hugefiles.c -- create huge files
   3  */
   4
   5 #define _XOPEN_SOURCE 600 /* for inclusion of PATH_MAX in Solaris */
   6 #define _BSD_SOURCE       /* for makedev() and major() */
   7
   8 #include "config.h"
   9 #include <stdio.h>
  10 #include <stdarg.h>
  11 #include <string.h>
  12 #include <strings.h>
  13 #include <fcntl.h>
  14 #include <ctype.h>
  15 #include <time.h>
  16 #ifdef __linux__
  17 #include <sys/utsname.h>
  18 #endif
  19 #ifdef HAVE_GETOPT_H
  20 #include <getopt.h>
  21 #else
  22 extern char *optarg;
  23 extern int optind;
  24 #endif
  25 #ifdef HAVE_UNISTD_H
  26 #include <unistd.h>
  27 #endif
  28 #ifdef HAVE_STDLIB_H
  29 #include <stdlib.h>
  30 #endif
  31 #ifdef HAVE_ERRNO_H
  32 #include <errno.h>
  33 #endif
  34 #include <sys/ioctl.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <libgen.h>
  38 #include <limits.h>
  39 #include <blkid/blkid.h>
  40
  41 #include "ext2fs/ext2_fs.h"
  42 #include "ext2fs/ext2fsP.h"
  43 #include "et/com_err.h"
  44 #include "uuid/uuid.h"
  45 #include "e2p/e2p.h"
  46 #include "ext2fs/ext2fs.h"
  47 #include "util.h"
  48 #include "profile.h"
  49 #include "prof_err.h"
  50 #include "nls-enable.h"
  51 #include "mke2fs.h"
  52
  53 static int uid;
  54 static int gid;
  55 static blk64_t num_blocks;
  56 static blk64_t num_slack;
  57 static unsigned long num_files;
  58 static blk64_t goal;
  59 static char *fn_prefix;
  60 static int idx_digits;
  61 static char *fn_buf;
  62 static char *fn_numbuf;
  63 int zero_hugefile = 1;
  64
  65 #define SYSFS_PATH_LEN 256
  66 typedef char sysfs_path_t[SYSFS_PATH_LEN];
  67
  68 #ifndef HAVE_SNPRINTF
  69 /*
  70  * We are very careful to avoid needing to worry about buffer
  71  * overflows, so we don't really need to use snprintf() except as an
  72  * additional safety check.  So if snprintf() is not present, it's
  73  * safe to fall back to vsprintf().  This provides portability since
  74  * vsprintf() is guaranteed by C89, while snprintf() is only
  75  * guaranteed by C99 --- which for example, Microsoft Visual Studio
  76  * has *still* not bothered to implement.  :-/  (Not that I expect
  77  * mke2fs to be ported to MS Visual Studio any time soon, but
  78  * libext2fs *does* get built on Microsoft platforms, and we might
  79  * want to move this into libext2fs some day.)
  80  */
  81 static int my_snprintf(char *str, size_t size, const char *format, ...)
  82 {
  83         va_list ap;
  84         int ret;
  85
  86         va_start(ap, format);
  87         ret = vsprintf(str, format, ap);
  88         va_end(ap);
  89         return ret;
  90 }
  91
  92 #define snprintf my_snprintf
  93 #endif
  94
  95 /*
  96  * Fall back to Linux's definitions of makedev and major are needed.
  97  * The search_sysfs_block() function is highly unlikely to work on
  98  * non-Linux systems anyway.
  99  */
 100 #ifndef makedev
 101 #define makedev(maj, min) (((maj) << 8) + (min))
 102 #endif
 103
 104 static char *search_sysfs_block(dev_t devno, sysfs_path_t ret_path)
 105 {
 106         struct dirent   *de, *p_de;
 107         DIR             *dir = NULL, *p_dir = NULL;
 108         FILE            *f;
 109         sysfs_path_t    path, p_path;
 110         unsigned int    major, minor;
 111         char            *ret = ret_path;
 112
 113         ret_path[0] = 0;
 114         if ((dir = opendir("/sys/block")) == NULL)
 115                 return NULL;
 116         while ((de = readdir(dir)) != NULL) {
 117                 if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..") ||
 118                     strlen(de->d_name) > sizeof(path)-32)
 119                         continue;
 120                 snprintf(path, SYSFS_PATH_LEN,
 121                          "/sys/block/%s/dev", de->d_name);
 122                 f = fopen(path, "r");
 123                 if (f &&
 124                     (fscanf(f, "%u:%u", &major, &minor) == 2)) {
 125                         fclose(f); f = NULL;
 126                         if (makedev(major, minor) == devno) {
 127                                 snprintf(ret_path, SYSFS_PATH_LEN,
 128                                          "/sys/block/%s", de->d_name);
 129                                 goto success;
 130                         }
 131 #ifdef major
 132                         if (major(devno) != major)
 133                                 continue;
 134 #endif
 135                 }
 136                 if (f)
 137                         fclose(f);
 138
 139                 snprintf(path, SYSFS_PATH_LEN, "/sys/block/%s", de->d_name);
 140
 141                 if (p_dir)
 142                         closedir(p_dir);
 143                 if ((p_dir = opendir(path)) == NULL)
 144                         continue;
 145                 while ((p_de = readdir(p_dir)) != NULL) {
 146                         if (!strcmp(p_de->d_name, ".") ||
 147                             !strcmp(p_de->d_name, "..") ||
 148                             (strlen(p_de->d_name) >
 149                              SYSFS_PATH_LEN - strlen(path) - 32))
 150                                 continue;
 151                         snprintf(p_path, SYSFS_PATH_LEN, "%s/%s/dev",
 152                                  path, p_de->d_name);
 153
 154                         f = fopen(p_path, "r");
 155                         if (f &&
 156                             (fscanf(f, "%u:%u", &major, &minor) == 2) &&
 157                             (((major << 8) + minor) == devno)) {
 158                                 fclose(f);
 159                                 snprintf(ret_path, SYSFS_PATH_LEN, "%s/%s",
 160                                          path, p_de->d_name);
 161                                 goto success;
 162                         }
 163                         if (f)
 164                                 fclose(f);
 165                 }
 166         }
 167         ret = NULL;
 168 success:
 169         if (dir)
 170                 closedir(dir);
 171         if (p_dir)
 172                 closedir(p_dir);
 173         return ret;
 174 }
 175
 176 static blk64_t get_partition_start(const char *device_name)
 177 {
 178         unsigned long long start;
 179         sysfs_path_t    path;
 180         struct stat     st;
 181         FILE            *f;
 182         char            *cp;
 183         int             n;
 184
 185         if ((stat(device_name, &st) < 0) || !S_ISBLK(st.st_mode))
 186                 return 0;
 187
 188         cp = search_sysfs_block(st.st_rdev, path);
 189         if (!cp)
 190                 return 0;
 191         if (strlen(path) > SYSFS_PATH_LEN - sizeof("/start"))
 192                 return 0;
 193         strcat(path, "/start");
 194         f = fopen(path, "r");
 195         if (!f)
 196                 return 0;
 197         n = fscanf(f, "%llu", &start);
 198         fclose(f);
 199         return (n == 1) ? start : 0;
 200 }
 201
 202 static errcode_t create_directory(ext2_filsys fs, char *dir,
 203                                   ext2_ino_t *ret_ino)
 204
 205 {
 206         struct ext2_inode       inode;
 207         ext2_ino_t              ino = EXT2_ROOT_INO;
 208         ext2_ino_t              newdir;
 209         errcode_t               retval = 0;
 210         char                    *fn, *cp, *next;
 211
 212         fn = malloc(strlen(dir) + 1);
 213         if (fn == NULL)
 214                 return ENOMEM;
 215
 216         strcpy(fn, dir);
 217         cp = fn;
 218         while(1) {
 219                 next = strchr(cp, '/');
 220                 if (next)
 221                         *next++ = 0;
 222                 if (*cp) {
 223                         retval = ext2fs_new_inode(fs, ino, LINUX_S_IFDIR,
 224                                                   NULL, &newdir);
 225                         if (retval)
 226                                 goto errout;
 227
 228                         retval = ext2fs_mkdir(fs, ino, newdir, cp);
 229                         if (retval)
 230                                 goto errout;
 231
 232                         ino = newdir;
 233                         retval = ext2fs_read_inode(fs, ino, &inode);
 234                         if (retval)
 235                                 goto errout;
 236
 237                         inode.i_uid = uid & 0xFFFF;
 238                         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 239                         inode.i_gid = gid & 0xFFFF;
 240                         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 241                         retval = ext2fs_write_inode(fs, ino, &inode);
 242                         if (retval)
 243                                 goto errout;
 244                 }
 245                 if (next == NULL || *next == '\0')
 246                         break;
 247                 cp = next;
 248         }
 249 errout:
 250         free(fn);
 251         if (retval == 0)
 252                 *ret_ino = ino;
 253         return retval;
 254 }
 255
 256 static errcode_t mk_hugefile(ext2_filsys fs, blk64_t num,
 257                              ext2_ino_t dir, unsigned long idx, ext2_ino_t *ino)
 258
 259 {
 260         errcode_t               retval;
 261         blk64_t                 lblk, bend = 0;
 262         __u64                   size;
 263         blk64_t                 left;
 264         blk64_t                 count = 0;
 265         struct ext2_inode       inode;
 266         ext2_extent_handle_t    handle;
 267
 268         retval = ext2fs_new_inode(fs, 0, LINUX_S_IFREG, NULL, ino);
 269         if (retval)
 270                 return retval;
 271
 272         memset(&inode, 0, sizeof(struct ext2_inode));
 273         inode.i_mode = LINUX_S_IFREG | (0666 & ~fs->umask);
 274         inode.i_links_count = 1;
 275         inode.i_uid = uid & 0xFFFF;
 276         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 277         inode.i_gid = gid & 0xFFFF;
 278         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 279
 280         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 281         if (retval)
 282                 return retval;
 283
 284         ext2fs_inode_alloc_stats2(fs, *ino, +1, 0);
 285
 286         retval = ext2fs_extent_open2(fs, *ino, &inode, &handle);
 287         if (retval)
 288                 return retval;
 289
 290         lblk = 0;
 291         left = num ? num : 1;
 292         while (left) {
 293                 blk64_t pblk, end;
 294                 blk64_t n = left;
 295
 296                 retval =  ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 297                         goal, ext2fs_blocks_count(fs->super) - 1, &end);
 298                 if (retval)
 299                         goto errout;
 300                 goal = end;
 301
 302                 retval =  ext2fs_find_first_set_block_bitmap2(fs->block_map, goal,
 303                                ext2fs_blocks_count(fs->super) - 1, &bend);
 304                 if (retval == ENOENT) {
 305                         bend = ext2fs_blocks_count(fs->super);
 306                         if (num == 0)
 307                                 left = 0;
 308                 }
 309                 if (!num || bend - goal < left)
 310                         n = bend - goal;
 311                 pblk = goal;
 312                 if (num)
 313                         left -= n;
 314                 goal += n;
 315                 count += n;
 316                 ext2fs_block_alloc_stats_range(fs, pblk, n, +1);
 317
 318                 if (zero_hugefile) {
 319                         blk64_t ret_blk;
 320                         retval = ext2fs_zero_blocks2(fs, pblk, n,
 321                                                      &ret_blk, NULL);
 322
 323                         if (retval)
 324                                 com_err(program_name, retval,
 325                                         _("while zeroing block %llu "
 326                                           "for hugefile"), ret_blk);
 327                 }
 328
 329                 while (n) {
 330                         blk64_t l = n;
 331                         struct ext2fs_extent newextent;
 332
 333                         if (l > EXT_INIT_MAX_LEN)
 334                                 l = EXT_INIT_MAX_LEN;
 335
 336                         newextent.e_len = l;
 337                         newextent.e_pblk = pblk;
 338                         newextent.e_lblk = lblk;
 339                         newextent.e_flags = 0;
 340
 341                         retval = ext2fs_extent_insert(handle,
 342                                         EXT2_EXTENT_INSERT_AFTER, &newextent);
 343                         if (retval)
 344                                 return retval;
 345                         pblk += l;
 346                         lblk += l;
 347                         n -= l;
 348                 }
 349         }
 350
 351         retval = ext2fs_read_inode(fs, *ino, &inode);
 352         if (retval)
 353                 goto errout;
 354
 355         retval = ext2fs_iblk_add_blocks(fs, &inode,
 356                                         count / EXT2FS_CLUSTER_RATIO(fs));
 357         if (retval)
 358                 goto errout;
 359         size = (__u64) count * fs->blocksize;
 360         inode.i_size = size & 0xffffffff;
 361         inode.i_size_high = (size >> 32);
 362
 363         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 364         if (retval)
 365                 goto errout;
 366
 367         if (idx_digits)
 368                 sprintf(fn_numbuf, "%0*lu", idx_digits, idx);
 369         else if (num_files > 1)
 370                 sprintf(fn_numbuf, "%lu", idx);
 371
 372 retry:
 373         retval = ext2fs_link(fs, dir, fn_buf, *ino, EXT2_FT_REG_FILE);
 374         if (retval == EXT2_ET_DIR_NO_SPACE) {
 375                 retval = ext2fs_expand_dir(fs, dir);
 376                 if (retval)
 377                         goto errout;
 378                 goto retry;
 379         }
 380
 381         if (retval)
 382                 goto errout;
 383
 384 errout:
 385         if (handle)
 386                 ext2fs_extent_free(handle);
 387
 388         return retval;
 389 }
 390
 391 static blk64_t calc_overhead(ext2_filsys fs, blk64_t num)
 392 {
 393         blk64_t e_blocks, e_blocks2, e_blocks3, e_blocks4;
 394         int extents_per_block;
 395         int extents = (num + EXT_INIT_MAX_LEN - 1) / EXT_INIT_MAX_LEN;
 396
 397         if (extents <= 4)
 398                 return 0;
 399
 400         /*
 401          * This calculation is due to the fact that we are inefficient
 402          * in how handle extent splits when appending to the end of
 403          * the extent tree.  Sigh.  We should fix this so that we can
 404          * actually store 340 extents per 4k block, instead of only 170.
 405          */
 406         extents_per_block = ((fs->blocksize -
 407                               sizeof(struct ext3_extent_header)) /
 408                              sizeof(struct ext3_extent));
 409         extents_per_block = (extents_per_block/ 2) - 1;
 410
 411         e_blocks = (extents + extents_per_block - 1) / extents_per_block;
 412         e_blocks2 = (e_blocks + extents_per_block - 1) / extents_per_block;
 413         e_blocks3 = (e_blocks2 + extents_per_block - 1) / extents_per_block;
 414         e_blocks4 = (e_blocks3 + extents_per_block - 1) / extents_per_block;
 415         return e_blocks + e_blocks2 + e_blocks3 + e_blocks4;
 416 }
 417
 418 /*
 419  * Find the place where we should start allocating blocks for the huge
 420  * files.  Leave <slack> free blocks at the beginning of the file
 421  * system for things like metadata blocks.
 422  */
 423 static blk64_t get_start_block(ext2_filsys fs, blk64_t slack)
 424 {
 425         errcode_t retval;
 426         blk64_t blk = fs->super->s_first_data_block, next;
 427         blk64_t last_blk = ext2fs_blocks_count(fs->super) - 1;
 428
 429         while (slack) {
 430                 retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 431                                                 blk, last_blk, &blk);
 432                 if (retval)
 433                         break;
 434
 435                 retval = ext2fs_find_first_set_block_bitmap2(fs->block_map,
 436                                                 blk, last_blk, &next);
 437                 if (retval)
 438                         next = last_blk;
 439                 next--;
 440
 441                 if (next - blk > slack) {
 442                         blk += slack;
 443                         break;
 444                 }
 445
 446                 slack -= (next - blk);
 447                 blk = next;
 448         }
 449         return blk;
 450 }
 451
 452 static blk64_t round_up_align(blk64_t b, unsigned long align,
 453                               blk64_t part_offset)
 454 {
 455         unsigned long m;
 456
 457         if (align == 0)
 458                 return b;
 459         part_offset = part_offset % align;
 460         m = (b + part_offset) % align;
 461         if (m)
 462                 b += align - m;
 463         return b;
 464 }
 465
 466 errcode_t mk_hugefiles(ext2_filsys fs, const char *device_name)
 467 {
 468         unsigned long   i;
 469         ext2_ino_t      dir;
 470         errcode_t       retval;
 471         blk64_t         fs_blocks, part_offset;
 472         unsigned long   align;
 473         int             d, dsize;
 474         char            *t;
 475
 476         if (!get_bool_from_profile(fs_types, "make_hugefiles", 0))
 477                 return 0;
 478
 479         uid = get_int_from_profile(fs_types, "hugefiles_uid", 0);
 480         gid = get_int_from_profile(fs_types, "hugefiles_gid", 0);
 481         fs->umask = get_int_from_profile(fs_types, "hugefiles_umask", 077);
 482         num_files = get_int_from_profile(fs_types, "num_hugefiles", 0);
 483         t = get_string_from_profile(fs_types, "hugefiles_slack", "1M");
 484         num_slack = parse_num_blocks2(t, fs->super->s_log_block_size);
 485         free(t);
 486         t = get_string_from_profile(fs_types, "hugefiles_size", "0");
 487         num_blocks = parse_num_blocks2(t, fs->super->s_log_block_size);
 488         free(t);
 489         t = get_string_from_profile(fs_types, "hugefiles_align", "0");
 490         align = parse_num_blocks2(t, fs->super->s_log_block_size);
 491         free(t);
 492         if (get_bool_from_profile(fs_types, "hugefiles_align_disk", 0))
 493                 part_offset = get_partition_start(device_name) /
 494                         (fs->blocksize / 512);
 495         num_blocks = round_up_align(num_blocks, align, 0);
 496         zero_hugefile = get_bool_from_profile(fs_types, "zero_hugefiles",
 497                                               zero_hugefile);
 498
 499         t = get_string_from_profile(fs_types, "hugefiles_dir", "/");
 500         retval = create_directory(fs, t, &dir);
 501         free(t);
 502         if (retval)
 503                 return retval;
 504
 505         fn_prefix = get_string_from_profile(fs_types, "hugefiles_name",
 506                                             "hugefile");
 507         idx_digits = get_int_from_profile(fs_types, "hugefiles_digits", 5);
 508         d = int_log10(num_files) + 1;
 509         if (idx_digits > d)
 510                 d = idx_digits;
 511         dsize = strlen(fn_prefix) + d + 16;
 512         fn_buf = malloc(dsize);
 513         if (!fn_buf) {
 514                 free(fn_prefix);
 515                 return ENOMEM;
 516         }
 517         strcpy(fn_buf, fn_prefix);
 518         fn_numbuf = fn_buf + strlen(fn_prefix);
 519         free(fn_prefix);
 520
 521         fs_blocks = ext2fs_free_blocks_count(fs->super);
 522         if (fs_blocks < num_slack + align)
 523                 return ENOMEM;
 524         fs_blocks -= num_slack + align;
 525         if (num_blocks && num_blocks > fs_blocks)
 526                 return ENOMEM;
 527         if (num_blocks == 0 && num_files == 0)
 528                 num_files = 1;
 529
 530         if (num_files == 0 && num_blocks) {
 531                 num_files = fs_blocks / num_blocks;
 532                 fs_blocks -= (num_files / 16) + 1;
 533                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 534                 num_files = fs_blocks / num_blocks;
 535         }
 536
 537         if (num_blocks == 0 && num_files > 1) {
 538                 num_blocks = fs_blocks / num_files;
 539                 fs_blocks -= (num_files / 16) + 1;
 540                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 541                 num_blocks = fs_blocks / num_files;
 542         }
 543
 544         num_slack += calc_overhead(fs, num_blocks) * num_files;
 545         num_slack += (num_files / 16) + 1; /* space for dir entries */
 546         goal = get_start_block(fs, num_slack);
 547         goal = round_up_align(goal, align, part_offset);
 548
 549         if ((num_blocks ? num_blocks : fs_blocks) >
 550             (0x80000000UL / fs->blocksize))
 551                 fs->super->s_feature_ro_compat |=
 552                         EXT2_FEATURE_RO_COMPAT_LARGE_FILE;
 553
 554         if (!quiet) {
 555                 if (zero_hugefile && verbose)
 556                         printf("%s", _("Huge files will be zero'ed\n"));
 557                 printf(_("Creating %lu huge file(s) "), num_files);
 558                 if (num_blocks)
 559                         printf(_("with %llu blocks each"), num_blocks);
 560                 fputs(": ", stdout);
 561         }
 562         for (i=0; i < num_files; i++) {
 563                 ext2_ino_t ino;
 564
 565                 retval = mk_hugefile(fs, num_blocks, dir, i, &ino);
 566                 if (retval) {
 567                         com_err(program_name, retval,
 568                                 _("while creating huge file %lu"), i);
 569                         goto errout;
 570                 }
 571         }
 572         if (!quiet)
 573                 fputs(_("done\n"), stdout);
 574
 575 errout:
 576         free(fn_buf);
 577         return retval;
 578 }