misc/mk_hugefiles.c

   1 /*
   2  * mk_hugefiles.c -- create huge files
   3  */
   4
   5 #define _XOPEN_SOURCE 600 /* for inclusion of PATH_MAX in Solaris */
   6 #define _BSD_SOURCE       /* for makedev() and major() */
   7 #define _DEFAULT_SOURCE   /* since glibc 2.20 _BSD_SOURCE is deprecated */
   8
   9 #include "config.h"
  10 #include <stdio.h>
  11 #include <stdarg.h>
  12 #include <string.h>
  13 #include <strings.h>
  14 #include <fcntl.h>
  15 #include <ctype.h>
  16 #include <time.h>
  17 #ifdef __linux__
  18 #include <sys/utsname.h>
  19 #endif
  20 #ifdef HAVE_GETOPT_H
  21 #include <getopt.h>
  22 #else
  23 extern char *optarg;
  24 extern int optind;
  25 #endif
  26 #ifdef HAVE_UNISTD_H
  27 #include <unistd.h>
  28 #endif
  29 #ifdef HAVE_STDLIB_H
  30 #include <stdlib.h>
  31 #endif
  32 #ifdef HAVE_ERRNO_H
  33 #include <errno.h>
  34 #endif
  35 #ifdef HAVE_SYS_IOCTL_H
  36 #include <sys/ioctl.h>
  37 #endif
  38 #include <sys/types.h>
  39 #include <sys/stat.h>
  40 #ifdef HAVE_SYS_SYSMACROS_H
  41 #include <sys/sysmacros.h>
  42 #endif
  43 #include <libgen.h>
  44 #include <limits.h>
  45 #include <blkid/blkid.h>
  46
  47 #include "ext2fs/ext2_fs.h"
  48 #include "ext2fs/ext2fsP.h"
  49 #include "et/com_err.h"
  50 #include "uuid/uuid.h"
  51 #include "e2p/e2p.h"
  52 #include "ext2fs/ext2fs.h"
  53 #include "util.h"
  54 #include "support/profile.h"
  55 #include "support/prof_err.h"
  56 #include "support/nls-enable.h"
  57 #include "mke2fs.h"
  58
  59 static int uid;
  60 static int gid;
  61 static blk64_t num_blocks;
  62 static blk64_t num_slack;
  63 static unsigned long num_files;
  64 static blk64_t goal;
  65 static char *fn_prefix;
  66 static int idx_digits;
  67 static char *fn_buf;
  68 static char *fn_numbuf;
  69 int zero_hugefile = 1;
  70
  71 #define SYSFS_PATH_LEN 300
  72 typedef char sysfs_path_t[SYSFS_PATH_LEN];
  73
  74 #ifndef HAVE_SNPRINTF
  75 /*
  76  * We are very careful to avoid needing to worry about buffer
  77  * overflows, so we don't really need to use snprintf() except as an
  78  * additional safety check.  So if snprintf() is not present, it's
  79  * safe to fall back to vsprintf().  This provides portability since
  80  * vsprintf() is guaranteed by C89, while snprintf() is only
  81  * guaranteed by C99 --- which for example, Microsoft Visual Studio
  82  * has *still* not bothered to implement.  :-/  (Not that I expect
  83  * mke2fs to be ported to MS Visual Studio any time soon, but
  84  * libext2fs *does* get built on Microsoft platforms, and we might
  85  * want to move this into libext2fs some day.)
  86  */
  87 static int my_snprintf(char *str, size_t size, const char *format, ...)
  88 {
  89         va_list ap;
  90         int ret;
  91
  92         va_start(ap, format);
  93         ret = vsprintf(str, format, ap);
  94         va_end(ap);
  95         return ret;
  96 }
  97
  98 #define snprintf my_snprintf
  99 #endif
 100
 101 /*
 102  * Fall back to Linux's definitions of makedev and major are needed.
 103  * The search_sysfs_block() function is highly unlikely to work on
 104  * non-Linux systems anyway.
 105  */
 106 #ifndef makedev
 107 #define makedev(maj, min) (((maj) << 8) + (min))
 108 #endif
 109
 110 static char *search_sysfs_block(dev_t devno, sysfs_path_t ret_path)
 111 {
 112         struct dirent   *de, *p_de;
 113         DIR             *dir = NULL, *p_dir = NULL;
 114         FILE            *f;
 115         sysfs_path_t    path, p_path;
 116         unsigned int    major, minor;
 117         char            *ret = ret_path;
 118
 119         ret_path[0] = 0;
 120         if ((dir = opendir("/sys/block")) == NULL)
 121                 return NULL;
 122         while ((de = readdir(dir)) != NULL) {
 123                 if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..") ||
 124                     strlen(de->d_name) > sizeof(path)-32)
 125                         continue;
 126                 snprintf(path, SYSFS_PATH_LEN,
 127                          "/sys/block/%s/dev", de->d_name);
 128                 f = fopen(path, "r");
 129                 if (f &&
 130                     (fscanf(f, "%u:%u", &major, &minor) == 2)) {
 131                         fclose(f); f = NULL;
 132                         if (makedev(major, minor) == devno) {
 133                                 snprintf(ret_path, SYSFS_PATH_LEN,
 134                                          "/sys/block/%s", de->d_name);
 135                                 goto success;
 136                         }
 137 #ifdef major
 138                         if (major(devno) != major)
 139                                 continue;
 140 #endif
 141                 }
 142                 if (f)
 143                         fclose(f);
 144
 145                 snprintf(path, SYSFS_PATH_LEN, "/sys/block/%s", de->d_name);
 146
 147                 if (p_dir)
 148                         closedir(p_dir);
 149                 if ((p_dir = opendir(path)) == NULL)
 150                         continue;
 151                 while ((p_de = readdir(p_dir)) != NULL) {
 152                         if (!strcmp(p_de->d_name, ".") ||
 153                             !strcmp(p_de->d_name, "..") ||
 154                             (strlen(p_de->d_name) >
 155                              SYSFS_PATH_LEN - strlen(path) - 32))
 156                                 continue;
 157                         snprintf(p_path, SYSFS_PATH_LEN, "%s/%s/dev",
 158                                  path, p_de->d_name);
 159
 160                         f = fopen(p_path, "r");
 161                         if (f &&
 162                             (fscanf(f, "%u:%u", &major, &minor) == 2) &&
 163                             (((major << 8) + minor) == devno)) {
 164                                 fclose(f);
 165                                 snprintf(ret_path, SYSFS_PATH_LEN, "%s/%s",
 166                                          path, p_de->d_name);
 167                                 goto success;
 168                         }
 169                         if (f)
 170                                 fclose(f);
 171                 }
 172         }
 173         ret = NULL;
 174 success:
 175         if (dir)
 176                 closedir(dir);
 177         if (p_dir)
 178                 closedir(p_dir);
 179         return ret;
 180 }
 181
 182 static blk64_t get_partition_start(const char *device_name)
 183 {
 184         unsigned long long start;
 185         sysfs_path_t    path;
 186         struct stat     st;
 187         FILE            *f;
 188         char            *cp;
 189         int             n;
 190
 191         if ((stat(device_name, &st) < 0) || !S_ISBLK(st.st_mode))
 192                 return 0;
 193
 194         cp = search_sysfs_block(st.st_rdev, path);
 195         if (!cp)
 196                 return 0;
 197         if (strlen(path) > SYSFS_PATH_LEN - sizeof("/start"))
 198                 return 0;
 199         strcat(path, "/start");
 200         f = fopen(path, "r");
 201         if (!f)
 202                 return 0;
 203         n = fscanf(f, "%llu", &start);
 204         fclose(f);
 205         return (n == 1) ? start : 0;
 206 }
 207
 208 static errcode_t create_directory(ext2_filsys fs, char *dir,
 209                                   ext2_ino_t *ret_ino)
 210
 211 {
 212         struct ext2_inode       inode;
 213         ext2_ino_t              ino = EXT2_ROOT_INO;
 214         ext2_ino_t              newdir;
 215         errcode_t               retval = 0;
 216         char                    *fn, *cp, *next;
 217
 218         fn = malloc(strlen(dir) + 1);
 219         if (fn == NULL)
 220                 return ENOMEM;
 221
 222         strcpy(fn, dir);
 223         cp = fn;
 224         while(1) {
 225                 next = strchr(cp, '/');
 226                 if (next)
 227                         *next++ = 0;
 228                 if (*cp) {
 229                         retval = ext2fs_new_inode(fs, ino, LINUX_S_IFDIR,
 230                                                   NULL, &newdir);
 231                         if (retval)
 232                                 goto errout;
 233
 234                         retval = ext2fs_mkdir(fs, ino, newdir, cp);
 235                         if (retval)
 236                                 goto errout;
 237
 238                         ino = newdir;
 239                         retval = ext2fs_read_inode(fs, ino, &inode);
 240                         if (retval)
 241                                 goto errout;
 242
 243                         inode.i_uid = uid & 0xFFFF;
 244                         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 245                         inode.i_gid = gid & 0xFFFF;
 246                         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 247                         retval = ext2fs_write_inode(fs, ino, &inode);
 248                         if (retval)
 249                                 goto errout;
 250                 }
 251                 if (next == NULL || *next == '\0')
 252                         break;
 253                 cp = next;
 254         }
 255 errout:
 256         free(fn);
 257         if (retval == 0)
 258                 *ret_ino = ino;
 259         return retval;
 260 }
 261
 262 static errcode_t mk_hugefile(ext2_filsys fs, blk64_t num,
 263                              ext2_ino_t dir, unsigned long idx, ext2_ino_t *ino)
 264
 265 {
 266         errcode_t               retval;
 267         blk64_t                 lblk, bend = 0;
 268         __u64                   size;
 269         blk64_t                 left;
 270         blk64_t                 count = 0;
 271         struct ext2_inode       inode;
 272         ext2_extent_handle_t    handle;
 273
 274         retval = ext2fs_new_inode(fs, 0, LINUX_S_IFREG, NULL, ino);
 275         if (retval)
 276                 return retval;
 277
 278         memset(&inode, 0, sizeof(struct ext2_inode));
 279         inode.i_mode = LINUX_S_IFREG | (0666 & ~fs->umask);
 280         inode.i_links_count = 1;
 281         inode.i_uid = uid & 0xFFFF;
 282         ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
 283         inode.i_gid = gid & 0xFFFF;
 284         ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
 285
 286         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 287         if (retval)
 288                 return retval;
 289
 290         ext2fs_inode_alloc_stats2(fs, *ino, +1, 0);
 291
 292         retval = ext2fs_extent_open2(fs, *ino, &inode, &handle);
 293         if (retval)
 294                 return retval;
 295
 296         /*
 297          * We don't use ext2fs_fallocate() here because hugefiles are
 298          * designed to be physically contiguous (if the block group
 299          * descriptors are configured to be in a single block at the
 300          * beginning of the file system, by using the
 301          * packed_meta_blocks layout), with the extent tree blocks
 302          * allocated near the beginning of the file system.
 303          */
 304         lblk = 0;
 305         left = num ? num : 1;
 306         while (left) {
 307                 blk64_t pblk, end;
 308                 blk64_t n = left;
 309
 310                 retval =  ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 311                         goal, ext2fs_blocks_count(fs->super) - 1, &end);
 312                 if (retval)
 313                         goto errout;
 314                 goal = end;
 315
 316                 retval =  ext2fs_find_first_set_block_bitmap2(fs->block_map, goal,
 317                                ext2fs_blocks_count(fs->super) - 1, &bend);
 318                 if (retval == ENOENT) {
 319                         bend = ext2fs_blocks_count(fs->super);
 320                         if (num == 0)
 321                                 left = 0;
 322                 }
 323                 if (!num || bend - goal < left)
 324                         n = bend - goal;
 325                 pblk = goal;
 326                 if (num)
 327                         left -= n;
 328                 goal += n;
 329                 count += n;
 330                 ext2fs_block_alloc_stats_range(fs, pblk, n, +1);
 331
 332                 if (zero_hugefile) {
 333                         blk64_t ret_blk;
 334                         retval = ext2fs_zero_blocks2(fs, pblk, n,
 335                                                      &ret_blk, NULL);
 336
 337                         if (retval)
 338                                 com_err(program_name, retval,
 339                                         _("while zeroing block %llu "
 340                                           "for hugefile"),
 341                                         (unsigned long long) ret_blk);
 342                 }
 343
 344                 while (n) {
 345                         blk64_t l = n;
 346                         struct ext2fs_extent newextent;
 347
 348                         if (l > EXT_INIT_MAX_LEN)
 349                                 l = EXT_INIT_MAX_LEN;
 350
 351                         newextent.e_len = l;
 352                         newextent.e_pblk = pblk;
 353                         newextent.e_lblk = lblk;
 354                         newextent.e_flags = 0;
 355
 356                         retval = ext2fs_extent_insert(handle,
 357                                         EXT2_EXTENT_INSERT_AFTER, &newextent);
 358                         if (retval)
 359                                 return retval;
 360                         pblk += l;
 361                         lblk += l;
 362                         n -= l;
 363                 }
 364         }
 365
 366         retval = ext2fs_read_inode(fs, *ino, &inode);
 367         if (retval)
 368                 goto errout;
 369
 370         retval = ext2fs_iblk_add_blocks(fs, &inode,
 371                                         count / EXT2FS_CLUSTER_RATIO(fs));
 372         if (retval)
 373                 goto errout;
 374         size = (__u64) count * fs->blocksize;
 375         retval = ext2fs_inode_size_set(fs, &inode, size);
 376         if (retval)
 377                 goto errout;
 378
 379         retval = ext2fs_write_new_inode(fs, *ino, &inode);
 380         if (retval)
 381                 goto errout;
 382
 383         if (idx_digits)
 384                 sprintf(fn_numbuf, "%0*lu", idx_digits, idx);
 385         else if (num_files > 1)
 386                 sprintf(fn_numbuf, "%lu", idx);
 387
 388 retry:
 389         retval = ext2fs_link(fs, dir, fn_buf, *ino, EXT2_FT_REG_FILE);
 390         if (retval == EXT2_ET_DIR_NO_SPACE) {
 391                 retval = ext2fs_expand_dir(fs, dir);
 392                 if (retval)
 393                         goto errout;
 394                 goto retry;
 395         }
 396
 397         if (retval)
 398                 goto errout;
 399
 400 errout:
 401         if (handle)
 402                 ext2fs_extent_free(handle);
 403
 404         return retval;
 405 }
 406
 407 static blk64_t calc_overhead(ext2_filsys fs, blk64_t num)
 408 {
 409         blk64_t e_blocks, e_blocks2, e_blocks3, e_blocks4;
 410         int extents_per_block;
 411         int extents = (num + EXT_INIT_MAX_LEN - 1) / EXT_INIT_MAX_LEN;
 412
 413         if (extents <= 4)
 414                 return 0;
 415
 416         /*
 417          * This calculation is due to the fact that we are inefficient
 418          * in how handle extent splits when appending to the end of
 419          * the extent tree.  Sigh.  We should fix this so that we can
 420          * actually store 340 extents per 4k block, instead of only 170.
 421          */
 422         extents_per_block = ((fs->blocksize -
 423                               sizeof(struct ext3_extent_header)) /
 424                              sizeof(struct ext3_extent));
 425         extents_per_block = (extents_per_block/ 2) - 1;
 426
 427         e_blocks = (extents + extents_per_block - 1) / extents_per_block;
 428         e_blocks2 = (e_blocks + extents_per_block - 1) / extents_per_block;
 429         e_blocks3 = (e_blocks2 + extents_per_block - 1) / extents_per_block;
 430         e_blocks4 = (e_blocks3 + extents_per_block - 1) / extents_per_block;
 431         return (e_blocks + e_blocks2 + e_blocks3 + e_blocks4) *
 432                 EXT2FS_CLUSTER_RATIO(fs);
 433 }
 434
 435 /*
 436  * Find the place where we should start allocating blocks for the huge
 437  * files.  Leave <slack> free blocks at the beginning of the file
 438  * system for things like metadata blocks.
 439  */
 440 static blk64_t get_start_block(ext2_filsys fs, blk64_t slack)
 441 {
 442         errcode_t retval;
 443         blk64_t blk = fs->super->s_first_data_block, next;
 444         blk64_t last_blk = ext2fs_blocks_count(fs->super) - 1;
 445
 446         while (slack) {
 447                 retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map,
 448                                                 blk, last_blk, &blk);
 449                 if (retval)
 450                         break;
 451
 452                 retval = ext2fs_find_first_set_block_bitmap2(fs->block_map,
 453                                                 blk, last_blk, &next);
 454                 if (retval)
 455                         next = last_blk;
 456
 457                 if (next - blk > slack) {
 458                         blk += slack;
 459                         break;
 460                 }
 461
 462                 slack -= (next - blk);
 463                 blk = next;
 464         }
 465         return blk;
 466 }
 467
 468 static blk64_t round_up_align(blk64_t b, unsigned long align,
 469                               blk64_t part_offset)
 470 {
 471         unsigned long m;
 472
 473         if (align == 0)
 474                 return b;
 475         part_offset = part_offset % align;
 476         m = (b + part_offset) % align;
 477         if (m)
 478                 b += align - m;
 479         return b;
 480 }
 481
 482 errcode_t mk_hugefiles(ext2_filsys fs, const char *device_name)
 483 {
 484         unsigned long   i;
 485         ext2_ino_t      dir;
 486         errcode_t       retval;
 487         blk64_t         fs_blocks, part_offset = 0;
 488         unsigned long   align;
 489         int             d, dsize;
 490         char            *t;
 491
 492         if (!get_bool_from_profile(fs_types, "make_hugefiles", 0))
 493                 return 0;
 494
 495         if (!ext2fs_has_feature_extents(fs->super))
 496                 return EXT2_ET_EXTENT_NOT_SUPPORTED;
 497
 498         uid = get_int_from_profile(fs_types, "hugefiles_uid", 0);
 499         gid = get_int_from_profile(fs_types, "hugefiles_gid", 0);
 500         fs->umask = get_int_from_profile(fs_types, "hugefiles_umask", 077);
 501         num_files = get_int_from_profile(fs_types, "num_hugefiles", 0);
 502         t = get_string_from_profile(fs_types, "hugefiles_slack", "1M");
 503         num_slack = parse_num_blocks2(t, fs->super->s_log_block_size);
 504         free(t);
 505         t = get_string_from_profile(fs_types, "hugefiles_size", "0");
 506         num_blocks = parse_num_blocks2(t, fs->super->s_log_block_size);
 507         free(t);
 508         t = get_string_from_profile(fs_types, "hugefiles_align", "0");
 509         align = parse_num_blocks2(t, fs->super->s_log_block_size);
 510         free(t);
 511         if (get_bool_from_profile(fs_types, "hugefiles_align_disk", 0)) {
 512                 part_offset = get_partition_start(device_name) /
 513                         (fs->blocksize / 512);
 514                 if (part_offset % EXT2FS_CLUSTER_RATIO(fs)) {
 515                         fprintf(stderr,
 516                                 _("Partition offset of %llu (%uk) blocks "
 517                                   "not compatible with cluster size %u.\n"),
 518                                 (unsigned long long) part_offset, fs->blocksize,
 519                                 EXT2_CLUSTER_SIZE(fs->super));
 520                         exit(1);
 521                 }
 522         }
 523         num_blocks = round_up_align(num_blocks, align, 0);
 524         zero_hugefile = get_bool_from_profile(fs_types, "zero_hugefiles",
 525                                               zero_hugefile);
 526
 527         t = get_string_from_profile(fs_types, "hugefiles_dir", "/");
 528         retval = create_directory(fs, t, &dir);
 529         free(t);
 530         if (retval)
 531                 return retval;
 532
 533         fn_prefix = get_string_from_profile(fs_types, "hugefiles_name",
 534                                             "hugefile");
 535         idx_digits = get_int_from_profile(fs_types, "hugefiles_digits", 5);
 536         d = int_log10(num_files) + 1;
 537         if (idx_digits > d)
 538                 d = idx_digits;
 539         dsize = strlen(fn_prefix) + d + 16;
 540         fn_buf = malloc(dsize);
 541         if (!fn_buf) {
 542                 free(fn_prefix);
 543                 return ENOMEM;
 544         }
 545         strcpy(fn_buf, fn_prefix);
 546         fn_numbuf = fn_buf + strlen(fn_prefix);
 547         free(fn_prefix);
 548
 549         fs_blocks = ext2fs_free_blocks_count(fs->super);
 550         if (fs_blocks < num_slack + align)
 551                 return ENOSPC;
 552         fs_blocks -= num_slack + align;
 553         if (num_blocks && num_blocks > fs_blocks)
 554                 return ENOSPC;
 555         if (num_blocks == 0 && num_files == 0)
 556                 num_files = 1;
 557
 558         if (num_files == 0 && num_blocks) {
 559                 num_files = fs_blocks / num_blocks;
 560                 fs_blocks -= (num_files / 16) + 1;
 561                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 562                 num_files = fs_blocks / num_blocks;
 563         }
 564
 565         if (num_blocks == 0 && num_files > 1) {
 566                 num_blocks = fs_blocks / num_files;
 567                 fs_blocks -= (num_files / 16) + 1;
 568                 fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
 569                 num_blocks = fs_blocks / num_files;
 570         }
 571
 572         num_slack += (calc_overhead(fs, num_blocks ? num_blocks : fs_blocks) *
 573                       num_files);
 574         num_slack += (num_files / 16) + 1; /* space for dir entries */
 575         goal = get_start_block(fs, num_slack);
 576         goal = round_up_align(goal, align, part_offset);
 577
 578         if ((num_blocks ? num_blocks : fs_blocks) >
 579             (0x80000000UL / fs->blocksize))
 580                 ext2fs_set_feature_large_file(fs->super);
 581
 582         if (!quiet) {
 583                 if (zero_hugefile && verbose)
 584                         printf("%s", _("Huge files will be zero'ed\n"));
 585                 printf(_("Creating %lu huge file(s) "), num_files);
 586                 if (num_blocks)
 587                         printf(_("with %llu blocks each"),
 588                                (unsigned long long) num_blocks);
 589                 fputs(": ", stdout);
 590         }
 591         for (i=0; i < num_files; i++) {
 592                 ext2_ino_t ino;
 593
 594                 retval = mk_hugefile(fs, num_blocks, dir, i, &ino);
 595                 if (retval) {
 596                         com_err(program_name, retval,
 597                                 _("while creating huge file %lu"), i);
 598                         goto errout;
 599                 }
 600         }
 601         if (!quiet)
 602                 fputs(_("done\n"), stdout);
 603
 604 errout:
 605         free(fn_buf);
 606         return retval;
 607 }