ldiskfs/kernel_patches/patches/rhel7.7/ext4-mballoc-prefetch.patch

   1 --- linux-4.18/fs/ext4/balloc.c 2019-11-28 14:55:26.506546036 +0300
   2 +++ linux-4.18/fs/ext4/balloc.c 2019-12-02 11:21:50.565975537 +0300
   3 @@ -404,7 +404,8 @@ verified:
   4   * Return buffer_head on success or NULL in case of failure.
   5   */
   6  struct buffer_head *
   7 -ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group)
   8 +ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group,
   9 +                                int ignore_locked)
  10  {
  11         struct ext4_group_desc *desc;
  12         struct ext4_sb_info *sbi = EXT4_SB(sb);
  13 @@ -435,6 +436,13 @@ ext4_read_block_bitmap_nowait(struct
  14         if (bitmap_uptodate(bh))
  15                 goto verify;
  16
  17 +       if (ignore_locked && buffer_locked(bh)) {
  18 +               /* buffer under IO already, do not wait
  19 +                * if called for prefetching */
  20 +               put_bh(bh);
  21 +               return NULL;
  22 +       }
  23 +
  24         lock_buffer(bh);
  25         if (bitmap_uptodate(bh)) {
  26                 unlock_buffer(bh);
  27 @@ -524,7 +532,7 @@ ext4_read_block_bitmap(struct super_b
  28  {
  29         struct buffer_head *bh;
  30
  31 -       bh = ext4_read_block_bitmap_nowait(sb, block_group);
  32 +       bh = ext4_read_block_bitmap_nowait(sb, block_group, 0);
  33         if (!bh)
  34                 return NULL;
  35         err = ext4_wait_block_bitmap(sb, block_group, bh);
  36 --- linux-4.18/fs/ext4/ext4.h   2019-11-28 14:55:26.470545343 +0300
  37 +++ linux-4.18/fs/ext4/ext4.h   2019-12-02 11:21:40.795779972 +0300
  38 @@ -1446,6 +1446,8 @@ struct ext4_sb_info {
  39         /* where last allocation was done - for stream allocation */
  40         unsigned long s_mb_last_group;
  41         unsigned long s_mb_last_start;
  42 +       unsigned int s_mb_prefetch;
  43 +       unsigned int s_mb_prefetch_limit;
  44
  45         /* stats for buddy allocator */
  46         atomic_t s_bal_reqs;    /* number of reqs with len > 1 */
  47 @@ -2401,7 +2403,8 @@ extern struct ext4_group_desc * ldisk
  48  extern int ext4_should_retry_alloc(struct super_block *sb, int *retries);
  49
  50  extern struct buffer_head *ext4_read_block_bitmap_nowait(struct super_block *sb,
  51 -                                               ext4_group_t block_group);
  52 +                                               ext4_group_t block_group,
  53 +                                               int ignore_locked);
  54  extern int ext4_wait_block_bitmap(struct super_block *sb,
  55                                   ext4_group_t block_group,
  56                                   struct buffer_head *bh);
  57 @@ -3047,6 +3051,7 @@ struct ext4_group_info {
  58  #define EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT    3
  59  #define EXT4_GROUP_INFO_IBITMAP_CORRUPT                \
  60         (1 << EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT)
  61 +#define EXT4_GROUP_INFO_BBITMAP_READ_BIT       4
  62
  63  #define EXT4_MB_GRP_NEED_INIT(grp)     \
  64         (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state)))
  65 @@ -3065,6 +3070,8 @@ struct ext4_group_info {
  66         (set_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
  67  #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \
  68         (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
  69 +#define EXT4_MB_GRP_TEST_AND_SET_READ(grp)     \
  70 +       (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
  71
  72  #define EXT4_MAX_CONTENTION            8
  73  #define EXT4_CONTENTION_THRESHOLD      2
  74 --- linux-4.18/fs/ext4/mballoc.c        2019-11-28 14:55:26.500545920 +0300
  75 +++ linux-4.18/fs/ext4/mballoc.c        2019-12-02 11:21:46.656897291 +0300
  76 @@ -868,7 +868,7 @@ static int ext4_mb_init_cache(struct
  77                         bh[i] = NULL;
  78                         continue;
  79                 }
  80 -               if (!(bh[i] = ext4_read_block_bitmap_nowait(sb, group))) {
  81 +               if (!(bh[i] = ext4_read_block_bitmap_nowait(sb, group, 0))) {
  82                         err = -ENOMEM;
  83                         goto out;
  84                 }
  85 @@ -2104,6 +2112,92 @@ static int ext4_mb_good_group(struct
  86         return 0;
  87  }
  88
  89 +/*
  90 + * each allocation context (i.e. a thread doing allocation) has own
  91 + * sliding prefetch window of @s_mb_prefetch size which starts at the
  92 + * very first goal and moves ahead of scaning.
  93 + * a side effect is that subsequent allocations will likely find
  94 + * the bitmaps in cache or at least in-flight.
  95 + */
  96 +static void
  97 +ext4_mb_prefetch(struct ext4_allocation_context *ac,
  98 +                   ext4_group_t start)
  99 +{
 100 +       struct super_block *sb = ac->ac_sb;
 101 +       ext4_group_t ngroups = ext4_get_groups_count(sb);
 102 +       struct ext4_sb_info *sbi = EXT4_SB(sb);
 103 +       struct ext4_group_info *grp;
 104 +       ext4_group_t group = start;
 105 +       struct buffer_head *bh;
 106 +       int nr;
 107 +
 108 +       /* limit prefetching at cr=0, otherwise mballoc can
 109 +        * spend a lot of time loading imperfect groups */
 110 +       if (ac->ac_criteria < 2 && ac->ac_prefetch_ios >= sbi->s_mb_prefetch_limit)
 111 +               return;
 112 +
 113 +       /* batch prefetching to get few READs in flight */
 114 +       nr = ac->ac_prefetch - group;
 115 +       if (ac->ac_prefetch < group)
 116 +               /* wrapped to the first groups */
 117 +               nr += ngroups;
 118 +       if (nr > 0)
 119 +               return;
 120 +       BUG_ON(nr < 0);
 121 +
 122 +       nr = sbi->s_mb_prefetch;
 123 +       if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_FLEX_BG)) {
 124 +               /* align to flex_bg to get more bitmas with a single IO */
 125 +               nr = (group / sbi->s_mb_prefetch) * sbi->s_mb_prefetch;
 126 +               nr = nr + sbi->s_mb_prefetch - group;
 127 +       }
 128 +       while (nr-- > 0) {
 129 +               grp = ext4_get_group_info(sb, group);
 130 +               /* prevent expensive getblk() on groups w/ IO in progress */
 131 +               if (EXT4_MB_GRP_TEST_AND_SET_READ(grp))
 132 +                       goto next;
 133 +
 134 +               /* ignore empty groups - those will be skipped
 135 +                * during the scanning as well */
 136 +               if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
 137 +                       bh = ext4_read_block_bitmap_nowait(sb, group, 1);
 138 +                       if (bh && !IS_ERR(bh)) {
 139 +                               if (!buffer_uptodate(bh))
 140 +                                       ac->ac_prefetch_ios++;
 141 +                               brelse(bh);
 142 +                       }
 143 +               }
 144 +next:
 145 +               if (++group >= ngroups)
 146 +                       group = 0;
 147 +       }
 148 +       ac->ac_prefetch = group;
 149 +}
 150 +
 151 +static void
 152 +ext4_mb_prefetch_fini(struct ext4_allocation_context *ac)
 153 +{
 154 +       struct ext4_group_info *grp;
 155 +       ext4_group_t group;
 156 +       int nr, rc;
 157 +
 158 +       /* initialize last window of prefetched groups */
 159 +       nr = ac->ac_prefetch_ios;
 160 +       if (nr > EXT4_SB(ac->ac_sb)->s_mb_prefetch)
 161 +               nr = EXT4_SB(ac->ac_sb)->s_mb_prefetch;
 162 +       group = ac->ac_prefetch;
 163 +       while (nr-- > 0) {
 164 +               grp = ext4_get_group_info(ac->ac_sb, group);
 165 +               if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
 166 +                       rc = ext4_mb_init_group(ac->ac_sb, group);
 167 +                       if (rc)
 168 +                               break;
 169 +               }
 170 +               if (group-- == 0)
 171 +                       group = ext4_get_groups_count(ac->ac_sb) - 1;
 172 +       }
 173 +}
 174 +
 175  static noinline_for_stack int
 176  ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
 177  {
 178 @@ -2176,6 +2264,7 @@ repeat:
 179                  * from the goal value specified
 180                  */
 181                 group = ac->ac_g_ex.fe_group;
 182 +               ac->ac_prefetch = group;
 183
 184                 for (i = 0; i < ngroups; group++, i++) {
 185                         int ret = 0;
 186 @@ -2188,6 +2277,8 @@ repeat:
 187                         if (group >= ngroups)
 188                                 group = 0;
 189
 190 +                       ext4_mb_prefetch(ac, group);
 191 +
 192                         /* This now checks without needing the buddy page */
 193                         ret = ext4_mb_good_group(ac, group, cr);
 194                         if (ret <= 0) {
 195 @@ -2260,6 +2351,8 @@ repeat:
 196  out:
 197         if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)
 198                 err = first_err;
 199 +       /* use prefetched bitmaps to init buddy so that read info is not lost */
 200 +       ext4_mb_prefetch_fini(ac);
 201         return err;
 202  }
 203
 204 @@ -2832,6 +2925,24 @@ int ext4_mb_init(struct super_block *
 205                 sbi->s_mb_large_req = sbi->s_stripe * 8;
 206                 sbi->s_mb_group_prealloc = sbi->s_stripe * 4;
 207         }
 208 +       if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_FLEX_BG)) {
 209 +               /* a single flex group is supposed to be read by a single IO */
 210 +               sbi->s_mb_prefetch = 1 << sbi->s_es->s_log_groups_per_flex;
 211 +               sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */
 212 +       } else {
 213 +               sbi->s_mb_prefetch = 32;
 214 +       }
 215 +       if (sbi->s_mb_prefetch >= ext4_get_groups_count(sb))
 216 +               sbi->s_mb_prefetch = ext4_get_groups_count(sb);
 217 +       /* now many real IOs to prefetch within a single allocation at cr=0
 218 +        * given cr=0 is an CPU-related optimization we shouldn't try to
 219 +        * load too many groups, at some point we should start to use what
 220 +        * we've got in memory.
 221 +        * with an average random access time 5ms, it'd take a second to get
 222 +        * 200 groups (* N with flex_bg), so let's make this limit 4 */
 223 +       sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;
 224 +       if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))
 225 +               sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);
 226
 227         sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
 228         if (sbi->s_locality_groups == NULL) {
 229 --- linux-4.18/fs/ext4/mballoc.h        2019-11-28 14:55:26.471545362 +0300
 230 +++ linux-4.18/fs/ext4/mballoc.h        2019-12-02 11:21:57.028104886 +0300
 231 @@ -177,6 +177,8 @@ struct ext4_allocation_context {
 232         struct page *ac_buddy_page;
 233         struct ext4_prealloc_space *ac_pa;
 234         struct ext4_locality_group *ac_lg;
 235 +       ext4_group_t ac_prefetch;
 236 +       int ac_prefetch_ios; /* number of initialied prefetch IO */
 237  };
 238
 239  #define AC_STATUS_CONTINUE     1
 240 --- linux-4.18/fs/ext4/super.c  2019-11-28 14:55:26.502545959 +0300
 241 +++ linux-4.18/fs/ext4/super.c  2019-11-28 20:07:48.104558177 +0300
 242 @@ -190,6 +190,8 @@ EXT4_RW_ATTR_SBI_UI(msg_ratelimit_bur
 243  EXT4_RW_ATTR_SBI_UI(mb_order2_req, s_mb_order2_reqs);
 244  EXT4_RW_ATTR_SBI_UI(mb_stream_req, s_mb_stream_request);
 245  EXT4_RW_ATTR_SBI_UI(mb_group_prealloc, s_mb_group_prealloc);
 246 +EXT4_RW_ATTR_SBI_UI(mb_prefetch, s_mb_prefetch);
 247 +EXT4_RW_ATTR_SBI_UI(mb_prefetch_limit, s_mb_prefetch_limit);
 248  EXT4_DEPRECATED_ATTR(max_writeback_mb_bump, 128);
 249  EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb);
 250  EXT4_ATTR(trigger_fs_error, 0200, NULL, trigger_test_error);
 251 @@ -223,6 +224,8 @@ static struct attribute *ext4_attrs[]
 252         ATTR_LIST(errors_count),
 253         ATTR_LIST(first_error_time),
 254         ATTR_LIST(last_error_time),
 255 +       ATTR_LIST(mb_prefetch),
 256 +       ATTR_LIST(mb_prefetch_limit),
 257         NULL,
 258  };
 259