ldiskfs/kernel_patches/patches/rhel8.7/ext4-mballoc-prefetch.patch

   1 --- linux-4.18/fs/ext4/balloc.c 2019-11-28 14:55:26.506546036 +0300
   2 +++ linux-4.18/fs/ext4/balloc.c 2019-12-02 11:21:50.565975537 +0300
   3 @@ -404,7 +404,8 @@ verified:
   4   * Return buffer_head on success or NULL in case of failure.
   5   */
   6  struct buffer_head *
   7 -ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group)
   8 +ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group,
   9 +                                int ignore_locked)
  10  {
  11         struct ext4_group_desc *desc;
  12         struct ext4_sb_info *sbi = EXT4_SB(sb);
  13 @@ -435,6 +436,13 @@ ext4_read_block_bitmap_nowait(struct
  14         if (bitmap_uptodate(bh))
  15                 goto verify;
  16
  17 +       if (ignore_locked && buffer_locked(bh)) {
  18 +               /* buffer under IO already, do not wait
  19 +                * if called for prefetching */
  20 +               put_bh(bh);
  21 +               return NULL;
  22 +       }
  23 +
  24         lock_buffer(bh);
  25         if (bitmap_uptodate(bh)) {
  26                 unlock_buffer(bh);
  27 @@ -524,7 +532,7 @@ ext4_read_block_bitmap(struct super_b
  28         struct buffer_head *bh;
  29         int err;
  30
  31 -       bh = ext4_read_block_bitmap_nowait(sb, block_group);
  32 +       bh = ext4_read_block_bitmap_nowait(sb, block_group, 0);
  33         if (IS_ERR(bh))
  34                 return bh;
  35         err = ext4_wait_block_bitmap(sb, block_group, bh);
  36 --- linux-4.18/fs/ext4/ext4.h   2019-11-28 14:55:26.470545343 +0300
  37 +++ linux-4.18/fs/ext4/ext4.h   2019-12-02 11:21:40.795779972 +0300
  38 @@ -1446,6 +1446,8 @@ struct ext4_sb_info {
  39         /* where last allocation was done - for stream allocation */
  40         unsigned long s_mb_last_group;
  41         unsigned long s_mb_last_start;
  42 +       unsigned int s_mb_prefetch;
  43 +       unsigned int s_mb_prefetch_limit;
  44
  45         /* stats for buddy allocator */
  46         atomic_t s_bal_reqs;    /* number of reqs with len > 1 */
  47 @@ -2401,7 +2403,8 @@ extern struct ext4_group_desc * ldisk
  48  extern int ext4_should_retry_alloc(struct super_block *sb, int *retries);
  49
  50  extern struct buffer_head *ext4_read_block_bitmap_nowait(struct super_block *sb,
  51 -                                               ext4_group_t block_group);
  52 +                                               ext4_group_t block_group,
  53 +                                               int ignore_locked);
  54  extern int ext4_wait_block_bitmap(struct super_block *sb,
  55                                   ext4_group_t block_group,
  56                                   struct buffer_head *bh);
  57 @@ -3047,6 +3051,7 @@ struct ext4_group_info {
  58  #define EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT    3
  59  #define EXT4_GROUP_INFO_IBITMAP_CORRUPT                \
  60         (1 << EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT)
  61 +#define EXT4_GROUP_INFO_BBITMAP_READ_BIT       4
  62
  63  #define EXT4_MB_GRP_NEED_INIT(grp)     \
  64         (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state)))
  65 @@ -3065,6 +3070,10 @@ struct ext4_group_info {
  66         (set_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
  67  #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \
  68         (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
  69 +#define EXT4_MB_GRP_TEST(grp)  \
  70 +       (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
  71 +#define EXT4_MB_GRP_TEST_AND_SET_READ(grp)     \
  72 +       (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
  73
  74  #define EXT4_MAX_CONTENTION            8
  75  #define EXT4_CONTENTION_THRESHOLD      2
  76 Index: kernel-4.18.0-423.el8/fs/ext4/mballoc.c
  77 ===================================================================
  78 --- kernel-4.18.0-423.el8.orig/fs/ext4/mballoc.c
  79 +++ kernel-4.18.0-423.el8/fs/ext4/mballoc.c
  80 @@ -929,7 +929,7 @@ static int ext4_mb_init_cache(struct pag
  81                         bh[i] = NULL;
  82                         continue;
  83                 }
  84 -               bh[i] = ext4_read_block_bitmap_nowait(sb, group);
  85 +               bh[i] = ext4_read_block_bitmap_nowait(sb, group, 0);
  86                 if (IS_ERR(bh[i])) {
  87                         err = PTR_ERR(bh[i]);
  88                         bh[i] = NULL;
  89 @@ -2245,6 +2245,92 @@ static u64 available_blocks_count(struct
  90         return bfree - (ext4_r_blocks_count(es) + resv_blocks);
  91  }
  92
  93 +/*
  94 + * each allocation context (i.e. a thread doing allocation) has own
  95 + * sliding prefetch window of @s_mb_prefetch size which starts at the
  96 + * very first goal and moves ahead of scaning.
  97 + * a side effect is that subsequent allocations will likely find
  98 + * the bitmaps in cache or at least in-flight.
  99 + */
 100 +static void
 101 +ext4_mb_prefetch(struct ext4_allocation_context *ac,
 102 +                   ext4_group_t start)
 103 +{
 104 +       struct super_block *sb = ac->ac_sb;
 105 +       ext4_group_t ngroups = ext4_get_groups_count(sb);
 106 +       struct ext4_sb_info *sbi = EXT4_SB(sb);
 107 +       struct ext4_group_info *grp;
 108 +       ext4_group_t group = start;
 109 +       struct buffer_head *bh;
 110 +       int nr;
 111 +
 112 +       /* limit prefetching at cr=0, otherwise mballoc can
 113 +        * spend a lot of time loading imperfect groups */
 114 +       if (ac->ac_criteria < 2 && ac->ac_prefetch_ios >= sbi->s_mb_prefetch_limit)
 115 +               return;
 116 +
 117 +       /* batch prefetching to get few READs in flight */
 118 +       nr = ac->ac_prefetch - group;
 119 +       if (ac->ac_prefetch < group)
 120 +               /* wrapped to the first groups */
 121 +               nr += ngroups;
 122 +       if (nr > 0)
 123 +               return;
 124 +       BUG_ON(nr < 0);
 125 +
 126 +       nr = sbi->s_mb_prefetch;
 127 +       if (ext4_has_feature_flex_bg(sb)) {
 128 +               /* align to flex_bg to get more bitmas with a single IO */
 129 +               nr = (group / sbi->s_mb_prefetch) * sbi->s_mb_prefetch;
 130 +               nr = nr + sbi->s_mb_prefetch - group;
 131 +       }
 132 +       while (nr-- > 0) {
 133 +               grp = ext4_get_group_info(sb, group);
 134 +               /* prevent expensive getblk() on groups w/ IO in progress */
 135 +               if (EXT4_MB_GRP_TEST(grp) || EXT4_MB_GRP_TEST_AND_SET_READ(grp))
 136 +                       goto next;
 137 +
 138 +               /* ignore empty groups - those will be skipped
 139 +                * during the scanning as well */
 140 +               if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
 141 +                       bh = ext4_read_block_bitmap_nowait(sb, group, 1);
 142 +                       if (bh && !IS_ERR(bh)) {
 143 +                               if (!buffer_uptodate(bh))
 144 +                                       ac->ac_prefetch_ios++;
 145 +                               brelse(bh);
 146 +                       }
 147 +               }
 148 +next:
 149 +               if (++group >= ngroups)
 150 +                       group = 0;
 151 +       }
 152 +       ac->ac_prefetch = group;
 153 +}
 154 +
 155 +static void
 156 +ext4_mb_prefetch_fini(struct ext4_allocation_context *ac)
 157 +{
 158 +       struct ext4_group_info *grp;
 159 +       ext4_group_t group;
 160 +       int nr, rc;
 161 +
 162 +       /* initialize last window of prefetched groups */
 163 +       nr = ac->ac_prefetch_ios;
 164 +       if (nr > EXT4_SB(ac->ac_sb)->s_mb_prefetch)
 165 +               nr = EXT4_SB(ac->ac_sb)->s_mb_prefetch;
 166 +       group = ac->ac_prefetch;
 167 +       while (nr-- > 0) {
 168 +               grp = ext4_get_group_info(ac->ac_sb, group);
 169 +               if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
 170 +                       rc = ext4_mb_init_group(ac->ac_sb, group, GFP_NOFS);
 171 +                       if (rc)
 172 +                               break;
 173 +               }
 174 +               if (group-- == 0)
 175 +                       group = ext4_get_groups_count(ac->ac_sb) - 1;
 176 +       }
 177 +}
 178 +
 179  static noinline_for_stack int
 180  ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
 181  {
 182 @@ -2334,6 +2420,7 @@ repeat:
 183                  * from the goal value specified
 184                  */
 185                 group = ac->ac_g_ex.fe_group;
 186 +               ac->ac_prefetch = group;
 187
 188                 for (i = 0; i < ngroups; group++, i++) {
 189                         int ret = 0;
 190 @@ -2345,6 +2432,8 @@ repeat:
 191                         if (group >= ngroups)
 192                                 group = 0;
 193
 194 +                       ext4_mb_prefetch(ac, group);
 195 +
 196                         /* This now checks without needing the buddy page */
 197                         ret = ext4_mb_good_group_nolock(ac, group, cr);
 198                         if (ret <= 0) {
 199 @@ -2422,6 +2511,9 @@ out:
 200         mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n",
 201                  ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,
 202                  ac->ac_flags, cr, err);
 203 +
 204 +       /* use prefetched bitmaps to init buddy so that read info is not lost */
 205 +       ext4_mb_prefetch_fini(ac);
 206         return err;
 207  }
 208
 209 @@ -3095,6 +3187,24 @@ int ext4_mb_init(struct super_block *sb)
 210                 sbi->s_mb_large_req = sbi->s_stripe * 8;
 211                 sbi->s_mb_group_prealloc = sbi->s_stripe * 4;
 212         }
 213 +       if (ext4_has_feature_flex_bg(sb)) {
 214 +               /* a single flex group is supposed to be read by a single IO */
 215 +               sbi->s_mb_prefetch = 1 << sbi->s_es->s_log_groups_per_flex;
 216 +               sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */
 217 +       } else {
 218 +               sbi->s_mb_prefetch = 32;
 219 +       }
 220 +       if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))
 221 +               sbi->s_mb_prefetch = ext4_get_groups_count(sb);
 222 +       /* now many real IOs to prefetch within a single allocation at cr=0
 223 +        * given cr=0 is an CPU-related optimization we shouldn't try to
 224 +        * load too many groups, at some point we should start to use what
 225 +        * we've got in memory.
 226 +        * with an average random access time 5ms, it'd take a second to get
 227 +        * 200 groups (* N with flex_bg), so let's make this limit 4 */
 228 +       sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;
 229 +       if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))
 230 +               sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);
 231
 232         sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
 233         if (sbi->s_locality_groups == NULL) {
 234 --- linux-4.18/fs/ext4/mballoc.h        2019-11-28 14:55:26.471545362 +0300
 235 +++ linux-4.18/fs/ext4/mballoc.h        2019-12-02 11:21:57.028104886 +0300
 236 @@ -177,6 +177,8 @@ struct ext4_allocation_context {
 237         struct page *ac_buddy_page;
 238         struct ext4_prealloc_space *ac_pa;
 239         struct ext4_locality_group *ac_lg;
 240 +       ext4_group_t ac_prefetch;
 241 +       int ac_prefetch_ios; /* number of initialied prefetch IO */
 242  };
 243
 244  #define AC_STATUS_CONTINUE     1
 245 --- linux-4.18/fs/ext4/sysfs.c  2019-11-28 14:55:26.502545959 +0300
 246 +++ linux-4.18/fs/ext4/sysfs.c  2019-11-28 20:07:48.104558177 +0300
 247 @@ -234,6 +234,8 @@ EXT4_RO_ATTR_ES_UI(errors_count, s_error
 248  EXT4_ATTR(first_error_time, 0444, first_error_time);
 249  EXT4_ATTR(last_error_time, 0444, last_error_time);
 250  EXT4_ATTR(journal_task, 0444, journal_task);
 251 +EXT4_RW_ATTR_SBI_UI(mb_prefetch, s_mb_prefetch);
 252 +EXT4_RW_ATTR_SBI_UI(mb_prefetch_limit, s_mb_prefetch_limit);
 253
 254  static unsigned int old_bump_val = 128;
 255  EXT4_ATTR_PTR(max_writeback_mb_bump, 0444, pointer_ui, &old_bump_val);
 256 @@ -271,6 +273,8 @@ static struct attribute *ext4_attrs[] =
 257         ATTR_LIST(first_error_time),
 258         ATTR_LIST(last_error_time),
 259         ATTR_LIST(journal_task),
 260 +       ATTR_LIST(mb_prefetch),
 261 +       ATTR_LIST(mb_prefetch_limit),
 262         NULL,
 263  };
 264