ldiskfs/kernel_patches/patches/rhel8/ext4-mballoc-prefetch.patch

   1 --- linux-4.18/fs/ext4/balloc.c 2019-11-28 14:55:26.506546036 +0300
   2 +++ linux-4.18/fs/ext4/balloc.c 2019-12-02 11:21:50.565975537 +0300
   3 @@ -404,7 +404,8 @@ verified:
   4   * Return buffer_head on success or NULL in case of failure.
   5   */
   6  struct buffer_head *
   7 -ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group)
   8 +ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group,
   9 +                                int ignore_locked)
  10  {
  11         struct ext4_group_desc *desc;
  12         struct ext4_sb_info *sbi = EXT4_SB(sb);
  13 @@ -435,6 +436,13 @@ ext4_read_block_bitmap_nowait(struct
  14         if (bitmap_uptodate(bh))
  15                 goto verify;
  16
  17 +       if (ignore_locked && buffer_locked(bh)) {
  18 +               /* buffer under IO already, do not wait
  19 +                * if called for prefetching */
  20 +               put_bh(bh);
  21 +               return NULL;
  22 +       }
  23 +
  24         lock_buffer(bh);
  25         if (bitmap_uptodate(bh)) {
  26                 unlock_buffer(bh);
  27 @@ -524,7 +532,7 @@ ext4_read_block_bitmap(struct super_b
  28         struct buffer_head *bh;
  29         int err;
  30
  31 -       bh = ext4_read_block_bitmap_nowait(sb, block_group);
  32 +       bh = ext4_read_block_bitmap_nowait(sb, block_group, 0);
  33         if (IS_ERR(bh))
  34                 return bh;
  35         err = ext4_wait_block_bitmap(sb, block_group, bh);
  36 --- linux-4.18/fs/ext4/ext4.h   2019-11-28 14:55:26.470545343 +0300
  37 +++ linux-4.18/fs/ext4/ext4.h   2019-12-02 11:21:40.795779972 +0300
  38 @@ -1446,6 +1446,8 @@ struct ext4_sb_info {
  39         /* where last allocation was done - for stream allocation */
  40         unsigned long s_mb_last_group;
  41         unsigned long s_mb_last_start;
  42 +       unsigned int s_mb_prefetch;
  43 +       unsigned int s_mb_prefetch_limit;
  44
  45         /* stats for buddy allocator */
  46         atomic_t s_bal_reqs;    /* number of reqs with len > 1 */
  47 @@ -2401,7 +2403,8 @@ extern struct ext4_group_desc * ldisk
  48  extern int ext4_should_retry_alloc(struct super_block *sb, int *retries);
  49
  50  extern struct buffer_head *ext4_read_block_bitmap_nowait(struct super_block *sb,
  51 -                                               ext4_group_t block_group);
  52 +                                               ext4_group_t block_group,
  53 +                                               int ignore_locked);
  54  extern int ext4_wait_block_bitmap(struct super_block *sb,
  55                                   ext4_group_t block_group,
  56                                   struct buffer_head *bh);
  57 @@ -3047,6 +3051,7 @@ struct ext4_group_info {
  58  #define EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT    3
  59  #define EXT4_GROUP_INFO_IBITMAP_CORRUPT                \
  60         (1 << EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT)
  61 +#define EXT4_GROUP_INFO_BBITMAP_READ_BIT       4
  62
  63  #define EXT4_MB_GRP_NEED_INIT(grp)     \
  64         (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state)))
  65 @@ -3065,6 +3070,10 @@ struct ext4_group_info {
  66         (set_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
  67  #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \
  68         (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
  69 +#define EXT4_MB_GRP_TEST(grp)  \
  70 +       (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
  71 +#define EXT4_MB_GRP_TEST_AND_SET_READ(grp)     \
  72 +       (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
  73
  74  #define EXT4_MAX_CONTENTION            8
  75  #define EXT4_CONTENTION_THRESHOLD      2
  76 --- linux-4.18/fs/ext4/mballoc.c        2019-11-28 14:55:26.500545920 +0300
  77 +++ linux-4.18/fs/ext4/mballoc.c        2019-12-02 11:21:46.656897291 +0300
  78 @@ -868,7 +868,7 @@ static int ext4_mb_init_cache(struct
  79                         bh[i] = NULL;
  80                         continue;
  81                 }
  82 -               bh[i] = ext4_read_block_bitmap_nowait(sb, group);
  83 +               bh[i] = ext4_read_block_bitmap_nowait(sb, group, 0);
  84                 if (IS_ERR(bh[i])) {
  85                         err = PTR_ERR(bh[i]);
  86                         bh[i] = NULL;
  87 @@ -2104,6 +2112,92 @@ static int ext4_mb_good_group(struct
  88         return 0;
  89  }
  90
  91 +/*
  92 + * each allocation context (i.e. a thread doing allocation) has own
  93 + * sliding prefetch window of @s_mb_prefetch size which starts at the
  94 + * very first goal and moves ahead of scaning.
  95 + * a side effect is that subsequent allocations will likely find
  96 + * the bitmaps in cache or at least in-flight.
  97 + */
  98 +static void
  99 +ext4_mb_prefetch(struct ext4_allocation_context *ac,
 100 +                   ext4_group_t start)
 101 +{
 102 +       struct super_block *sb = ac->ac_sb;
 103 +       ext4_group_t ngroups = ext4_get_groups_count(sb);
 104 +       struct ext4_sb_info *sbi = EXT4_SB(sb);
 105 +       struct ext4_group_info *grp;
 106 +       ext4_group_t group = start;
 107 +       struct buffer_head *bh;
 108 +       int nr;
 109 +
 110 +       /* limit prefetching at cr=0, otherwise mballoc can
 111 +        * spend a lot of time loading imperfect groups */
 112 +       if (ac->ac_criteria < 2 && ac->ac_prefetch_ios >= sbi->s_mb_prefetch_limit)
 113 +               return;
 114 +
 115 +       /* batch prefetching to get few READs in flight */
 116 +       nr = ac->ac_prefetch - group;
 117 +       if (ac->ac_prefetch < group)
 118 +               /* wrapped to the first groups */
 119 +               nr += ngroups;
 120 +       if (nr > 0)
 121 +               return;
 122 +       BUG_ON(nr < 0);
 123 +
 124 +       nr = sbi->s_mb_prefetch;
 125 +       if (ext4_has_feature_flex_bg(sb)) {
 126 +               /* align to flex_bg to get more bitmas with a single IO */
 127 +               nr = (group / sbi->s_mb_prefetch) * sbi->s_mb_prefetch;
 128 +               nr = nr + sbi->s_mb_prefetch - group;
 129 +       }
 130 +       while (nr-- > 0) {
 131 +               grp = ext4_get_group_info(sb, group);
 132 +               /* prevent expensive getblk() on groups w/ IO in progress */
 133 +               if (EXT4_MB_GRP_TEST(grp) || EXT4_MB_GRP_TEST_AND_SET_READ(grp))
 134 +                       goto next;
 135 +
 136 +               /* ignore empty groups - those will be skipped
 137 +                * during the scanning as well */
 138 +               if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
 139 +                       bh = ext4_read_block_bitmap_nowait(sb, group, 1);
 140 +                       if (bh && !IS_ERR(bh)) {
 141 +                               if (!buffer_uptodate(bh))
 142 +                                       ac->ac_prefetch_ios++;
 143 +                               brelse(bh);
 144 +                       }
 145 +               }
 146 +next:
 147 +               if (++group >= ngroups)
 148 +                       group = 0;
 149 +       }
 150 +       ac->ac_prefetch = group;
 151 +}
 152 +
 153 +static void
 154 +ext4_mb_prefetch_fini(struct ext4_allocation_context *ac)
 155 +{
 156 +       struct ext4_group_info *grp;
 157 +       ext4_group_t group;
 158 +       int nr, rc;
 159 +
 160 +       /* initialize last window of prefetched groups */
 161 +       nr = ac->ac_prefetch_ios;
 162 +       if (nr > EXT4_SB(ac->ac_sb)->s_mb_prefetch)
 163 +               nr = EXT4_SB(ac->ac_sb)->s_mb_prefetch;
 164 +       group = ac->ac_prefetch;
 165 +       while (nr-- > 0) {
 166 +               grp = ext4_get_group_info(ac->ac_sb, group);
 167 +               if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
 168 +                       rc = ext4_mb_init_group(ac->ac_sb, group, GFP_NOFS);
 169 +                       if (rc)
 170 +                               break;
 171 +               }
 172 +               if (group-- == 0)
 173 +                       group = ext4_get_groups_count(ac->ac_sb) - 1;
 174 +       }
 175 +}
 176 +
 177  static noinline_for_stack int
 178  ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
 179  {
 180 @@ -2176,6 +2264,7 @@ repeat:
 181                  * from the goal value specified
 182                  */
 183                 group = ac->ac_g_ex.fe_group;
 184 +               ac->ac_prefetch = group;
 185
 186                 for (i = 0; i < ngroups; group++, i++) {
 187                         int ret = 0;
 188 @@ -2188,6 +2277,8 @@ repeat:
 189                         if (group >= ngroups)
 190                                 group = 0;
 191
 192 +                       ext4_mb_prefetch(ac, group);
 193 +
 194                         /* This now checks without needing the buddy page */
 195                         ret = ext4_mb_good_group(ac, group, cr);
 196                         if (ret <= 0) {
 197 @@ -2260,6 +2351,8 @@ repeat:
 198  out:
 199         if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)
 200                 err = first_err;
 201 +       /* use prefetched bitmaps to init buddy so that read info is not lost */
 202 +       ext4_mb_prefetch_fini(ac);
 203         return err;
 204  }
 205
 206 @@ -2832,6 +2925,24 @@ int ext4_mb_init(struct super_block *
 207                 sbi->s_mb_large_req = sbi->s_stripe * 8;
 208                 sbi->s_mb_group_prealloc = sbi->s_stripe * 4;
 209         }
 210 +       if (ext4_has_feature_flex_bg(sb)) {
 211 +               /* a single flex group is supposed to be read by a single IO */
 212 +               sbi->s_mb_prefetch = 1 << sbi->s_es->s_log_groups_per_flex;
 213 +               sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */
 214 +       } else {
 215 +               sbi->s_mb_prefetch = 32;
 216 +       }
 217 +       if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))
 218 +               sbi->s_mb_prefetch = ext4_get_groups_count(sb);
 219 +       /* now many real IOs to prefetch within a single allocation at cr=0
 220 +        * given cr=0 is an CPU-related optimization we shouldn't try to
 221 +        * load too many groups, at some point we should start to use what
 222 +        * we've got in memory.
 223 +        * with an average random access time 5ms, it'd take a second to get
 224 +        * 200 groups (* N with flex_bg), so let's make this limit 4 */
 225 +       sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;
 226 +       if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))
 227 +               sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);
 228
 229         sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
 230         if (sbi->s_locality_groups == NULL) {
 231 --- linux-4.18/fs/ext4/mballoc.h        2019-11-28 14:55:26.471545362 +0300
 232 +++ linux-4.18/fs/ext4/mballoc.h        2019-12-02 11:21:57.028104886 +0300
 233 @@ -177,6 +177,8 @@ struct ext4_allocation_context {
 234         struct page *ac_buddy_page;
 235         struct ext4_prealloc_space *ac_pa;
 236         struct ext4_locality_group *ac_lg;
 237 +       ext4_group_t ac_prefetch;
 238 +       int ac_prefetch_ios; /* number of initialied prefetch IO */
 239  };
 240
 241  #define AC_STATUS_CONTINUE     1
 242 --- linux-4.18/fs/ext4/sysfs.c  2019-11-28 14:55:26.502545959 +0300
 243 +++ linux-4.18/fs/ext4/sysfs.c  2019-11-28 20:07:48.104558177 +0300
 244 @@ -190,6 +190,8 @@ EXT4_RW_ATTR_SBI_UI(msg_ratelimit_bur
 245  EXT4_RO_ATTR_ES_UI(errors_count, s_error_count);
 246  EXT4_RO_ATTR_ES_UI(first_error_time, s_first_error_time);
 247  EXT4_RO_ATTR_ES_UI(last_error_time, s_last_error_time);
 248 +EXT4_RW_ATTR_SBI_UI(mb_prefetch, s_mb_prefetch);
 249 +EXT4_RW_ATTR_SBI_UI(mb_prefetch_limit, s_mb_prefetch_limit);
 250
 251  static unsigned int old_bump_val = 128;
 252  EXT4_ATTR_PTR(max_writeback_mb_bump, 0444, pointer_ui, &old_bump_val);
 253 @@ -223,6 +224,8 @@ static struct attribute *ext4_attrs[]
 254         ATTR_LIST(errors_count),
 255         ATTR_LIST(first_error_time),
 256         ATTR_LIST(last_error_time),
 257 +       ATTR_LIST(mb_prefetch),
 258 +       ATTR_LIST(mb_prefetch_limit),
 259         NULL,
 260  };
 261