1 --- linux-4.18/fs/ext4/balloc.c 2019-11-28 14:55:26.506546036 +0300
2 +++ linux-4.18/fs/ext4/balloc.c 2019-12-02 11:21:50.565975537 +0300
3 @@ -404,7 +404,8 @@ verified:
4 * Return buffer_head on success or NULL in case of failure.
7 -ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group)
8 +ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group,
11 struct ext4_group_desc *desc;
12 struct ext4_sb_info *sbi = EXT4_SB(sb);
13 @@ -435,6 +436,13 @@ ext4_read_block_bitmap_nowait(struct
14 if (bitmap_uptodate(bh))
17 + if (ignore_locked && buffer_locked(bh)) {
18 + /* buffer under IO already, do not wait
19 + * if called for prefetching */
25 if (bitmap_uptodate(bh)) {
27 @@ -524,7 +532,7 @@ ext4_read_block_bitmap(struct super_b
28 struct buffer_head *bh;
31 - bh = ext4_read_block_bitmap_nowait(sb, block_group);
32 + bh = ext4_read_block_bitmap_nowait(sb, block_group, 1);
35 err = ext4_wait_block_bitmap(sb, block_group, bh);
36 --- linux-4.18/fs/ext4/ext4.h 2019-11-28 14:55:26.470545343 +0300
37 +++ linux-4.18/fs/ext4/ext4.h 2019-12-02 11:21:40.795779972 +0300
38 @@ -1446,6 +1446,8 @@ struct ext4_sb_info {
39 /* where last allocation was done - for stream allocation */
40 unsigned long s_mb_last_group;
41 unsigned long s_mb_last_start;
42 + unsigned int s_mb_prefetch;
43 + unsigned int s_mb_prefetch_limit;
45 /* stats for buddy allocator */
46 atomic_t s_bal_reqs; /* number of reqs with len > 1 */
47 @@ -2401,7 +2403,8 @@ extern struct ext4_group_desc * ldisk
48 extern int ext4_should_retry_alloc(struct super_block *sb, int *retries);
50 extern struct buffer_head *ext4_read_block_bitmap_nowait(struct super_block *sb,
51 - ext4_group_t block_group);
52 + ext4_group_t block_group,
54 extern int ext4_wait_block_bitmap(struct super_block *sb,
55 ext4_group_t block_group,
56 struct buffer_head *bh);
57 @@ -3047,6 +3051,7 @@ struct ext4_group_info {
58 #define EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT 3
59 #define EXT4_GROUP_INFO_IBITMAP_CORRUPT \
60 (1 << EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT)
61 +#define EXT4_GROUP_INFO_BBITMAP_READ_BIT 4
63 #define EXT4_MB_GRP_NEED_INIT(grp) \
64 (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state)))
65 @@ -3065,6 +3070,10 @@ struct ext4_group_info {
66 (set_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
67 #define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \
68 (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
69 +#define EXT4_MB_GRP_TEST(grp) \
70 + (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
71 +#define EXT4_MB_GRP_TEST_AND_SET_READ(grp) \
72 + (test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
74 #define EXT4_MAX_CONTENTION 8
75 #define EXT4_CONTENTION_THRESHOLD 2
76 --- linux-4.18/fs/ext4/mballoc.c 2019-11-28 14:55:26.500545920 +0300
77 +++ linux-4.18/fs/ext4/mballoc.c 2019-12-02 11:21:46.656897291 +0300
78 @@ -868,7 +868,7 @@ static int ext4_mb_init_cache(struct
82 - if (!(bh[i] = ext4_read_block_bitmap_nowait(sb, group))) {
83 + if (!(bh[i] = ext4_read_block_bitmap_nowait(sb, group, 0))) {
87 @@ -2104,6 +2112,92 @@ static int ext4_mb_good_group(struct
92 + * each allocation context (i.e. a thread doing allocation) has own
93 + * sliding prefetch window of @s_mb_prefetch size which starts at the
94 + * very first goal and moves ahead of scaning.
95 + * a side effect is that subsequent allocations will likely find
96 + * the bitmaps in cache or at least in-flight.
99 +ext4_mb_prefetch(struct ext4_allocation_context *ac,
100 + ext4_group_t start)
102 + struct super_block *sb = ac->ac_sb;
103 + ext4_group_t ngroups = ext4_get_groups_count(sb);
104 + struct ext4_sb_info *sbi = EXT4_SB(sb);
105 + struct ext4_group_info *grp;
106 + ext4_group_t group = start;
107 + struct buffer_head *bh;
110 + /* limit prefetching at cr=0, otherwise mballoc can
111 + * spend a lot of time loading imperfect groups */
112 + if (ac->ac_criteria < 2 && ac->ac_prefetch_ios >= sbi->s_mb_prefetch_limit)
115 + /* batch prefetching to get few READs in flight */
116 + nr = ac->ac_prefetch - group;
117 + if (ac->ac_prefetch < group)
118 + /* wrapped to the first groups */
124 + nr = sbi->s_mb_prefetch;
125 + if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_FLEX_BG)) {
126 + /* align to flex_bg to get more bitmas with a single IO */
127 + nr = (group / sbi->s_mb_prefetch) * sbi->s_mb_prefetch;
128 + nr = nr + sbi->s_mb_prefetch - group;
131 + grp = ext4_get_group_info(sb, group);
132 + /* prevent expensive getblk() on groups w/ IO in progress */
133 + if (EXT4_MB_GRP_TEST(grp) || EXT4_MB_GRP_TEST_AND_SET_READ(grp))
136 + /* ignore empty groups - those will be skipped
137 + * during the scanning as well */
138 + if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
139 + bh = ext4_read_block_bitmap_nowait(sb, group, 1);
140 + if (bh && !IS_ERR(bh)) {
141 + if (!buffer_uptodate(bh))
142 + ac->ac_prefetch_ios++;
147 + if (++group >= ngroups)
150 + ac->ac_prefetch = group;
154 +ext4_mb_prefetch_fini(struct ext4_allocation_context *ac)
156 + struct ext4_group_info *grp;
157 + ext4_group_t group;
160 + /* initialize last window of prefetched groups */
161 + nr = ac->ac_prefetch_ios;
162 + if (nr > EXT4_SB(ac->ac_sb)->s_mb_prefetch)
163 + nr = EXT4_SB(ac->ac_sb)->s_mb_prefetch;
164 + group = ac->ac_prefetch;
166 + grp = ext4_get_group_info(ac->ac_sb, group);
167 + if (grp->bb_free > 0 && EXT4_MB_GRP_NEED_INIT(grp)) {
168 + rc = ext4_mb_init_group(ac->ac_sb, group);
173 + group = ext4_get_groups_count(ac->ac_sb) - 1;
177 static noinline_for_stack int
178 ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
180 @@ -2176,6 +2264,7 @@ repeat:
181 * from the goal value specified
183 group = ac->ac_g_ex.fe_group;
184 + ac->ac_prefetch = group;
186 for (i = 0; i < ngroups; group++, i++) {
188 @@ -2188,6 +2277,8 @@ repeat:
189 if (group >= ngroups)
192 + ext4_mb_prefetch(ac, group);
194 /* This now checks without needing the buddy page */
195 ret = ext4_mb_good_group(ac, group, cr);
197 @@ -2260,6 +2351,8 @@ repeat:
201 + /* use prefetched bitmaps to init buddy so that read info is not lost */
202 + ext4_mb_prefetch_fini(ac);
206 @@ -2832,6 +2925,22 @@ int ext4_mb_init(struct super_block *
207 sbi->s_mb_large_req = sbi->s_stripe * 8;
208 sbi->s_mb_group_prealloc = sbi->s_stripe * 4;
210 + if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_FLEX_BG)) {
211 + /* a single flex group is supposed to be read by a single IO */
212 + sbi->s_mb_prefetch = 1 << sbi->s_es->s_log_groups_per_flex;
213 + sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */
215 + sbi->s_mb_prefetch = 32;
217 + if (sbi->s_mb_prefetch >= ext4_get_groups_count(sb))
218 + sbi->s_mb_prefetch = ext4_get_groups_count(sb);
219 + /* now many real IOs to prefetch within a single allocation at cr=0
220 + * given cr=0 is an CPU-related optimization we shouldn't try to
221 + * load too many groups, at some point we should start to use what
222 + * we've got in memory.
223 + * with an average random access time 5ms, it'd take a second to get
224 + * 200 groups (* N with flex_bg), so let's make this limit 32 */
225 + sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 32;
227 sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
228 if (sbi->s_locality_groups == NULL) {
229 --- linux-4.18/fs/ext4/mballoc.h 2019-11-28 14:55:26.471545362 +0300
230 +++ linux-4.18/fs/ext4/mballoc.h 2019-12-02 11:21:57.028104886 +0300
231 @@ -177,6 +177,8 @@ struct ext4_allocation_context {
232 struct page *ac_buddy_page;
233 struct ext4_prealloc_space *ac_pa;
234 struct ext4_locality_group *ac_lg;
235 + ext4_group_t ac_prefetch;
236 + int ac_prefetch_ios; /* number of initialied prefetch IO */
239 #define AC_STATUS_CONTINUE 1
240 --- linux-4.18/fs/ext4/super.c 2019-11-28 14:55:26.502545959 +0300
241 +++ linux-4.18/fs/ext4/super.c 2019-11-28 20:07:48.104558177 +0300
242 @@ -190,6 +190,8 @@ EXT4_RW_ATTR_SBI_UI(msg_ratelimit_bur
243 EXT4_RW_ATTR_SBI_UI(mb_small_req, s_mb_small_req);
244 EXT4_RW_ATTR_SBI_UI(mb_large_req, s_mb_large_req);
245 EXT4_RW_ATTR_SBI_UI(mb_group_prealloc, s_mb_group_prealloc);
246 +EXT4_RW_ATTR_SBI_UI(mb_prefetch, s_mb_prefetch);
247 +EXT4_RW_ATTR_SBI_UI(mb_prefetch_limit, s_mb_prefetch_limit);
248 EXT4_DEPRECATED_ATTR(max_writeback_mb_bump, 128);
249 EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb);
250 EXT4_ATTR(trigger_fs_error, 0200, NULL, trigger_test_error);
251 @@ -223,6 +224,8 @@ static struct attribute *ext4_attrs[]
252 ATTR_LIST(errors_count),
253 ATTR_LIST(first_error_time),
254 ATTR_LIST(last_error_time),
255 + ATTR_LIST(mb_prefetch),
256 + ATTR_LIST(mb_prefetch_limit),