Whamcloud - gitweb
LU-14072 llite: fix client evicition with DIO
[fs/lustre-release.git] / ldiskfs / kernel_patches / patches / rhel8 / ext4-pdirop.patch
1 Single directory performance is a critical for HPC workloads. In a
2 typical use case an application creates a separate output file for
3 each node and task in a job. As nodes and tasks increase, hundreds
4 of thousands of files may be created in a single directory within
5 a short window of time.
6 Today, both filename lookup and file system modifying operations
7 (such as create and unlink) are protected with a single lock for
8 an entire ldiskfs directory. PDO project will remove this
9 bottleneck by introducing a parallel locking mechanism for entire
10 ldiskfs directories. This work will enable multiple application
11 threads to simultaneously lookup, create and unlink in parallel.
12
13 This patch contains:
14  - pdirops support for ldiskfs
15  - integrate with osd-ldiskfs
16
17 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/Makefile
18 ===================================================================
19 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/Makefile
20 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/Makefile
21 @@ -7,6 +7,7 @@ obj-$(CONFIG_EXT4_FS) += ext4.o
22  
23  ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \
24                 extents_status.o file.o fsmap.o fsync.o hash.o ialloc.o \
25 +               htree_lock.o \
26                 indirect.o inline.o inode.o ioctl.o mballoc.o migrate.o \
27                 mmp.o move_extent.o namei.o page-io.o readpage.o resize.o \
28                 super.o symlink.o sysfs.o xattr.o xattr_trusted.o xattr_user.o
29 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/ext4.h
30 ===================================================================
31 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/ext4.h
32 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/ext4.h
33 @@ -29,6 +29,7 @@
34  #include <linux/timer.h>
35  #include <linux/version.h>
36  #include <linux/wait.h>
37 +#include <linux/htree_lock.h>
38  #include <linux/sched/signal.h>
39  #include <linux/blockgroup_lock.h>
40  #include <linux/percpu_counter.h>
41 @@ -934,6 +935,9 @@ struct ext4_inode_info {
42         __u32   i_dtime;
43         ext4_fsblk_t    i_file_acl;
44  
45 +       /* following fields for parallel directory operations -bzzz */
46 +       struct semaphore i_append_sem;
47 +
48         /*
49          * i_block_group is the number of the block group which contains
50          * this file's inode.  Constant across the lifetime of the inode,
51 @@ -2109,6 +2113,72 @@ struct dx_hash_info
52   */
53  #define HASH_NB_ALWAYS         1
54  
55 +/* assume name-hash is protected by upper layer */
56 +#define EXT4_HTREE_LOCK_HASH   0
57 +
58 +enum ext4_pdo_lk_types {
59 +#if EXT4_HTREE_LOCK_HASH
60 +       EXT4_LK_HASH,
61 +#endif
62 +       EXT4_LK_DX,             /* index block */
63 +       EXT4_LK_DE,             /* directory entry block */
64 +       EXT4_LK_SPIN,           /* spinlock */
65 +       EXT4_LK_MAX,
66 +};
67 +
68 +/* read-only bit */
69 +#define EXT4_LB_RO(b)          (1 << (b))
70 +/* read + write, high bits for writer */
71 +#define EXT4_LB_RW(b)          ((1 << (b)) | (1 << (EXT4_LK_MAX + (b))))
72 +
73 +enum ext4_pdo_lock_bits {
74 +       /* DX lock bits */
75 +       EXT4_LB_DX_RO           = EXT4_LB_RO(EXT4_LK_DX),
76 +       EXT4_LB_DX              = EXT4_LB_RW(EXT4_LK_DX),
77 +       /* DE lock bits */
78 +       EXT4_LB_DE_RO           = EXT4_LB_RO(EXT4_LK_DE),
79 +       EXT4_LB_DE              = EXT4_LB_RW(EXT4_LK_DE),
80 +       /* DX spinlock bits */
81 +       EXT4_LB_SPIN_RO         = EXT4_LB_RO(EXT4_LK_SPIN),
82 +       EXT4_LB_SPIN            = EXT4_LB_RW(EXT4_LK_SPIN),
83 +       /* accurate searching */
84 +       EXT4_LB_EXACT           = EXT4_LB_RO(EXT4_LK_MAX << 1),
85 +};
86 +
87 +enum ext4_pdo_lock_opc {
88 +       /* external */
89 +       EXT4_HLOCK_READDIR      = (EXT4_LB_DE_RO | EXT4_LB_DX_RO),
90 +       EXT4_HLOCK_LOOKUP       = (EXT4_LB_DE_RO | EXT4_LB_SPIN_RO |
91 +                                  EXT4_LB_EXACT),
92 +       EXT4_HLOCK_DEL          = (EXT4_LB_DE | EXT4_LB_SPIN_RO |
93 +                                  EXT4_LB_EXACT),
94 +       EXT4_HLOCK_ADD          = (EXT4_LB_DE | EXT4_LB_SPIN_RO),
95 +
96 +       /* internal */
97 +       EXT4_HLOCK_LOOKUP_SAFE  = (EXT4_LB_DE_RO | EXT4_LB_DX_RO |
98 +                                  EXT4_LB_EXACT),
99 +       EXT4_HLOCK_DEL_SAFE     = (EXT4_LB_DE | EXT4_LB_DX_RO | EXT4_LB_EXACT),
100 +       EXT4_HLOCK_SPLIT        = (EXT4_LB_DE | EXT4_LB_DX | EXT4_LB_SPIN),
101 +};
102 +
103 +extern struct htree_lock_head *ext4_htree_lock_head_alloc(unsigned hbits);
104 +#define ext4_htree_lock_head_free(lhead)       htree_lock_head_free(lhead)
105 +
106 +extern struct htree_lock *ext4_htree_lock_alloc(void);
107 +#define ext4_htree_lock_free(lck)              htree_lock_free(lck)
108 +
109 +extern void ext4_htree_lock(struct htree_lock *lck,
110 +                           struct htree_lock_head *lhead,
111 +                           struct inode *dir, unsigned flags);
112 +#define ext4_htree_unlock(lck)                  htree_unlock(lck)
113 +
114 +extern struct buffer_head *__ext4_find_entry(struct inode *dir,
115 +                                       const struct qstr *d_name,
116 +                                       struct ext4_dir_entry_2 **res_dir,
117 +                                       int *inlined, struct htree_lock *lck);
118 +extern int __ext4_add_entry(handle_t *handle, struct dentry *dentry,
119 +                     struct inode *inode, struct htree_lock *lck);
120 +
121  struct ext4_filename {
122         const struct qstr *usr_fname;
123         struct fscrypt_str disk_name;
124 @@ -2416,8 +2486,16 @@ void ext4_insert_dentry(struct inode *in
125                         struct ext4_filename *fname, void *data);
126  static inline void ext4_update_dx_flag(struct inode *inode)
127  {
128 +       /* Disable it for ldiskfs, because going from a DX directory to
129 +        * a non-DX directory while it is in use will completely break
130 +        * the htree-locking.
131 +        * If we really want to support this operation in the future,
132 +        * we need to exclusively lock the directory at here which will
133 +        * increase complexity of code */
134 +#if 0
135         if (!ext4_has_feature_dir_index(inode->i_sb))
136                 ext4_clear_inode_flag(inode, EXT4_INODE_INDEX);
137 +#endif
138  }
139  static const unsigned char ext4_filetype_table[] = {
140         DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
141 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/namei.c
142 ===================================================================
143 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/namei.c
144 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/namei.c
145 @@ -54,6 +54,7 @@ struct buffer_head *ext4_append(handle_t
146                                         ext4_lblk_t *block)
147  {
148         struct buffer_head *bh;
149 +       struct ext4_inode_info *ei = EXT4_I(inode);
150         int err;
151  
152         if (unlikely(EXT4_SB(inode->i_sb)->s_max_dir_size_kb &&
153 @@ -61,15 +62,22 @@ struct buffer_head *ext4_append(handle_t
154                       EXT4_SB(inode->i_sb)->s_max_dir_size_kb)))
155                 return ERR_PTR(-ENOSPC);
156  
157 +       /* with parallel dir operations all appends
158 +       * have to be serialized -bzzz */
159 +       down(&ei->i_append_sem);
160 +
161         *block = inode->i_size >> inode->i_sb->s_blocksize_bits;
162  
163         bh = ext4_bread(handle, inode, *block, EXT4_GET_BLOCKS_CREATE);
164 -       if (IS_ERR(bh))
165 +       if (IS_ERR(bh)) {
166 +               up(&ei->i_append_sem);
167                 return bh;
168 +       }
169         inode->i_size += inode->i_sb->s_blocksize;
170         EXT4_I(inode)->i_disksize = inode->i_size;
171         BUFFER_TRACE(bh, "get_write_access");
172         err = ext4_journal_get_write_access(handle, bh);
173 +       up(&ei->i_append_sem);
174         if (err) {
175                 brelse(bh);
176                 ext4_std_error(inode->i_sb, err);
177 @@ -250,7 +258,8 @@ static unsigned dx_node_limit(struct ino
178  static struct dx_frame *dx_probe(struct ext4_filename *fname,
179                                  struct inode *dir,
180                                  struct dx_hash_info *hinfo,
181 -                                struct dx_frame *frame);
182 +                                struct dx_frame *frame,
183 +                                struct htree_lock *lck);
184  static void dx_release(struct dx_frame *frames);
185  static int dx_make_map(struct inode *dir, struct ext4_dir_entry_2 *de,
186                        unsigned blocksize, struct dx_hash_info *hinfo,
187 @@ -264,12 +273,13 @@ static void dx_insert_block(struct dx_fr
188  static int ext4_htree_next_block(struct inode *dir, __u32 hash,
189                                  struct dx_frame *frame,
190                                  struct dx_frame *frames,
191 -                                __u32 *start_hash);
192 +                                __u32 *start_hash, struct htree_lock *lck);
193  static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
194                 struct ext4_filename *fname,
195 -               struct ext4_dir_entry_2 **res_dir);
196 +               struct ext4_dir_entry_2 **res_dir, struct htree_lock *lck);
197  static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
198 -                            struct inode *dir, struct inode *inode);
199 +                            struct inode *dir, struct inode *inode,
200 +                            struct htree_lock *lck);
201  
202  /* checksumming functions */
203  void initialize_dirent_tail(struct ext4_dir_entry_tail *t,
204 @@ -733,6 +743,227 @@ struct stats dx_show_entries(struct dx_h
205  }
206  #endif /* DX_DEBUG */
207  
208 +/* private data for htree_lock */
209 +struct ext4_dir_lock_data {
210 +       unsigned                ld_flags;  /* bits-map for lock types */
211 +       unsigned                ld_count;  /* # entries of the last DX block */
212 +       struct dx_entry         ld_at_entry; /* copy of leaf dx_entry */
213 +       struct dx_entry         *ld_at;    /* position of leaf dx_entry */
214 +};
215 +
216 +#define ext4_htree_lock_data(l)        ((struct ext4_dir_lock_data *)(l)->lk_private)
217 +#define ext4_find_entry(dir, name, dirent, inline) \
218 +                       __ext4_find_entry(dir, name, dirent, inline, NULL)
219 +#define ext4_add_entry(handle, dentry, inode) \
220 +                       __ext4_add_entry(handle, dentry, inode, NULL)
221 +
222 +/* NB: ext4_lblk_t is 32 bits so we use high bits to identify invalid blk */
223 +#define EXT4_HTREE_NODE_CHANGED        (0xcafeULL << 32)
224 +
225 +static void ext4_htree_event_cb(void *target, void *event)
226 +{
227 +       u64 *block = (u64 *)target;
228 +
229 +       if (*block == dx_get_block((struct dx_entry *)event))
230 +               *block = EXT4_HTREE_NODE_CHANGED;
231 +}
232 +
233 +struct htree_lock_head *ext4_htree_lock_head_alloc(unsigned hbits)
234 +{
235 +       struct htree_lock_head *lhead;
236 +
237 +       lhead = htree_lock_head_alloc(EXT4_LK_MAX, hbits, 0);
238 +       if (lhead != NULL) {
239 +               htree_lock_event_attach(lhead, EXT4_LK_SPIN, HTREE_EVENT_WR,
240 +                                       ext4_htree_event_cb);
241 +       }
242 +       return lhead;
243 +}
244 +EXPORT_SYMBOL(ext4_htree_lock_head_alloc);
245 +
246 +struct htree_lock *ext4_htree_lock_alloc(void)
247 +{
248 +       return htree_lock_alloc(EXT4_LK_MAX,
249 +                               sizeof(struct ext4_dir_lock_data));
250 +}
251 +EXPORT_SYMBOL(ext4_htree_lock_alloc);
252 +
253 +static htree_lock_mode_t ext4_htree_mode(unsigned flags)
254 +{
255 +       switch (flags) {
256 +       default: /* 0 or unknown flags require EX lock */
257 +               return HTREE_LOCK_EX;
258 +       case EXT4_HLOCK_READDIR:
259 +               return HTREE_LOCK_PR;
260 +       case EXT4_HLOCK_LOOKUP:
261 +               return HTREE_LOCK_CR;
262 +       case EXT4_HLOCK_DEL:
263 +       case EXT4_HLOCK_ADD:
264 +               return HTREE_LOCK_CW;
265 +       }
266 +}
267 +
268 +/* return PR for read-only operations, otherwise return EX */
269 +static inline htree_lock_mode_t ext4_htree_safe_mode(unsigned flags)
270 +{
271 +       int writer = (flags & EXT4_LB_DE) == EXT4_LB_DE;
272 +
273 +       /* 0 requires EX lock */
274 +       return (flags == 0 || writer) ? HTREE_LOCK_EX : HTREE_LOCK_PR;
275 +}
276 +
277 +static int ext4_htree_safe_locked(struct htree_lock *lck)
278 +{
279 +       int writer;
280 +
281 +       if (lck == NULL || lck->lk_mode == HTREE_LOCK_EX)
282 +               return 1;
283 +
284 +       writer = (ext4_htree_lock_data(lck)->ld_flags & EXT4_LB_DE) ==
285 +                EXT4_LB_DE;
286 +       if (writer) /* all readers & writers are excluded? */
287 +               return lck->lk_mode == HTREE_LOCK_EX;
288 +
289 +       /* all writers are excluded? */
290 +       return lck->lk_mode == HTREE_LOCK_PR ||
291 +              lck->lk_mode == HTREE_LOCK_PW ||
292 +              lck->lk_mode == HTREE_LOCK_EX;
293 +}
294 +
295 +/* relock htree_lock with EX mode if it's change operation, otherwise
296 + * relock it with PR mode. It's noop if PDO is disabled. */
297 +static void ext4_htree_safe_relock(struct htree_lock *lck)
298 +{
299 +       if (!ext4_htree_safe_locked(lck)) {
300 +               unsigned flags = ext4_htree_lock_data(lck)->ld_flags;
301 +
302 +               htree_change_lock(lck, ext4_htree_safe_mode(flags));
303 +       }
304 +}
305 +
306 +void ext4_htree_lock(struct htree_lock *lck, struct htree_lock_head *lhead,
307 +                    struct inode *dir, unsigned flags)
308 +{
309 +       htree_lock_mode_t mode = is_dx(dir) ? ext4_htree_mode(flags) :
310 +                                             ext4_htree_safe_mode(flags);
311 +
312 +       ext4_htree_lock_data(lck)->ld_flags = flags;
313 +       htree_lock(lck, lhead, mode);
314 +       if (!is_dx(dir))
315 +               ext4_htree_safe_relock(lck); /* make sure it's safe locked */
316 +}
317 +EXPORT_SYMBOL(ext4_htree_lock);
318 +
319 +static int ext4_htree_node_lock(struct htree_lock *lck, struct dx_entry *at,
320 +                               unsigned lmask, int wait, void *ev)
321 +{
322 +       u32     key = (at == NULL) ? 0 : dx_get_block(at);
323 +       u32     mode;
324 +
325 +       /* NOOP if htree is well protected or caller doesn't require the lock */
326 +       if (ext4_htree_safe_locked(lck) ||
327 +          !(ext4_htree_lock_data(lck)->ld_flags & lmask))
328 +               return 1;
329 +
330 +       mode = (ext4_htree_lock_data(lck)->ld_flags & lmask) == lmask ?
331 +               HTREE_LOCK_PW : HTREE_LOCK_PR;
332 +       while (1) {
333 +               if (htree_node_lock_try(lck, mode, key, ffz(~lmask), wait, ev))
334 +                       return 1;
335 +               if (!(lmask & EXT4_LB_SPIN)) /* not a spinlock */
336 +                       return 0;
337 +               cpu_relax(); /* spin until granted */
338 +       }
339 +}
340 +
341 +static int ext4_htree_node_locked(struct htree_lock *lck, unsigned lmask)
342 +{
343 +       return ext4_htree_safe_locked(lck) ||
344 +              htree_node_is_granted(lck, ffz(~lmask));
345 +}
346 +
347 +static void ext4_htree_node_unlock(struct htree_lock *lck,
348 +                                  unsigned lmask, void *buf)
349 +{
350 +       /* NB: it's safe to call mutiple times or even it's not locked */
351 +       if (!ext4_htree_safe_locked(lck) &&
352 +            htree_node_is_granted(lck, ffz(~lmask)))
353 +               htree_node_unlock(lck, ffz(~lmask), buf);
354 +}
355 +
356 +#define ext4_htree_dx_lock(lck, key)           \
357 +       ext4_htree_node_lock(lck, key, EXT4_LB_DX, 1, NULL)
358 +#define ext4_htree_dx_lock_try(lck, key)       \
359 +       ext4_htree_node_lock(lck, key, EXT4_LB_DX, 0, NULL)
360 +#define ext4_htree_dx_unlock(lck)              \
361 +       ext4_htree_node_unlock(lck, EXT4_LB_DX, NULL)
362 +#define ext4_htree_dx_locked(lck)              \
363 +       ext4_htree_node_locked(lck, EXT4_LB_DX)
364 +
365 +static void ext4_htree_dx_need_lock(struct htree_lock *lck)
366 +{
367 +       struct ext4_dir_lock_data *ld;
368 +
369 +       if (ext4_htree_safe_locked(lck))
370 +               return;
371 +
372 +       ld = ext4_htree_lock_data(lck);
373 +       switch (ld->ld_flags) {
374 +       default:
375 +               return;
376 +       case EXT4_HLOCK_LOOKUP:
377 +               ld->ld_flags = EXT4_HLOCK_LOOKUP_SAFE;
378 +               return;
379 +       case EXT4_HLOCK_DEL:
380 +               ld->ld_flags = EXT4_HLOCK_DEL_SAFE;
381 +               return;
382 +       case EXT4_HLOCK_ADD:
383 +               ld->ld_flags = EXT4_HLOCK_SPLIT;
384 +               return;
385 +       }
386 +}
387 +
388 +#define ext4_htree_de_lock(lck, key)           \
389 +       ext4_htree_node_lock(lck, key, EXT4_LB_DE, 1, NULL)
390 +#define ext4_htree_de_unlock(lck)              \
391 +       ext4_htree_node_unlock(lck, EXT4_LB_DE, NULL)
392 +
393 +#define ext4_htree_spin_lock(lck, key, event)  \
394 +       ext4_htree_node_lock(lck, key, EXT4_LB_SPIN, 0, event)
395 +#define ext4_htree_spin_unlock(lck)            \
396 +       ext4_htree_node_unlock(lck, EXT4_LB_SPIN, NULL)
397 +#define ext4_htree_spin_unlock_listen(lck, p)  \
398 +       ext4_htree_node_unlock(lck, EXT4_LB_SPIN, p)
399 +
400 +static void ext4_htree_spin_stop_listen(struct htree_lock *lck)
401 +{
402 +       if (!ext4_htree_safe_locked(lck) &&
403 +           htree_node_is_listening(lck, ffz(~EXT4_LB_SPIN)))
404 +               htree_node_stop_listen(lck, ffz(~EXT4_LB_SPIN));
405 +}
406 +
407 +enum {
408 +       DX_HASH_COL_IGNORE,     /* ignore collision while probing frames */
409 +       DX_HASH_COL_YES,        /* there is collision and it does matter */
410 +       DX_HASH_COL_NO,         /* there is no collision */
411 +};
412 +
413 +static int dx_probe_hash_collision(struct htree_lock *lck,
414 +                                  struct dx_entry *entries,
415 +                                  struct dx_entry *at, u32 hash)
416 +{
417 +       if (!(lck && ext4_htree_lock_data(lck)->ld_flags & EXT4_LB_EXACT)) {
418 +               return DX_HASH_COL_IGNORE; /* don't care about collision */
419 +
420 +       } else if (at == entries + dx_get_count(entries) - 1) {
421 +               return DX_HASH_COL_IGNORE; /* not in any leaf of this DX */
422 +
423 +       } else { /* hash collision? */
424 +               return ((dx_get_hash(at + 1) & ~1) == hash) ?
425 +                       DX_HASH_COL_YES : DX_HASH_COL_NO;
426 +       }
427 +}
428 +
429  /*
430   * Probe for a directory leaf block to search.
431   *
432 @@ -744,10 +975,11 @@ struct stats dx_show_entries(struct dx_h
433   */
434  static struct dx_frame *
435  dx_probe(struct ext4_filename *fname, struct inode *dir,
436 -        struct dx_hash_info *hinfo, struct dx_frame *frame_in)
437 +        struct dx_hash_info *hinfo, struct dx_frame *frame_in,
438 +        struct htree_lock *lck)
439  {
440         unsigned count, indirect;
441 -       struct dx_entry *at, *entries, *p, *q, *m;
442 +       struct dx_entry *at, *entries, *p, *q, *m, *dx = NULL;
443         struct dx_root_info *info;
444         struct dx_frame *frame = frame_in;
445         struct dx_frame *ret_err = ERR_PTR(ERR_BAD_DX_DIR);
446 @@ -809,8 +1041,15 @@ dx_probe(struct ext4_filename *fname, st
447  
448         dxtrace(printk("Look up %x", hash));
449         while (1) {
450 +               if (indirect == 0) { /* the last index level */
451 +                       /* NB: ext4_htree_dx_lock() could be noop if
452 +                        * DX-lock flag is not set for current operation */
453 +                       ext4_htree_dx_lock(lck, dx);
454 +                       ext4_htree_spin_lock(lck, dx, NULL);
455 +               }
456                 count = dx_get_count(entries);
457 -               if (!count || count > dx_get_limit(entries)) {
458 +               if (count == 0 || count > dx_get_limit(entries)) {
459 +                       ext4_htree_spin_unlock(lck); /* release spin */
460                         ext4_warning_inode(dir,
461                                            "dx entry: count %u beyond limit %u",
462                                            count, dx_get_limit(entries));
463 @@ -849,8 +1088,70 @@ dx_probe(struct ext4_filename *fname, st
464                                dx_get_block(at)));
465                 frame->entries = entries;
466                 frame->at = at;
467 -               if (!indirect--)
468 +
469 +               if (indirect == 0) { /* the last index level */
470 +                       struct ext4_dir_lock_data *ld;
471 +                       u64 myblock;
472 +
473 +                       /* By default we only lock DE-block, however, we will
474 +                        * also lock the last level DX-block if:
475 +                        * a) there is hash collision
476 +                        *    we will set DX-lock flag (a few lines below)
477 +                        *    and redo to lock DX-block
478 +                        *    see detail in dx_probe_hash_collision()
479 +                        * b) it's a retry from splitting
480 +                        *    we need to lock the last level DX-block so nobody
481 +                        *    else can split any leaf blocks under the same
482 +                        *    DX-block, see detail in ext4_dx_add_entry()
483 +                        */
484 +                       if (ext4_htree_dx_locked(lck)) {
485 +                               /* DX-block is locked, just lock DE-block
486 +                                * and return */
487 +                               ext4_htree_spin_unlock(lck);
488 +                               if (!ext4_htree_safe_locked(lck))
489 +                                       ext4_htree_de_lock(lck, frame->at);
490 +                               return frame;
491 +                       }
492 +                       /* it's pdirop and no DX lock */
493 +                       if (dx_probe_hash_collision(lck, entries, at, hash) ==
494 +                           DX_HASH_COL_YES) {
495 +                               /* found hash collision, set DX-lock flag
496 +                                * and retry to abtain DX-lock */
497 +                               ext4_htree_spin_unlock(lck);
498 +                               ext4_htree_dx_need_lock(lck);
499 +                               continue;
500 +                       }
501 +                       ld = ext4_htree_lock_data(lck);
502 +                       /* because I don't lock DX, so @at can't be trusted
503 +                        * after I release spinlock so I have to save it */
504 +                       ld->ld_at = at;
505 +                       ld->ld_at_entry = *at;
506 +                       ld->ld_count = dx_get_count(entries);
507 +
508 +                       frame->at = &ld->ld_at_entry;
509 +                       myblock = dx_get_block(at);
510 +
511 +                       /* NB: ordering locking */
512 +                       ext4_htree_spin_unlock_listen(lck, &myblock);
513 +                       /* other thread can split this DE-block because:
514 +                        * a) I don't have lock for the DE-block yet
515 +                        * b) I released spinlock on DX-block
516 +                        * if it happened I can detect it by listening
517 +                        * splitting event on this DE-block */
518 +                       ext4_htree_de_lock(lck, frame->at);
519 +                       ext4_htree_spin_stop_listen(lck);
520 +
521 +                       if (myblock == EXT4_HTREE_NODE_CHANGED) {
522 +                               /* someone split this DE-block before
523 +                                * I locked it, I need to retry and lock
524 +                                * valid DE-block */
525 +                               ext4_htree_de_unlock(lck);
526 +                               continue;
527 +                       }
528                         return frame;
529 +               }
530 +               dx = at;
531 +               indirect--;
532                 frame++;
533                 frame->bh = ext4_read_dirblock(dir, dx_get_block(at), INDEX);
534                 if (IS_ERR(frame->bh)) {
535 @@ -916,7 +1217,7 @@ static void dx_release(struct dx_frame *
536  static int ext4_htree_next_block(struct inode *dir, __u32 hash,
537                                  struct dx_frame *frame,
538                                  struct dx_frame *frames,
539 -                                __u32 *start_hash)
540 +                                __u32 *start_hash, struct htree_lock *lck)
541  {
542         struct dx_frame *p;
543         struct buffer_head *bh;
544 @@ -931,12 +1232,22 @@ static int ext4_htree_next_block(struct
545          * this loop, num_frames indicates the number of interior
546          * nodes need to be read.
547          */
548 +       ext4_htree_de_unlock(lck);
549         while (1) {
550 -               if (++(p->at) < p->entries + dx_get_count(p->entries))
551 -                       break;
552 +               if (num_frames > 0 || ext4_htree_dx_locked(lck)) {
553 +                       /* num_frames > 0 :
554 +                        *   DX block
555 +                        * ext4_htree_dx_locked:
556 +                        *   frame->at is reliable pointer returned by dx_probe,
557 +                        *   otherwise dx_probe already knew no collision */
558 +                       if (++(p->at) < p->entries + dx_get_count(p->entries))
559 +                               break;
560 +               }
561                 if (p == frames)
562                         return 0;
563                 num_frames++;
564 +               if (num_frames == 1)
565 +                       ext4_htree_dx_unlock(lck);
566                 p--;
567         }
568  
569 @@ -959,6 +1270,13 @@ static int ext4_htree_next_block(struct
570          * block so no check is necessary
571          */
572         while (num_frames--) {
573 +               if (num_frames == 0) {
574 +                       /* it's not always necessary, we just don't want to
575 +                        * detect hash collision again */
576 +                       ext4_htree_dx_need_lock(lck);
577 +                       ext4_htree_dx_lock(lck, p->at);
578 +               }
579 +
580                 bh = ext4_read_dirblock(dir, dx_get_block(p->at), INDEX);
581                 if (IS_ERR(bh))
582                         return PTR_ERR(bh);
583 @@ -967,6 +1285,7 @@ static int ext4_htree_next_block(struct
584                 p->bh = bh;
585                 p->at = p->entries = ((struct dx_node *) bh->b_data)->entries;
586         }
587 +       ext4_htree_de_lock(lck, p->at);
588         return 1;
589  }
590  
591 @@ -1114,10 +1433,10 @@ int ext4_htree_fill_tree(struct file *di
592         }
593         hinfo.hash = start_hash;
594         hinfo.minor_hash = 0;
595 -       frame = dx_probe(NULL, dir, &hinfo, frames);
596 +       /* assume it's PR locked */
597 +       frame = dx_probe(NULL, dir, &hinfo, frames, NULL);
598         if (IS_ERR(frame))
599                 return PTR_ERR(frame);
600 -
601         /* Add '.' and '..' from the htree header */
602         if (!start_hash && !start_minor_hash) {
603                 de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data;
604 @@ -1157,7 +1476,7 @@ int ext4_htree_fill_tree(struct file *di
605                 count += ret;
606                 hashval = ~0;
607                 ret = ext4_htree_next_block(dir, HASH_NB_ALWAYS,
608 -                                           frame, frames, &hashval);
609 +                                           frame, frames, &hashval, NULL);
610                 *next_hash = hashval;
611                 if (ret < 0) {
612                         err = ret;
613 @@ -1349,10 +1668,10 @@ static int is_dx_internal_node(struct in
614   * The returned buffer_head has ->b_count elevated.  The caller is expected
615   * to brelse() it when appropriate.
616   */
617 -static struct buffer_head * ext4_find_entry (struct inode *dir,
618 +struct buffer_head *__ext4_find_entry(struct inode *dir,
619                                         const struct qstr *d_name,
620                                         struct ext4_dir_entry_2 **res_dir,
621 -                                       int *inlined)
622 +                                       int *inlined, struct htree_lock *lck)
623  {
624         struct super_block *sb;
625         struct buffer_head *bh_use[NAMEI_RA_SIZE];
626 @@ -1401,7 +1720,7 @@ static struct buffer_head * ext4_find_en
627                 goto restart;
628         }
629         if (is_dx(dir)) {
630 -               ret = ext4_dx_find_entry(dir, &fname, res_dir);
631 +               ret = ext4_dx_find_entry(dir, &fname, res_dir, lck);
632                 /*
633                  * On success, or if the error was file not found,
634                  * return.  Otherwise, fall back to doing a search the
635 @@ -1411,6 +1730,7 @@ static struct buffer_head * ext4_find_en
636                         goto cleanup_and_exit;
637                 dxtrace(printk(KERN_DEBUG "ext4_find_entry: dx failed, "
638                                "falling back\n"));
639 +               ext4_htree_safe_relock(lck);
640                 ret = NULL;
641         }
642         nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb);
643 @@ -1499,10 +1819,12 @@ cleanup_and_exit:
644         ext4_fname_free_filename(&fname);
645         return ret;
646  }
647 +EXPORT_SYMBOL(__ext4_find_entry);
648  
649  static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
650                         struct ext4_filename *fname,
651 -                       struct ext4_dir_entry_2 **res_dir)
652 +                       struct ext4_dir_entry_2 **res_dir,
653 +                       struct htree_lock *lck)
654  {
655         struct super_block * sb = dir->i_sb;
656         struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
657 @@ -1513,7 +1835,7 @@ static struct buffer_head * ext4_dx_find
658  #ifdef CONFIG_EXT4_FS_ENCRYPTION
659         *res_dir = NULL;
660  #endif
661 -       frame = dx_probe(fname, dir, NULL, frames);
662 +       frame = dx_probe(fname, dir, NULL, frames, lck);
663         if (IS_ERR(frame))
664                 return (struct buffer_head *) frame;
665         do {
666 @@ -1535,7 +1857,7 @@ static struct buffer_head * ext4_dx_find
667  
668                 /* Check to see if we should continue to search */
669                 retval = ext4_htree_next_block(dir, fname->hinfo.hash, frame,
670 -                                              frames, NULL);
671 +                                              frames, NULL, lck);
672                 if (retval < 0) {
673                         ext4_warning_inode(dir,
674                                 "error %d reading directory index block",
675 @@ -1710,8 +2032,9 @@ static struct ext4_dir_entry_2* dx_pack_
676   * Returns pointer to de in block into which the new entry will be inserted.
677   */
678  static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir,
679 -                       struct buffer_head **bh,struct dx_frame *frame,
680 -                       struct dx_hash_info *hinfo)
681 +                       struct buffer_head **bh, struct dx_frame *frames,
682 +                       struct dx_frame *frame, struct dx_hash_info *hinfo,
683 +                       struct htree_lock *lck)
684  {
685         unsigned blocksize = dir->i_sb->s_blocksize;
686         unsigned count, continued;
687 @@ -1773,8 +2096,14 @@ static struct ext4_dir_entry_2 *do_split
688                                         hash2, split, count-split));
689  
690         /* Fancy dance to stay within two buffers */
691 -       de2 = dx_move_dirents(data1, data2, map + split, count - split,
692 -                             blocksize);
693 +       if (hinfo->hash < hash2) {
694 +               de2 = dx_move_dirents(data1, data2, map + split,
695 +                                     count - split, blocksize);
696 +       } else {
697 +               /* make sure we will add entry to the same block which
698 +                * we have already locked */
699 +               de2 = dx_move_dirents(data1, data2, map, split, blocksize);
700 +       }
701         de = dx_pack_dirents(data1, blocksize);
702         de->rec_len = ext4_rec_len_to_disk(data1 + (blocksize - csum_size) -
703                                            (char *) de,
704 @@ -1795,12 +2124,21 @@ static struct ext4_dir_entry_2 *do_split
705         dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data2,
706                         blocksize, 1));
707  
708 -       /* Which block gets the new entry? */
709 -       if (hinfo->hash >= hash2) {
710 -               swap(*bh, bh2);
711 -               de = de2;
712 +       ext4_htree_spin_lock(lck, frame > frames ? (frame - 1)->at : NULL,
713 +                            frame->at); /* notify block is being split */
714 +       if (hinfo->hash < hash2) {
715 +               dx_insert_block(frame, hash2 + continued, newblock);
716 +
717 +       } else {
718 +               /* switch block number */
719 +               dx_insert_block(frame, hash2 + continued,
720 +                               dx_get_block(frame->at));
721 +               dx_set_block(frame->at, newblock);
722 +               (frame->at)++;
723         }
724 -       dx_insert_block(frame, hash2 + continued, newblock);
725 +       ext4_htree_spin_unlock(lck);
726 +       ext4_htree_dx_unlock(lck);
727 +
728         err = ext4_handle_dirty_dirent_node(handle, dir, bh2);
729         if (err)
730                 goto journal_error;
731 @@ -2074,7 +2412,7 @@ static int make_indexed_dir(handle_t *ha
732         if (retval)
733                 goto out_frames;        
734  
735 -       de = do_split(handle,dir, &bh2, frame, &fname->hinfo);
736 +       de = do_split(handle, dir, &bh2, frames, frame, &fname->hinfo, NULL);
737         if (IS_ERR(de)) {
738                 retval = PTR_ERR(de);
739                 goto out_frames;
740 @@ -2184,8 +2522,8 @@ out:
741   * may not sleep between calling this and putting something into
742   * the entry, as someone else might have used it while you slept.
743   */
744 -static int ext4_add_entry(handle_t *handle, struct dentry *dentry,
745 -                         struct inode *inode)
746 +int __ext4_add_entry(handle_t *handle, struct dentry *dentry,
747 +                     struct inode *inode, struct htree_lock *lck)
748  {
749         struct inode *dir = d_inode(dentry->d_parent);
750         struct buffer_head *bh = NULL;
751 @@ -2226,9 +2564,10 @@ static int ext4_add_entry(handle_t *hand
752                 if (dentry->d_name.len == 2 &&
753                     memcmp(dentry->d_name.name, "..", 2) == 0)
754                         return ext4_update_dotdot(handle, dentry, inode);
755 -               retval = ext4_dx_add_entry(handle, &fname, dir, inode);
756 +               retval = ext4_dx_add_entry(handle, &fname, dir, inode, lck);
757                 if (!retval || (retval != ERR_BAD_DX_DIR))
758                         goto out;
759 +               ext4_htree_safe_relock(lck);
760                 ext4_clear_inode_flag(dir, EXT4_INODE_INDEX);
761                 dx_fallback++;
762                 ext4_mark_inode_dirty(handle, dir);
763 @@ -2278,12 +2617,14 @@ out:
764                 ext4_set_inode_state(inode, EXT4_STATE_NEWENTRY);
765         return retval;
766  }
767 +EXPORT_SYMBOL(__ext4_add_entry);
768  
769  /*
770   * Returns 0 for success, or a negative error value
771   */
772  static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
773 -                            struct inode *dir, struct inode *inode)
774 +                            struct inode *dir, struct inode *inode,
775 +                            struct htree_lock *lck)
776  {
777         struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
778         struct dx_entry *entries, *at;
779 @@ -2295,7 +2636,7 @@ static int ext4_dx_add_entry(handle_t *h
780  
781  again:
782         restart = 0;
783 -       frame = dx_probe(fname, dir, NULL, frames);
784 +       frame = dx_probe(fname, dir, NULL, frames, lck);
785         if (IS_ERR(frame))
786                 return PTR_ERR(frame);
787         entries = frame->entries;
788 @@ -2330,6 +2671,11 @@ again:
789                 struct dx_node *node2;
790                 struct buffer_head *bh2;
791  
792 +               if (!ext4_htree_safe_locked(lck)) { /* retry with EX lock */
793 +                       ext4_htree_safe_relock(lck);
794 +                       restart = 1;
795 +                       goto cleanup;
796 +               }
797                 while (frame > frames) {
798                         if (dx_get_count((frame - 1)->entries) <
799                             dx_get_limit((frame - 1)->entries)) {
800 @@ -2432,8 +2778,32 @@ again:
801                         restart = 1;
802                         goto journal_error;
803                 }
804 +       } else if (!ext4_htree_dx_locked(lck)) {
805 +               struct ext4_dir_lock_data *ld = ext4_htree_lock_data(lck);
806 +
807 +               /* not well protected, require DX lock */
808 +               ext4_htree_dx_need_lock(lck);
809 +               at = frame > frames ? (frame - 1)->at : NULL;
810 +
811 +               /* NB: no risk of deadlock because it's just a try.
812 +                *
813 +                * NB: we check ld_count for twice, the first time before
814 +                * having DX lock, the second time after holding DX lock.
815 +                *
816 +                * NB: We never free blocks for directory so far, which
817 +                * means value returned by dx_get_count() should equal to
818 +                * ld->ld_count if nobody split any DE-block under @at,
819 +                * and ld->ld_at still points to valid dx_entry. */
820 +               if ((ld->ld_count != dx_get_count(entries)) ||
821 +                   !ext4_htree_dx_lock_try(lck, at) ||
822 +                   (ld->ld_count != dx_get_count(entries))) {
823 +                       restart = 1;
824 +                       goto cleanup;
825 +               }
826 +               /* OK, I've got DX lock and nothing changed */
827 +               frame->at = ld->ld_at;
828         }
829 -       de = do_split(handle, dir, &bh, frame, &fname->hinfo);
830 +       de = do_split(handle, dir, &bh, frames, frame, &fname->hinfo, lck);
831         if (IS_ERR(de)) {
832                 err = PTR_ERR(de);
833                 goto cleanup;
834 @@ -2444,6 +2814,8 @@ again:
835  journal_error:
836         ext4_std_error(dir->i_sb, err); /* this is a no-op if err == 0 */
837  cleanup:
838 +       ext4_htree_dx_unlock(lck);
839 +       ext4_htree_de_unlock(lck);
840         brelse(bh);
841         dx_release(frames);
842         /* @restart is true means htree-path has been changed, we need to
843 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/super.c
844 ===================================================================
845 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/super.c
846 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/super.c
847 @@ -1009,6 +1009,7 @@ static struct inode *ext4_alloc_inode(st
848  
849         inode_set_iversion(&ei->vfs_inode, 1);
850         spin_lock_init(&ei->i_raw_lock);
851 +       sema_init(&ei->i_append_sem, 1);
852         INIT_LIST_HEAD(&ei->i_prealloc_list);
853         spin_lock_init(&ei->i_prealloc_lock);
854         ext4_es_init_tree(&ei->i_es_tree);