1 LU-50 ldiskfs: pdirops patch for ldiskfs
3 Single directory performance is a critical for HPC workloads. In a
4 typical use case an application creates a separate output file for
5 each node and task in a job. As nodes and tasks increase, hundreds
6 of thousands of files may be created in a single directory within
7 a short window of time.
8 Today, both filename lookup and file system modifying operations
9 (such as create and unlink) are protected with a single lock for
10 an entire ldiskfs directory. PDO project will remove this
11 bottleneck by introducing a parallel locking mechanism for entire
12 ldiskfs directories. This work will enable multiple application
13 threads to simultaneously lookup, create and unlink in parallel.
16 - pdirops support for ldiskfs
17 - N-level htree directory
18 - integrate with osd-ldiskfs
20 Signed-off-by: Liang Zhen <liang@whamcloud.com>
21 Change-Id: I269c0e3112e68f3acd79e860dab052a68c7d7aaa
22 Reviewed-on: http://review.whamcloud.com/375
23 Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
25 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/Makefile
26 ===================================================================
27 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/Makefile
28 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/Makefile
29 @@ -7,6 +7,7 @@ obj-$(CONFIG_EXT4_FS) += ext4.o
31 ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \
32 extents_status.o file.o fsmap.o fsync.o hash.o ialloc.o \
34 indirect.o inline.o inode.o ioctl.o mballoc.o migrate.o \
35 mmp.o move_extent.o namei.o page-io.o readpage.o resize.o \
36 super.o symlink.o sysfs.o xattr.o xattr_trusted.o xattr_user.o
37 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/ext4.h
38 ===================================================================
39 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/ext4.h
40 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/ext4.h
42 #include <linux/timer.h>
43 #include <linux/version.h>
44 #include <linux/wait.h>
45 +#include <linux/htree_lock.h>
46 #include <linux/sched/signal.h>
47 #include <linux/blockgroup_lock.h>
48 #include <linux/percpu_counter.h>
49 @@ -934,6 +935,9 @@ struct ext4_inode_info {
51 ext4_fsblk_t i_file_acl;
53 + /* following fields for parallel directory operations -bzzz */
54 + struct semaphore i_append_sem;
57 * i_block_group is the number of the block group which contains
58 * this file's inode. Constant across the lifetime of the inode,
59 @@ -2109,6 +2113,72 @@ struct dx_hash_info
61 #define HASH_NB_ALWAYS 1
63 +/* assume name-hash is protected by upper layer */
64 +#define EXT4_HTREE_LOCK_HASH 0
66 +enum ext4_pdo_lk_types {
67 +#if EXT4_HTREE_LOCK_HASH
70 + EXT4_LK_DX, /* index block */
71 + EXT4_LK_DE, /* directory entry block */
72 + EXT4_LK_SPIN, /* spinlock */
77 +#define EXT4_LB_RO(b) (1 << (b))
78 +/* read + write, high bits for writer */
79 +#define EXT4_LB_RW(b) ((1 << (b)) | (1 << (EXT4_LK_MAX + (b))))
81 +enum ext4_pdo_lock_bits {
83 + EXT4_LB_DX_RO = EXT4_LB_RO(EXT4_LK_DX),
84 + EXT4_LB_DX = EXT4_LB_RW(EXT4_LK_DX),
86 + EXT4_LB_DE_RO = EXT4_LB_RO(EXT4_LK_DE),
87 + EXT4_LB_DE = EXT4_LB_RW(EXT4_LK_DE),
88 + /* DX spinlock bits */
89 + EXT4_LB_SPIN_RO = EXT4_LB_RO(EXT4_LK_SPIN),
90 + EXT4_LB_SPIN = EXT4_LB_RW(EXT4_LK_SPIN),
91 + /* accurate searching */
92 + EXT4_LB_EXACT = EXT4_LB_RO(EXT4_LK_MAX << 1),
95 +enum ext4_pdo_lock_opc {
97 + EXT4_HLOCK_READDIR = (EXT4_LB_DE_RO | EXT4_LB_DX_RO),
98 + EXT4_HLOCK_LOOKUP = (EXT4_LB_DE_RO | EXT4_LB_SPIN_RO |
100 + EXT4_HLOCK_DEL = (EXT4_LB_DE | EXT4_LB_SPIN_RO |
102 + EXT4_HLOCK_ADD = (EXT4_LB_DE | EXT4_LB_SPIN_RO),
105 + EXT4_HLOCK_LOOKUP_SAFE = (EXT4_LB_DE_RO | EXT4_LB_DX_RO |
107 + EXT4_HLOCK_DEL_SAFE = (EXT4_LB_DE | EXT4_LB_DX_RO | EXT4_LB_EXACT),
108 + EXT4_HLOCK_SPLIT = (EXT4_LB_DE | EXT4_LB_DX | EXT4_LB_SPIN),
111 +extern struct htree_lock_head *ext4_htree_lock_head_alloc(unsigned hbits);
112 +#define ext4_htree_lock_head_free(lhead) htree_lock_head_free(lhead)
114 +extern struct htree_lock *ext4_htree_lock_alloc(void);
115 +#define ext4_htree_lock_free(lck) htree_lock_free(lck)
117 +extern void ext4_htree_lock(struct htree_lock *lck,
118 + struct htree_lock_head *lhead,
119 + struct inode *dir, unsigned flags);
120 +#define ext4_htree_unlock(lck) htree_unlock(lck)
122 +extern struct buffer_head *__ext4_find_entry(struct inode *dir,
123 + const struct qstr *d_name,
124 + struct ext4_dir_entry_2 **res_dir,
125 + int *inlined, struct htree_lock *lck);
126 +extern int __ext4_add_entry(handle_t *handle, struct dentry *dentry,
127 + struct inode *inode, struct htree_lock *lck);
129 struct ext4_filename {
130 const struct qstr *usr_fname;
131 struct fscrypt_str disk_name;
132 @@ -2416,8 +2486,16 @@ void ext4_insert_dentry(struct inode *in
133 struct ext4_filename *fname, void *data);
134 static inline void ext4_update_dx_flag(struct inode *inode)
136 + /* Disable it for ldiskfs, because going from a DX directory to
137 + * a non-DX directory while it is in use will completely break
138 + * the htree-locking.
139 + * If we really want to support this operation in the future,
140 + * we need to exclusively lock the directory at here which will
141 + * increase complexity of code */
143 if (!ext4_has_feature_dir_index(inode->i_sb))
144 ext4_clear_inode_flag(inode, EXT4_INODE_INDEX);
147 static const unsigned char ext4_filetype_table[] = {
148 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
149 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/namei.c
150 ===================================================================
151 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/namei.c
152 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/namei.c
153 @@ -54,6 +54,7 @@ struct buffer_head *ext4_append(handle_t
156 struct buffer_head *bh;
157 + struct ext4_inode_info *ei = EXT4_I(inode);
160 if (unlikely(EXT4_SB(inode->i_sb)->s_max_dir_size_kb &&
161 @@ -61,15 +62,22 @@ struct buffer_head *ext4_append(handle_t
162 EXT4_SB(inode->i_sb)->s_max_dir_size_kb)))
163 return ERR_PTR(-ENOSPC);
165 + /* with parallel dir operations all appends
166 + * have to be serialized -bzzz */
167 + down(&ei->i_append_sem);
169 *block = inode->i_size >> inode->i_sb->s_blocksize_bits;
171 bh = ext4_bread(handle, inode, *block, EXT4_GET_BLOCKS_CREATE);
174 + up(&ei->i_append_sem);
177 inode->i_size += inode->i_sb->s_blocksize;
178 EXT4_I(inode)->i_disksize = inode->i_size;
179 BUFFER_TRACE(bh, "get_write_access");
180 err = ext4_journal_get_write_access(handle, bh);
181 + up(&ei->i_append_sem);
184 ext4_std_error(inode->i_sb, err);
185 @@ -250,7 +258,8 @@ static unsigned dx_node_limit(struct ino
186 static struct dx_frame *dx_probe(struct ext4_filename *fname,
188 struct dx_hash_info *hinfo,
189 - struct dx_frame *frame);
190 + struct dx_frame *frame,
191 + struct htree_lock *lck);
192 static void dx_release(struct dx_frame *frames);
193 static int dx_make_map(struct inode *dir, struct ext4_dir_entry_2 *de,
194 unsigned blocksize, struct dx_hash_info *hinfo,
195 @@ -264,12 +273,13 @@ static void dx_insert_block(struct dx_fr
196 static int ext4_htree_next_block(struct inode *dir, __u32 hash,
197 struct dx_frame *frame,
198 struct dx_frame *frames,
199 - __u32 *start_hash);
200 + __u32 *start_hash, struct htree_lock *lck);
201 static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
202 struct ext4_filename *fname,
203 - struct ext4_dir_entry_2 **res_dir);
204 + struct ext4_dir_entry_2 **res_dir, struct htree_lock *lck);
205 static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
206 - struct inode *dir, struct inode *inode);
207 + struct inode *dir, struct inode *inode,
208 + struct htree_lock *lck);
210 /* checksumming functions */
211 void initialize_dirent_tail(struct ext4_dir_entry_tail *t,
212 @@ -733,6 +743,227 @@ struct stats dx_show_entries(struct dx_h
214 #endif /* DX_DEBUG */
216 +/* private data for htree_lock */
217 +struct ext4_dir_lock_data {
218 + unsigned ld_flags; /* bits-map for lock types */
219 + unsigned ld_count; /* # entries of the last DX block */
220 + struct dx_entry ld_at_entry; /* copy of leaf dx_entry */
221 + struct dx_entry *ld_at; /* position of leaf dx_entry */
224 +#define ext4_htree_lock_data(l) ((struct ext4_dir_lock_data *)(l)->lk_private)
225 +#define ext4_find_entry(dir, name, dirent, inline) \
226 + __ext4_find_entry(dir, name, dirent, inline, NULL)
227 +#define ext4_add_entry(handle, dentry, inode) \
228 + __ext4_add_entry(handle, dentry, inode, NULL)
230 +/* NB: ext4_lblk_t is 32 bits so we use high bits to identify invalid blk */
231 +#define EXT4_HTREE_NODE_CHANGED (0xcafeULL << 32)
233 +static void ext4_htree_event_cb(void *target, void *event)
235 + u64 *block = (u64 *)target;
237 + if (*block == dx_get_block((struct dx_entry *)event))
238 + *block = EXT4_HTREE_NODE_CHANGED;
241 +struct htree_lock_head *ext4_htree_lock_head_alloc(unsigned hbits)
243 + struct htree_lock_head *lhead;
245 + lhead = htree_lock_head_alloc(EXT4_LK_MAX, hbits, 0);
246 + if (lhead != NULL) {
247 + htree_lock_event_attach(lhead, EXT4_LK_SPIN, HTREE_EVENT_WR,
248 + ext4_htree_event_cb);
252 +EXPORT_SYMBOL(ext4_htree_lock_head_alloc);
254 +struct htree_lock *ext4_htree_lock_alloc(void)
256 + return htree_lock_alloc(EXT4_LK_MAX,
257 + sizeof(struct ext4_dir_lock_data));
259 +EXPORT_SYMBOL(ext4_htree_lock_alloc);
261 +static htree_lock_mode_t ext4_htree_mode(unsigned flags)
264 + default: /* 0 or unknown flags require EX lock */
265 + return HTREE_LOCK_EX;
266 + case EXT4_HLOCK_READDIR:
267 + return HTREE_LOCK_PR;
268 + case EXT4_HLOCK_LOOKUP:
269 + return HTREE_LOCK_CR;
270 + case EXT4_HLOCK_DEL:
271 + case EXT4_HLOCK_ADD:
272 + return HTREE_LOCK_CW;
276 +/* return PR for read-only operations, otherwise return EX */
277 +static inline htree_lock_mode_t ext4_htree_safe_mode(unsigned flags)
279 + int writer = (flags & EXT4_LB_DE) == EXT4_LB_DE;
281 + /* 0 requires EX lock */
282 + return (flags == 0 || writer) ? HTREE_LOCK_EX : HTREE_LOCK_PR;
285 +static int ext4_htree_safe_locked(struct htree_lock *lck)
289 + if (lck == NULL || lck->lk_mode == HTREE_LOCK_EX)
292 + writer = (ext4_htree_lock_data(lck)->ld_flags & EXT4_LB_DE) ==
294 + if (writer) /* all readers & writers are excluded? */
295 + return lck->lk_mode == HTREE_LOCK_EX;
297 + /* all writers are excluded? */
298 + return lck->lk_mode == HTREE_LOCK_PR ||
299 + lck->lk_mode == HTREE_LOCK_PW ||
300 + lck->lk_mode == HTREE_LOCK_EX;
303 +/* relock htree_lock with EX mode if it's change operation, otherwise
304 + * relock it with PR mode. It's noop if PDO is disabled. */
305 +static void ext4_htree_safe_relock(struct htree_lock *lck)
307 + if (!ext4_htree_safe_locked(lck)) {
308 + unsigned flags = ext4_htree_lock_data(lck)->ld_flags;
310 + htree_change_lock(lck, ext4_htree_safe_mode(flags));
314 +void ext4_htree_lock(struct htree_lock *lck, struct htree_lock_head *lhead,
315 + struct inode *dir, unsigned flags)
317 + htree_lock_mode_t mode = is_dx(dir) ? ext4_htree_mode(flags) :
318 + ext4_htree_safe_mode(flags);
320 + ext4_htree_lock_data(lck)->ld_flags = flags;
321 + htree_lock(lck, lhead, mode);
323 + ext4_htree_safe_relock(lck); /* make sure it's safe locked */
325 +EXPORT_SYMBOL(ext4_htree_lock);
327 +static int ext4_htree_node_lock(struct htree_lock *lck, struct dx_entry *at,
328 + unsigned lmask, int wait, void *ev)
330 + u32 key = (at == NULL) ? 0 : dx_get_block(at);
333 + /* NOOP if htree is well protected or caller doesn't require the lock */
334 + if (ext4_htree_safe_locked(lck) ||
335 + !(ext4_htree_lock_data(lck)->ld_flags & lmask))
338 + mode = (ext4_htree_lock_data(lck)->ld_flags & lmask) == lmask ?
339 + HTREE_LOCK_PW : HTREE_LOCK_PR;
341 + if (htree_node_lock_try(lck, mode, key, ffz(~lmask), wait, ev))
343 + if (!(lmask & EXT4_LB_SPIN)) /* not a spinlock */
345 + cpu_relax(); /* spin until granted */
349 +static int ext4_htree_node_locked(struct htree_lock *lck, unsigned lmask)
351 + return ext4_htree_safe_locked(lck) ||
352 + htree_node_is_granted(lck, ffz(~lmask));
355 +static void ext4_htree_node_unlock(struct htree_lock *lck,
356 + unsigned lmask, void *buf)
358 + /* NB: it's safe to call mutiple times or even it's not locked */
359 + if (!ext4_htree_safe_locked(lck) &&
360 + htree_node_is_granted(lck, ffz(~lmask)))
361 + htree_node_unlock(lck, ffz(~lmask), buf);
364 +#define ext4_htree_dx_lock(lck, key) \
365 + ext4_htree_node_lock(lck, key, EXT4_LB_DX, 1, NULL)
366 +#define ext4_htree_dx_lock_try(lck, key) \
367 + ext4_htree_node_lock(lck, key, EXT4_LB_DX, 0, NULL)
368 +#define ext4_htree_dx_unlock(lck) \
369 + ext4_htree_node_unlock(lck, EXT4_LB_DX, NULL)
370 +#define ext4_htree_dx_locked(lck) \
371 + ext4_htree_node_locked(lck, EXT4_LB_DX)
373 +static void ext4_htree_dx_need_lock(struct htree_lock *lck)
375 + struct ext4_dir_lock_data *ld;
377 + if (ext4_htree_safe_locked(lck))
380 + ld = ext4_htree_lock_data(lck);
381 + switch (ld->ld_flags) {
384 + case EXT4_HLOCK_LOOKUP:
385 + ld->ld_flags = EXT4_HLOCK_LOOKUP_SAFE;
387 + case EXT4_HLOCK_DEL:
388 + ld->ld_flags = EXT4_HLOCK_DEL_SAFE;
390 + case EXT4_HLOCK_ADD:
391 + ld->ld_flags = EXT4_HLOCK_SPLIT;
396 +#define ext4_htree_de_lock(lck, key) \
397 + ext4_htree_node_lock(lck, key, EXT4_LB_DE, 1, NULL)
398 +#define ext4_htree_de_unlock(lck) \
399 + ext4_htree_node_unlock(lck, EXT4_LB_DE, NULL)
401 +#define ext4_htree_spin_lock(lck, key, event) \
402 + ext4_htree_node_lock(lck, key, EXT4_LB_SPIN, 0, event)
403 +#define ext4_htree_spin_unlock(lck) \
404 + ext4_htree_node_unlock(lck, EXT4_LB_SPIN, NULL)
405 +#define ext4_htree_spin_unlock_listen(lck, p) \
406 + ext4_htree_node_unlock(lck, EXT4_LB_SPIN, p)
408 +static void ext4_htree_spin_stop_listen(struct htree_lock *lck)
410 + if (!ext4_htree_safe_locked(lck) &&
411 + htree_node_is_listening(lck, ffz(~EXT4_LB_SPIN)))
412 + htree_node_stop_listen(lck, ffz(~EXT4_LB_SPIN));
416 + DX_HASH_COL_IGNORE, /* ignore collision while probing frames */
417 + DX_HASH_COL_YES, /* there is collision and it does matter */
418 + DX_HASH_COL_NO, /* there is no collision */
421 +static int dx_probe_hash_collision(struct htree_lock *lck,
422 + struct dx_entry *entries,
423 + struct dx_entry *at, u32 hash)
425 + if (!(lck && ext4_htree_lock_data(lck)->ld_flags & EXT4_LB_EXACT)) {
426 + return DX_HASH_COL_IGNORE; /* don't care about collision */
428 + } else if (at == entries + dx_get_count(entries) - 1) {
429 + return DX_HASH_COL_IGNORE; /* not in any leaf of this DX */
431 + } else { /* hash collision? */
432 + return ((dx_get_hash(at + 1) & ~1) == hash) ?
433 + DX_HASH_COL_YES : DX_HASH_COL_NO;
438 * Probe for a directory leaf block to search.
440 @@ -744,10 +975,11 @@ struct stats dx_show_entries(struct dx_h
442 static struct dx_frame *
443 dx_probe(struct ext4_filename *fname, struct inode *dir,
444 - struct dx_hash_info *hinfo, struct dx_frame *frame_in)
445 + struct dx_hash_info *hinfo, struct dx_frame *frame_in,
446 + struct htree_lock *lck)
448 unsigned count, indirect;
449 - struct dx_entry *at, *entries, *p, *q, *m;
450 + struct dx_entry *at, *entries, *p, *q, *m, *dx = NULL;
451 struct dx_root_info *info;
452 struct dx_frame *frame = frame_in;
453 struct dx_frame *ret_err = ERR_PTR(ERR_BAD_DX_DIR);
454 @@ -809,8 +1041,15 @@ dx_probe(struct ext4_filename *fname, st
456 dxtrace(printk("Look up %x", hash));
458 + if (indirect == 0) { /* the last index level */
459 + /* NB: ext4_htree_dx_lock() could be noop if
460 + * DX-lock flag is not set for current operation */
461 + ext4_htree_dx_lock(lck, dx);
462 + ext4_htree_spin_lock(lck, dx, NULL);
464 count = dx_get_count(entries);
465 - if (!count || count > dx_get_limit(entries)) {
466 + if (count == 0 || count > dx_get_limit(entries)) {
467 + ext4_htree_spin_unlock(lck); /* release spin */
468 ext4_warning_inode(dir,
469 "dx entry: count %u beyond limit %u",
470 count, dx_get_limit(entries));
471 @@ -849,8 +1088,70 @@ dx_probe(struct ext4_filename *fname, st
473 frame->entries = entries;
477 + if (indirect == 0) { /* the last index level */
478 + struct ext4_dir_lock_data *ld;
481 + /* By default we only lock DE-block, however, we will
482 + * also lock the last level DX-block if:
483 + * a) there is hash collision
484 + * we will set DX-lock flag (a few lines below)
485 + * and redo to lock DX-block
486 + * see detail in dx_probe_hash_collision()
487 + * b) it's a retry from splitting
488 + * we need to lock the last level DX-block so nobody
489 + * else can split any leaf blocks under the same
490 + * DX-block, see detail in ext4_dx_add_entry()
492 + if (ext4_htree_dx_locked(lck)) {
493 + /* DX-block is locked, just lock DE-block
495 + ext4_htree_spin_unlock(lck);
496 + if (!ext4_htree_safe_locked(lck))
497 + ext4_htree_de_lock(lck, frame->at);
500 + /* it's pdirop and no DX lock */
501 + if (dx_probe_hash_collision(lck, entries, at, hash) ==
503 + /* found hash collision, set DX-lock flag
504 + * and retry to abtain DX-lock */
505 + ext4_htree_spin_unlock(lck);
506 + ext4_htree_dx_need_lock(lck);
509 + ld = ext4_htree_lock_data(lck);
510 + /* because I don't lock DX, so @at can't be trusted
511 + * after I release spinlock so I have to save it */
513 + ld->ld_at_entry = *at;
514 + ld->ld_count = dx_get_count(entries);
516 + frame->at = &ld->ld_at_entry;
517 + myblock = dx_get_block(at);
519 + /* NB: ordering locking */
520 + ext4_htree_spin_unlock_listen(lck, &myblock);
521 + /* other thread can split this DE-block because:
522 + * a) I don't have lock for the DE-block yet
523 + * b) I released spinlock on DX-block
524 + * if it happened I can detect it by listening
525 + * splitting event on this DE-block */
526 + ext4_htree_de_lock(lck, frame->at);
527 + ext4_htree_spin_stop_listen(lck);
529 + if (myblock == EXT4_HTREE_NODE_CHANGED) {
530 + /* someone split this DE-block before
531 + * I locked it, I need to retry and lock
532 + * valid DE-block */
533 + ext4_htree_de_unlock(lck);
541 frame->bh = ext4_read_dirblock(dir, dx_get_block(at), INDEX);
542 if (IS_ERR(frame->bh)) {
543 @@ -916,7 +1217,7 @@ static void dx_release(struct dx_frame *
544 static int ext4_htree_next_block(struct inode *dir, __u32 hash,
545 struct dx_frame *frame,
546 struct dx_frame *frames,
548 + __u32 *start_hash, struct htree_lock *lck)
551 struct buffer_head *bh;
552 @@ -931,12 +1232,22 @@ static int ext4_htree_next_block(struct
553 * this loop, num_frames indicates the number of interior
554 * nodes need to be read.
556 + ext4_htree_de_unlock(lck);
558 - if (++(p->at) < p->entries + dx_get_count(p->entries))
560 + if (num_frames > 0 || ext4_htree_dx_locked(lck)) {
561 + /* num_frames > 0 :
563 + * ext4_htree_dx_locked:
564 + * frame->at is reliable pointer returned by dx_probe,
565 + * otherwise dx_probe already knew no collision */
566 + if (++(p->at) < p->entries + dx_get_count(p->entries))
572 + if (num_frames == 1)
573 + ext4_htree_dx_unlock(lck);
577 @@ -959,6 +1270,13 @@ static int ext4_htree_next_block(struct
578 * block so no check is necessary
580 while (num_frames--) {
581 + if (num_frames == 0) {
582 + /* it's not always necessary, we just don't want to
583 + * detect hash collision again */
584 + ext4_htree_dx_need_lock(lck);
585 + ext4_htree_dx_lock(lck, p->at);
588 bh = ext4_read_dirblock(dir, dx_get_block(p->at), INDEX);
591 @@ -967,6 +1285,7 @@ static int ext4_htree_next_block(struct
593 p->at = p->entries = ((struct dx_node *) bh->b_data)->entries;
595 + ext4_htree_de_lock(lck, p->at);
599 @@ -1114,10 +1433,10 @@ int ext4_htree_fill_tree(struct file *di
601 hinfo.hash = start_hash;
602 hinfo.minor_hash = 0;
603 - frame = dx_probe(NULL, dir, &hinfo, frames);
604 + /* assume it's PR locked */
605 + frame = dx_probe(NULL, dir, &hinfo, frames, NULL);
607 return PTR_ERR(frame);
609 /* Add '.' and '..' from the htree header */
610 if (!start_hash && !start_minor_hash) {
611 de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data;
612 @@ -1157,7 +1476,7 @@ int ext4_htree_fill_tree(struct file *di
615 ret = ext4_htree_next_block(dir, HASH_NB_ALWAYS,
616 - frame, frames, &hashval);
617 + frame, frames, &hashval, NULL);
618 *next_hash = hashval;
621 @@ -1349,10 +1668,10 @@ static int is_dx_internal_node(struct in
622 * The returned buffer_head has ->b_count elevated. The caller is expected
623 * to brelse() it when appropriate.
625 -static struct buffer_head * ext4_find_entry (struct inode *dir,
626 +struct buffer_head *__ext4_find_entry(struct inode *dir,
627 const struct qstr *d_name,
628 struct ext4_dir_entry_2 **res_dir,
630 + int *inlined, struct htree_lock *lck)
632 struct super_block *sb;
633 struct buffer_head *bh_use[NAMEI_RA_SIZE];
634 @@ -1401,7 +1720,7 @@ static struct buffer_head * ext4_find_en
638 - ret = ext4_dx_find_entry(dir, &fname, res_dir);
639 + ret = ext4_dx_find_entry(dir, &fname, res_dir, lck);
641 * On success, or if the error was file not found,
642 * return. Otherwise, fall back to doing a search the
643 @@ -1411,6 +1730,7 @@ static struct buffer_head * ext4_find_en
644 goto cleanup_and_exit;
645 dxtrace(printk(KERN_DEBUG "ext4_find_entry: dx failed, "
647 + ext4_htree_safe_relock(lck);
650 nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb);
651 @@ -1499,10 +1819,12 @@ cleanup_and_exit:
652 ext4_fname_free_filename(&fname);
655 +EXPORT_SYMBOL(__ext4_find_entry);
657 static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
658 struct ext4_filename *fname,
659 - struct ext4_dir_entry_2 **res_dir)
660 + struct ext4_dir_entry_2 **res_dir,
661 + struct htree_lock *lck)
663 struct super_block * sb = dir->i_sb;
664 struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
665 @@ -1513,7 +1835,7 @@ static struct buffer_head * ext4_dx_find
666 #ifdef CONFIG_EXT4_FS_ENCRYPTION
669 - frame = dx_probe(fname, dir, NULL, frames);
670 + frame = dx_probe(fname, dir, NULL, frames, lck);
672 return (struct buffer_head *) frame;
674 @@ -1535,7 +1857,7 @@ static struct buffer_head * ext4_dx_find
676 /* Check to see if we should continue to search */
677 retval = ext4_htree_next_block(dir, fname->hinfo.hash, frame,
679 + frames, NULL, lck);
681 ext4_warning_inode(dir,
682 "error %d reading directory index block",
683 @@ -1710,8 +2032,9 @@ static struct ext4_dir_entry_2* dx_pack_
684 * Returns pointer to de in block into which the new entry will be inserted.
686 static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir,
687 - struct buffer_head **bh,struct dx_frame *frame,
688 - struct dx_hash_info *hinfo)
689 + struct buffer_head **bh, struct dx_frame *frames,
690 + struct dx_frame *frame, struct dx_hash_info *hinfo,
691 + struct htree_lock *lck)
693 unsigned blocksize = dir->i_sb->s_blocksize;
694 unsigned count, continued;
695 @@ -1773,8 +2096,14 @@ static struct ext4_dir_entry_2 *do_split
696 hash2, split, count-split));
698 /* Fancy dance to stay within two buffers */
699 - de2 = dx_move_dirents(data1, data2, map + split, count - split,
701 + if (hinfo->hash < hash2) {
702 + de2 = dx_move_dirents(data1, data2, map + split,
703 + count - split, blocksize);
705 + /* make sure we will add entry to the same block which
706 + * we have already locked */
707 + de2 = dx_move_dirents(data1, data2, map, split, blocksize);
709 de = dx_pack_dirents(data1, blocksize);
710 de->rec_len = ext4_rec_len_to_disk(data1 + (blocksize - csum_size) -
712 @@ -1795,12 +2124,21 @@ static struct ext4_dir_entry_2 *do_split
713 dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data2,
716 - /* Which block gets the new entry? */
717 - if (hinfo->hash >= hash2) {
720 + ext4_htree_spin_lock(lck, frame > frames ? (frame - 1)->at : NULL,
721 + frame->at); /* notify block is being split */
722 + if (hinfo->hash < hash2) {
723 + dx_insert_block(frame, hash2 + continued, newblock);
726 + /* switch block number */
727 + dx_insert_block(frame, hash2 + continued,
728 + dx_get_block(frame->at));
729 + dx_set_block(frame->at, newblock);
732 - dx_insert_block(frame, hash2 + continued, newblock);
733 + ext4_htree_spin_unlock(lck);
734 + ext4_htree_dx_unlock(lck);
736 err = ext4_handle_dirty_dirent_node(handle, dir, bh2);
739 @@ -2074,7 +2412,7 @@ static int make_indexed_dir(handle_t *ha
743 - de = do_split(handle,dir, &bh2, frame, &fname->hinfo);
744 + de = do_split(handle, dir, &bh2, frames, frame, &fname->hinfo, NULL);
746 retval = PTR_ERR(de);
748 @@ -2184,8 +2522,8 @@ out:
749 * may not sleep between calling this and putting something into
750 * the entry, as someone else might have used it while you slept.
752 -static int ext4_add_entry(handle_t *handle, struct dentry *dentry,
753 - struct inode *inode)
754 +int __ext4_add_entry(handle_t *handle, struct dentry *dentry,
755 + struct inode *inode, struct htree_lock *lck)
757 struct inode *dir = d_inode(dentry->d_parent);
758 struct buffer_head *bh = NULL;
759 @@ -2226,9 +2564,10 @@ static int ext4_add_entry(handle_t *hand
760 if (dentry->d_name.len == 2 &&
761 memcmp(dentry->d_name.name, "..", 2) == 0)
762 return ext4_update_dotdot(handle, dentry, inode);
763 - retval = ext4_dx_add_entry(handle, &fname, dir, inode);
764 + retval = ext4_dx_add_entry(handle, &fname, dir, inode, lck);
765 if (!retval || (retval != ERR_BAD_DX_DIR))
767 + ext4_htree_safe_relock(lck);
768 ext4_clear_inode_flag(dir, EXT4_INODE_INDEX);
770 ext4_mark_inode_dirty(handle, dir);
771 @@ -2278,12 +2617,14 @@ out:
772 ext4_set_inode_state(inode, EXT4_STATE_NEWENTRY);
775 +EXPORT_SYMBOL(__ext4_add_entry);
778 * Returns 0 for success, or a negative error value
780 static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
781 - struct inode *dir, struct inode *inode)
782 + struct inode *dir, struct inode *inode,
783 + struct htree_lock *lck)
785 struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
786 struct dx_entry *entries, *at;
787 @@ -2295,7 +2636,7 @@ static int ext4_dx_add_entry(handle_t *h
791 - frame = dx_probe(fname, dir, NULL, frames);
792 + frame = dx_probe(fname, dir, NULL, frames, lck);
794 return PTR_ERR(frame);
795 entries = frame->entries;
796 @@ -2330,6 +2671,11 @@ again:
797 struct dx_node *node2;
798 struct buffer_head *bh2;
800 + if (!ext4_htree_safe_locked(lck)) { /* retry with EX lock */
801 + ext4_htree_safe_relock(lck);
805 while (frame > frames) {
806 if (dx_get_count((frame - 1)->entries) <
807 dx_get_limit((frame - 1)->entries)) {
808 @@ -2432,8 +2778,32 @@ again:
812 + } else if (!ext4_htree_dx_locked(lck)) {
813 + struct ext4_dir_lock_data *ld = ext4_htree_lock_data(lck);
815 + /* not well protected, require DX lock */
816 + ext4_htree_dx_need_lock(lck);
817 + at = frame > frames ? (frame - 1)->at : NULL;
819 + /* NB: no risk of deadlock because it's just a try.
821 + * NB: we check ld_count for twice, the first time before
822 + * having DX lock, the second time after holding DX lock.
824 + * NB: We never free blocks for directory so far, which
825 + * means value returned by dx_get_count() should equal to
826 + * ld->ld_count if nobody split any DE-block under @at,
827 + * and ld->ld_at still points to valid dx_entry. */
828 + if ((ld->ld_count != dx_get_count(entries)) ||
829 + !ext4_htree_dx_lock_try(lck, at) ||
830 + (ld->ld_count != dx_get_count(entries))) {
834 + /* OK, I've got DX lock and nothing changed */
835 + frame->at = ld->ld_at;
837 - de = do_split(handle, dir, &bh, frame, &fname->hinfo);
838 + de = do_split(handle, dir, &bh, frames, frame, &fname->hinfo, lck);
842 @@ -2444,6 +2814,8 @@ again:
844 ext4_std_error(dir->i_sb, err); /* this is a no-op if err == 0 */
846 + ext4_htree_dx_unlock(lck);
847 + ext4_htree_de_unlock(lck);
850 /* @restart is true means htree-path has been changed, we need to
851 Index: linux-4.18.0-80.1.2.el8_0/fs/ext4/super.c
852 ===================================================================
853 --- linux-4.18.0-80.1.2.el8_0.orig/fs/ext4/super.c
854 +++ linux-4.18.0-80.1.2.el8_0/fs/ext4/super.c
855 @@ -1009,6 +1009,7 @@ static struct inode *ext4_alloc_inode(st
857 inode_set_iversion(&ei->vfs_inode, 1);
858 spin_lock_init(&ei->i_raw_lock);
859 + sema_init(&ei->i_append_sem, 1);
860 INIT_LIST_HEAD(&ei->i_prealloc_list);
861 spin_lock_init(&ei->i_prealloc_lock);
862 ext4_es_init_tree(&ei->i_es_tree);