From 1fca698f6293cf2dadc907238a37a5366959de58 Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Sat, 13 Jan 2024 17:50:37 -0700 Subject: [PATCH] LU-17131 ldiskfs: Refresh ubuntu 5.11 server Refresh ext4-pdirop and ext4-delayed-iput, Add ext4-filename-encode support ext4-add-periodic-superblock-update Test-Parameters: trivial Signed-off-by: Shaun Tancheff Change-Id: Icd066a4f507842312924f7c7818208d8f07c8c70 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52413 Reviewed-by: James Simmons Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- .../ext4-add-periodic-superblock-update.patch | 99 +++++ .../ubuntu20.04.3/ext4-filename-encode.patch | 437 +++++++++++++++++++++ .../series/ldiskfs-5.11.0-40-ubuntu20.series | 5 +- 3 files changed, 539 insertions(+), 2 deletions(-) create mode 100644 ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-add-periodic-superblock-update.patch create mode 100644 ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-filename-encode.patch diff --git a/ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-add-periodic-superblock-update.patch b/ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-add-periodic-superblock-update.patch new file mode 100644 index 0000000..56d3580 --- /dev/null +++ b/ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-add-periodic-superblock-update.patch @@ -0,0 +1,99 @@ +Subject: [PATCH] ext4: Add periodic superblock update check + +This patch introduces a mechanism to periodically check and update +the superblock within the ext4 file system. The main purpose of this +patch is to keep the disk superblock up to date. The update will be +performed if more than one hour has passed since the last update, and +if more than 16MB of data have been written to disk. + +This check and update is performed within the ext4_journal_commit_callback +function, ensuring that the superblock is written while the disk is +active, rather than based on a timer that may trigger during disk idle +periods. + +Signed-off-by: Vitaliy Kuznetsov +--- + fs/ext4/super.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ + fs/ext4/sysfs.c | 1 - + 2 files changed, 50 insertions(+), 1 deletion(-) + +diff --git a/fs/ext4/super.c b/fs/ext4/super.c +index 82da5cc6..758a5b08 100644 +--- a/fs/ext4/super.c ++++ b/fs/ext4/super.c +@@ -440,6 +440,55 @@ static int block_device_ejected(struct super_block *sb) + return bdi->dev == NULL; + } + ++#define EXT4_SB_REFRESH_INTERVAL_SEC (3600) /* seconds (1 hour) */ ++#define EXT4_SB_REFRESH_INTERVAL_KB (16384) /* kilobytes (16MB) */ ++ ++/* ++ * The ext4_maybe_update_superblock() function checks and updates the ++ * superblock if needed. ++ * ++ * This function is designed to update the on-disk superblock only under ++ * certain conditions to prevent excessive disk writes and unnecessary ++ * waking of the disk from sleep. The superblock will be updated if: ++ * 1. More than an hour has passed since the last superblock update, and ++ * 2. More than 16MB have been written since the last superblock update. ++ * ++ * @sb: The superblock ++ */ ++static void ext4_maybe_update_superblock(struct super_block *sb) ++{ ++ struct ext4_sb_info *sbi = EXT4_SB(sb); ++ struct ext4_super_block *es = sbi->s_es; ++ time64_t now; ++ __u64 last_update; ++ __u64 lifetime_write_kbytes; ++ __u64 diff_size; ++ ++ if (sb_rdonly(sb)) ++ return; ++ ++ now = ktime_get_real_seconds(); ++ last_update = ext4_get_tstamp(es, s_wtime); ++ ++ if (likely(now - last_update < EXT4_SB_REFRESH_INTERVAL_SEC)) ++ return; ++ ++ lifetime_write_kbytes = sbi->s_kbytes_written + ++ ((part_stat_read(sb->s_bdev, sectors[STAT_WRITE]) - ++ sbi->s_sectors_written_start) >> 1); ++ ++ /* Get the number of kilobytes not written to disk to account ++ * for statistics and compare with a multiple of 16 MB. This ++ * is used to determine when the next superblock commit should ++ * occur (i.e. not more often than once per 16MB if there was ++ * less written in an hour). ++ */ ++ diff_size = lifetime_write_kbytes - le64_to_cpu(es->s_kbytes_written); ++ ++ if (diff_size > EXT4_SB_REFRESH_INTERVAL_KB) ++ schedule_work(&EXT4_SB(sb)->s_error_work); ++} ++ + static void ext4_journal_commit_callback(journal_t *journal, transaction_t *txn) + { + struct super_block *sb = journal->j_private; +@@ -450,6 +499,7 @@ static void ext4_journal_commit_callback(journal_t *journal, transaction_t *txn) + BUG_ON(txn->t_state == T_FINISHED); + + ext4_process_freed_data(sb, txn->t_tid); ++ ext4_maybe_update_superblock(sb); + + spin_lock(&sbi->s_md_lock); + while (!list_empty(&txn->t_private_list)) { +diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c +index cfac5a2e..76f2f6ae 100644 +--- a/fs/ext4/sysfs.c ++++ b/fs/ext4/sysfs.c +@@ -651,4 +651,3 @@ void ext4_exit_sysfs(void) + remove_proc_entry(proc_dirname, NULL); + ext4_proc_root = NULL; + } +- +-- +2.25.1 + diff --git a/ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-filename-encode.patch b/ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-filename-encode.patch new file mode 100644 index 0000000..5330a9c --- /dev/null +++ b/ldiskfs/kernel_patches/patches/ubuntu20.04.3/ext4-filename-encode.patch @@ -0,0 +1,437 @@ +From 44da2f187eca19a37da514354b947208bad7d6c2 Mon Sep 17 00:00:00 2001 +From: Shaun Tancheff +Date: Wed, 6 Dec 2023 03:40:11 -0600 +Subject: [PATCH] ubuntu20.04.3/ext4-filename-encode + +--- + fs/ext4/critical_encode.h | 173 ++++++++++++++++++++++++++++++++++++++ + fs/ext4/dir.c | 33 ++++++-- + fs/ext4/ialloc.c | 1 + + fs/ext4/namei.c | 52 +++++++++--- + 4 files changed, 237 insertions(+), 22 deletions(-) + create mode 100644 fs/ext4/critical_encode.h + +diff --git a/fs/ext4/critical_encode.h b/fs/ext4/critical_encode.h +new file mode 100644 +index 00000000..83d94393 +--- /dev/null ++++ b/fs/ext4/critical_encode.h +@@ -0,0 +1,170 @@ ++/* ++ * critical_encode.h ++ * ++ * Copyright (c) 2022 Whamcloud ++ */ ++ ++#ifndef _CRITICAL_ENCODE_H ++#define _CRITICAL_ENCODE_H ++ ++#include ++ ++/* Encoding/decoding routines inspired from yEnc principles. ++ * We just take care of a few critical characters: ++ * NULL, LF, CR, /, DEL and =. ++ * If such a char is found, it is replaced with '=' followed by ++ * the char value + 64. ++ * All other chars are left untouched. ++ * Efficiency of this encoding depends on the occurences of the ++ * critical chars, but statistically on binary data it can be much higher ++ * than base64 for instance. ++ */ ++static inline int critical_encode(const u8 *src, int len, char *dst) ++{ ++ u8 *p = (u8 *)src, *q = dst; ++ ++ while (p - src < len) { ++ /* escape NULL, LF, CR, /, DEL and = */ ++ if (unlikely(*p == 0x0 || *p == 0xA || *p == 0xD || ++ *p == '/' || *p == 0x7F || *p == '=')) { ++ *(q++) = '='; ++ *(q++) = *(p++) + 64; ++ } else { ++ *(q++) = *(p++); ++ } ++ } ++ ++ return (char *)q - dst; ++} ++ ++/* returns the number of chars encoding would produce */ ++static inline int critical_chars(const u8 *src, int len) ++{ ++ u8 *p = (u8 *)src; ++ int newlen = len; ++ ++ while (p - src < len) { ++ /* NULL, LF, CR, /, DEL and = cost an additional '=' */ ++ if (unlikely(*p == 0x0 || *p == 0xA || *p == 0xD || ++ *p == '/' || *p == 0x7F || *p == '=')) ++ newlen++; ++ p++; ++ } ++ ++ return newlen; ++} ++ ++/* decoding routine - returns the number of chars in output */ ++static inline int critical_decode(const u8 *src, int len, char *dst) ++{ ++ u8 *p = (u8 *)src, *q = dst; ++ ++ while (p - src < len) { ++ if (unlikely(*p == '=')) { ++ *(q++) = *(++p) - 64; ++ p++; ++ } else { ++ *(q++) = *(p++); ++ } ++ } ++ ++ return (char *)q - dst; ++} ++ ++#define fscrypt_get_encryption_info(inode) \ ++ (unlikely(!IS_LUSTRE_MOUNT(inode->i_sb)) ? 0 : -EOPNOTSUPP) ++ ++static inline int ext4_has_permitted_context(struct inode *parent, ++ struct inode *child) ++{ ++ if (unlikely(!IS_LUSTRE_MOUNT(parent->i_sb))) ++ return 1; ++ return fscrypt_has_permitted_context(parent, child); ++} ++ ++struct ext4_filename; ++ ++static inline int ext4_prepare_readdir(struct inode *dir) ++{ ++ if (unlikely(!IS_LUSTRE_MOUNT(dir->i_sb))) ++ return 0; ++ return fscrypt_prepare_readdir(dir); ++} ++ ++static inline int ext4_fname_alloc_buffer(const struct inode *inode, ++ u32 max_encrypted_len, ++ struct fscrypt_str *crypto_str) ++{ ++ crypto_str->name = kmalloc(max_encrypted_len + 1, GFP_NOFS); ++ if (!crypto_str->name) ++ return -ENOMEM; ++ crypto_str->len = max_encrypted_len; ++ return 0; ++} ++ ++static inline void ext4_fname_free_buffer(struct fscrypt_str *crypto_str) ++{ ++ if (!crypto_str) ++ return; ++ kfree(crypto_str->name); ++ crypto_str->name = NULL; ++} ++ ++static inline int ext4_fname_disk_to_usr(struct inode *inode, ++ u32 hash, u32 minor_hash, ++ const struct fscrypt_str *iname, ++ struct fscrypt_str *oname) ++{ ++ int presented_len; ++ ++ presented_len = critical_encode(iname->name, iname->len, oname->name); ++ if (presented_len > NAME_MAX) { ++ /* truncate at NAME_MAX, ++ * or NAME_MAX-1 if name ends with '=' to avoid decoding issue ++ */ ++ presented_len = NAME_MAX; ++ if (oname->name[presented_len - 1] == '=') ++ presented_len--; ++ oname->len = presented_len; ++ } ++ oname->name[presented_len] = '\0'; ++ ++ return 0; ++} ++ ++static inline int ext4_setup_filename(struct inode *dir, ++ const struct qstr *iname, ++ int lookup, ++ struct ext4_filename *fname) ++{ ++ fname->usr_fname = iname; ++ ++ if (lookup && IS_ENCRYPTED(dir) && ++ unlikely(!IS_LUSTRE_MOUNT(dir->i_sb) && ++ strnchr(iname->name, iname->len, '='))) { ++ /* Only proceed to critical decode if ++ * iname contains escape char '='. ++ */ ++ int len = iname->len; ++ char *buf; ++ ++ buf = kmalloc(len, GFP_NOFS); ++ if (!buf) ++ return -ENOMEM; ++ ++ len = critical_decode(iname->name, len, buf); ++ fname->disk_name.name = (unsigned char *)buf; ++ fname->disk_name.len = len; ++ return 0; ++ } ++ ++ fname->disk_name.name = (unsigned char *) iname->name; ++ fname->disk_name.len = iname->len; ++ ++#ifdef CONFIG_UNICODE ++ ext4_fname_setup_ci_filename(dir, iname, &fname->cf_name); ++#endif ++ return 0; ++} ++ ++#endif /* _CRITICAL_ENCODE_H */ +diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c +index 5ac186aa..852bf8dd 100644 +--- a/fs/ext4/dir.c ++++ b/fs/ext4/dir.c +@@ -29,6 +29,7 @@ + #include + #include "ext4.h" + #include "xattr.h" ++#include "critical_encode.h" + + static int ext4_dx_readdir(struct file *, struct dir_context *); + +@@ -118,7 +119,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) + struct buffer_head *bh = NULL; + struct fscrypt_str fstr = FSTR_INIT(NULL, 0); + +- err = fscrypt_prepare_readdir(inode); ++ err = ext4_prepare_readdir(inode); + if (err) + return err; + +@@ -145,7 +146,8 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) + return err; + } + +- if (IS_ENCRYPTED(inode)) { ++ /* disable decryption of filename, present only escaped name */ ++ if (0 && IS_ENCRYPTED(inode)) { + err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN, &fstr); + if (err < 0) + return err; +@@ -258,22 +260,33 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) + get_dtype(sb, de->file_type))) + goto done; + } else { +- int save_len = fstr.len; + struct fscrypt_str de_name = + FSTR_INIT(de->name, + de->name_len); ++ int presented_len; + + /* Directory is encrypted */ +- err = fscrypt_fname_disk_to_usr(inode, ++ presented_len = critical_chars(de->name, ++ de->name_len); ++ err = ext4_fname_alloc_buffer(inode, ++ presented_len, ++ &fstr); ++ if (err) ++ goto errout; ++ ++ err = ext4_fname_disk_to_usr(inode, + 0, 0, &de_name, &fstr); + de_name = fstr; +- fstr.len = save_len; +- if (err) ++ if (err) { ++ ext4_fname_free_buffer(&fstr); + goto errout; +- if (!dir_emit(ctx, ++ } ++ err = dir_emit(ctx, + de_name.name, de_name.len, + le32_to_cpu(de->inode), +- get_dtype(sb, de->file_type))) ++ get_dtype(sb, de->file_type)); ++ ext4_fname_free_buffer(&fstr); ++ if (!err) + goto done; + } + } +diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c +index bf29a4e3..989834b3 100644 +--- a/fs/ext4/ialloc.c ++++ b/fs/ext4/ialloc.c +@@ -30,6 +30,7 @@ + #include "ext4_jbd2.h" + #include "xattr.h" + #include "acl.h" ++#include "critical_encode.h" + + #include + +diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c +index b1af5ef5..4c2456fe 100644 +--- a/fs/ext4/namei.c ++++ b/fs/ext4/namei.c +@@ -41,6 +41,7 @@ + + #include "xattr.h" + #include "acl.h" ++#include "critical_encode.h" + + #include + /* +@@ -1342,7 +1343,7 @@ static int htree_dirblock_to_tree(struct file *dir_file, + dir->i_sb->s_blocksize - + EXT4_DIR_REC_LEN(0)); + /* Check if the directory is encrypted */ +- if (IS_ENCRYPTED(dir)) { ++ if (0 && IS_ENCRYPTED(dir)) { + err = fscrypt_prepare_readdir(dir); + if (err < 0) { + brelse(bh); +@@ -1378,22 +1379,31 @@ static int htree_dirblock_to_tree(struct file *dir_file, + hinfo->hash, hinfo->minor_hash, de, + &tmp_str); + } else { +- int save_len = fname_crypto_str.len; + struct fscrypt_str de_name = FSTR_INIT(de->name, + de->name_len); ++ int presented_len; + + /* Directory is encrypted */ +- err = fscrypt_fname_disk_to_usr(dir, hinfo->hash, ++ presented_len = critical_chars(de->name, de->name_len); ++ err = ext4_fname_alloc_buffer(dir, presented_len, ++ &fname_crypto_str); ++ if (err) { ++ count = err; ++ goto errout; ++ } ++ ++ err = ext4_fname_disk_to_usr(dir, hinfo->hash, + hinfo->minor_hash, &de_name, + &fname_crypto_str); + if (err) { ++ ext4_fname_free_buffer(&fname_crypto_str); + count = err; + goto errout; + } + err = ext4_htree_store_dirent(dir_file, + hinfo->hash, hinfo->minor_hash, de, + &fname_crypto_str); +- fname_crypto_str.len = save_len; ++ ext4_fname_free_buffer(&fname_crypto_str); + } + if (err != 0) { + count = err; +@@ -1677,7 +1687,7 @@ void ext4_fname_setup_ci_filename(struct inode *dir, const struct qstr *iname, + */ + static inline bool ext4_match(const struct inode *parent, + const struct ext4_filename *fname, +- const struct ext4_dir_entry_2 *de) ++ struct ext4_dir_entry_2 *de, int denamelen) + { + struct fscrypt_name f; + #ifdef CONFIG_UNICODE +@@ -1705,7 +1715,7 @@ static inline bool ext4_match(const struct inode *parent, + } + #endif + +- return fscrypt_match_name(&f, de->name, de->name_len); ++ return fscrypt_match_name(&f, de->name, denamelen); + } + + /* +@@ -1716,16 +1726,30 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, + unsigned int offset, struct ext4_dir_entry_2 **res_dir) + { + struct ext4_dir_entry_2 * de; ++ bool probablytrunc; + char * dlimit; +- int de_len; ++ int de_len, denamelen; + + de = (struct ext4_dir_entry_2 *)search_buf; + dlimit = search_buf + buf_size; ++ /* fname is probably truncated if it is the decoded representation of ++ * an encrypted filename not aligned on a 32-byte boundary ++ */ ++ probablytrunc = !IS_LUSTRE_MOUNT(dir->i_sb) && IS_ENCRYPTED(dir) && ++ fname->disk_name.len & 31; + while ((char *) de < dlimit) { + /* this code is executed quadratically often */ + /* do minimal checking `by hand' */ ++ denamelen = de->name_len; ++ if (unlikely(probablytrunc) && ++ de->name_len > fname->disk_name.len) ++ /* Adjust name len to look for a partial match. ++ * Since it is binary encrypted names, there ++ * should not be any collision between names. ++ */ ++ denamelen = fname->disk_name.len; + if ((char *) de + de->name_len <= dlimit && +- ext4_match(dir, fname, de)) { ++ ext4_match(dir, fname, de, denamelen)) { + /* found a match - just to be sure, do + * a full check */ + if (ext4_check_dir_entry(dir, NULL, de, bh, search_buf, +@@ -1927,7 +1951,7 @@ struct buffer_head *ext4_find_entry_locked(struct inode *dir, + struct ext4_filename fname; + struct buffer_head *bh; + +- err = ext4_fname_setup_filename(dir, d_name, 1, &fname); ++ err = ext4_setup_filename(dir, d_name, 1, &fname); + if (err == -ENOENT) + return NULL; + if (err) +@@ -1935,7 +1959,9 @@ struct buffer_head *ext4_find_entry_locked(struct inode *dir, + + bh = __ext4_find_entry(dir, &fname, res_dir, inlined, lck); + +- ext4_fname_free_filename(&fname); ++ if (fname.disk_name.name != d_name->name) ++ kfree(fname.disk_name.name); ++ + return bh; + } + +@@ -1949,7 +1975,7 @@ static struct buffer_head *ext4_lookup_entry(struct inode *dir, + struct ext4_filename fname; + struct buffer_head *bh; + +- err = ext4_fname_prepare_lookup(dir, dentry, &fname); ++ err = ext4_setup_filename(dir, &dentry->d_name, 1, &fname); + generic_set_encrypted_ci_d_ops(dentry); + if (err == -ENOENT) + return NULL; +@@ -1958,7 +1984,9 @@ + + bh = __ext4_find_entry(dir, &fname, res_dir, NULL, NULL); + +- ext4_fname_free_filename(&fname); ++ if (fname.disk_name.name != dentry->d_name.name) ++ kfree(fname.disk_name.name); ++ + return bh; + } + +@@ -2050,7 +2078,7 @@ static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsi + } + if (!IS_ERR(inode) && IS_ENCRYPTED(dir) && + (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) && +- !fscrypt_has_permitted_context(dir, inode)) { ++ !ext4_has_permitted_context(dir, inode)) { + ext4_warning(inode->i_sb, + "Inconsistent encryption contexts: %lu/%lu", + dir->i_ino, inode->i_ino); +@@ -2329,7 +2357,7 @@ int ext4_find_dest_de(struct inode *dir, struct inode *inode, + if (ext4_check_dir_entry(dir, NULL, de, bh, + buf, buf_size, offset)) + return -EFSCORRUPTED; +- if (ext4_match(dir, fname, de)) ++ if (ext4_match(dir, fname, de, de->name_len)) + return -EEXIST; + nlen = EXT4_DIR_ENTRY_LEN(de); + rlen = ext4_rec_len_from_disk(de->rec_len, buf_size); diff --git a/ldiskfs/kernel_patches/series/ldiskfs-5.11.0-40-ubuntu20.series b/ldiskfs/kernel_patches/series/ldiskfs-5.11.0-40-ubuntu20.series index 68a0f09..cca167d 100644 --- a/ldiskfs/kernel_patches/series/ldiskfs-5.11.0-40-ubuntu20.series +++ b/ldiskfs/kernel_patches/series/ldiskfs-5.11.0-40-ubuntu20.series @@ -29,8 +29,9 @@ linux-5.8/ext4-no-max-dir-size-limit-for-iam-objects.patch rhel8/ext4-ialloc-uid-gid-and-pass-owner-down.patch base/ext4-projid-xattrs.patch linux-5.8/ext4-enc-flag.patch -base/ext4-delayed-iput.patch +oe2203/ext4-delayed-iput.patch linux-5.10/ext4-fiemap-kernel-data.patch rhel8/ext4-old_ea_inodes_handling_fix.patch +ubuntu20.04.3/ext4-filename-encode.patch rhel8/ext4-encdata.patch -rhel9/ext4-add-periodic-superblock-update.patch +ubuntu20.04.3/ext4-add-periodic-superblock-update.patch -- 1.8.3.1