From 915239f279f9ae3a5d510aed4763a263a9156628 Mon Sep 17 00:00:00 2001 From: Alexey Lyashkov Date: Wed, 23 Apr 2025 11:23:53 +0300 Subject: [PATCH 01/16] LU-18942 obdclass: rework limits for zfs ZFS ARC is uncontrolled by Linux memory, so this size should not be accounted for all caches, not just the lu_object cache. Let's reduce the number of objects freed in a single batch to avoid high CPU usage in ARC prune threads and increase latency in providing free space. Test-Parameters: trivial HPE-bug-id: LUS-12814, LUS-12813 Fixes: 79b4ae9139c ("LU-1305 osd: osd_handler") Fixes: 0123baecc4e ("LU-5164 osd: Limit lu_object cache") Signed-off-by: Alexey Lyashkov Change-Id: I5342149b185c61c56087d970f26eb4f197a597ef Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/58918 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- libcfs/include/libcfs/linux/linux-mem.h | 10 +-------- libcfs/libcfs/linux/linux-prim.c | 19 +++++++++++++++++ lustre/include/lu_object.h | 2 ++ lustre/obdclass/lu_object.c | 36 ++++++++++++++------------------- lustre/osd-zfs/osd_handler.c | 2 +- 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/libcfs/include/libcfs/linux/linux-mem.h b/libcfs/include/libcfs/linux/linux-mem.h index 18a900e..fa35779 100644 --- a/libcfs/include/libcfs/linux/linux-mem.h +++ b/libcfs/include/libcfs/linux/linux-mem.h @@ -28,15 +28,7 @@ #include #endif -#ifdef HAVE_TOTALRAM_PAGES_AS_FUNC - #ifndef cfs_totalram_pages - #define cfs_totalram_pages() totalram_pages() - #endif -#else - #ifndef cfs_totalram_pages - #define cfs_totalram_pages() totalram_pages - #endif -#endif +unsigned long cfs_totalram_pages(void); #ifndef HAVE_MEMALLOC_RECLAIM static inline unsigned int memalloc_noreclaim_save(void) diff --git a/libcfs/libcfs/linux/linux-prim.c b/libcfs/libcfs/linux/linux-prim.c index d50db3d..bbb24c7 100644 --- a/libcfs/libcfs/linux/linux-prim.c +++ b/libcfs/libcfs/linux/linux-prim.c @@ -408,3 +408,22 @@ char *nla_strdup(const struct nlattr *nla, gfp_t flags) } EXPORT_SYMBOL(nla_strdup); #endif /* !HAVE_NLA_STRDUP */ + +static unsigned int libcfs_reserved_cache; +module_param(libcfs_reserved_cache, int, 0644); +MODULE_PARM_DESC(libcfs_reserved_cache, "system page cache reservation in mbytes (for arc cache)"); + +#ifdef HAVE_TOTALRAM_PAGES_AS_FUNC + #define _totalram_pages() totalram_pages() +#else + #define _totalram_pages() totalram_pages +#endif + +unsigned long cfs_totalram_pages(void) +{ + if (libcfs_reserved_cache > _totalram_pages()/2) + libcfs_reserved_cache = _totalram_pages() / 2; + + return _totalram_pages() - libcfs_reserved_cache; +} +EXPORT_SYMBOL(cfs_totalram_pages); diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index 1e83626..4b57649 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -737,6 +737,8 @@ void lu_object_unhash(const struct lu_env *env, struct lu_object *o); int lu_site_purge_objects(const struct lu_env *env, struct lu_site *s, int nr, int canblock); +void lu_site_limit(const struct lu_env *env, struct lu_site *s, u64 limit); + static inline int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr) { diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index b77dec4..61b6a01 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -62,11 +62,9 @@ enum { LU_CACHE_PERCENT_DEFAULT = 20 }; -#define LU_CACHE_NR_MAX_ADJUST 512 +#define LU_CACHE_NR_MAX_ADJUST 1024 #define LU_CACHE_NR_UNLIMITED -1 #define LU_CACHE_NR_DEFAULT LU_CACHE_NR_UNLIMITED -/** This is set to roughly (20 * OSS_NTHRS_MAX) to prevent thrashing */ -#define LU_CACHE_NR_ZFS_LIMIT 10240 #define LU_CACHE_NR_MIN 4096 #define LU_CACHE_NR_MAX 0x80000000UL @@ -622,23 +620,30 @@ int lu_object_invariant(const struct lu_object *o) * maximum number of objects is capped by LU_CACHE_MAX_ADJUST. This ensures * that many concurrent threads will not accidentally purge the entire cache. */ -static void lu_object_limit(const struct lu_env *env, - struct lu_device *dev) +void lu_site_limit(const struct lu_env *env, struct lu_site *s, + u64 nr) { - u64 size, nr; + u64 size; - if (lu_cache_nr == LU_CACHE_NR_UNLIMITED) + if (nr == LU_CACHE_NR_UNLIMITED) return; - size = atomic_read(&dev->ld_site->ls_obj_hash.nelems); - nr = (u64)lu_cache_nr; + size = atomic_read(&s->ls_obj_hash.nelems); if (size <= nr) return; - lu_site_purge_objects(env, dev->ld_site, + lu_site_purge_objects(env, s, min_t(u64, size - nr, LU_CACHE_NR_MAX_ADJUST), 0); } +EXPORT_SYMBOL(lu_site_limit); + +static void lu_object_limit(const struct lu_env *env, + struct lu_device *dev) +{ + lu_site_limit(env, dev->ld_site, (u64)lu_cache_nr); +} + static struct lu_object *htable_lookup(const struct lu_env *env, struct lu_device *dev, @@ -1001,17 +1006,6 @@ static void lu_htable_limits(struct lu_device *top) unsigned long cache_size; /* - * For ZFS based OSDs the cache should be disabled by default. This - * allows the ZFS ARC maximum flexibility in determining what buffers - * to cache. If Lustre has objects or buffer which it wants to ensure - * always stay cached it must maintain a hold on them. - */ - if (strcmp(top->ld_type->ldt_name, LUSTRE_OSD_ZFS_NAME) == 0) { - lu_cache_nr = LU_CACHE_NR_ZFS_LIMIT; - return; - } - - /* * Calculate hash table size, assuming that we want reasonable * performance when 20% of total memory is occupied by cache of * lu_objects. diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index 1853b4c..6ef6fd5 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -88,7 +88,7 @@ static void arc_prune_func(int64_t bytes, void *private) return; } - lu_site_purge(&env, site, (bytes >> 10)); + lu_site_limit(&env, site, (bytes >> 10)); lu_env_fini(&env); } -- 1.8.3.1 From 0efa1b99e26e454bb6dd71574541e30a10030936 Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Sat, 3 May 2025 17:52:19 +0300 Subject: [PATCH 02/16] LU-18973 mgc: account failovers from all sources Once set up initially MGC is not updating import failovers from other mounts. That causes problems with MGC on MGS - it is always set up with only @lo interface, so if MGS failed over to other node, all targets/clients on primary node are unable to find MGS, because MGC has only @lo peer Patch reworks lustre_start_mgc() code to account all failover peers from each user of that MGC. It adds new failover NIDs even if MGC exists already. Patch re-organizes also the way how peers are identified. It uses peer UUID as 'Primary NID' string instead of naming it as 'MGC_##' so same NIDs don't produces new mappings and don't pollute import with duplicated connections. That makes LCFG_DEL_UUID obsoleted as well, because lustre_stop_mgc() was its last user. Signed-off-by: Mikhail Pershin Change-Id: Icea5b74a16972e8a5f2737257086074630e652a8 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59076 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Sebastien Buisson Reviewed-by: Marc Vef Reviewed-by: Oleg Drokin --- lustre/obdclass/obd_mount.c | 167 ++++++++++++++++---------------------------- lustre/tests/conf-sanity.sh | 31 ++++++++ 2 files changed, 92 insertions(+), 106 deletions(-) diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 1165b5d..35030fc 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -199,44 +199,58 @@ SERVER_ONLY_EXPORT_SYMBOL(lustre_start_simple); static DEFINE_MUTEX(mgc_start_lock); -/* 9 for '_%x' (INT_MAX as hex is 8 chars - '7FFFFFFF') and 1 for '\0' */ -#define NIDUUID_SUFFIX_MAX_LEN 10 -static inline int mgc_niduuid_create(char **niduuid, char *nidstr) +/** + * Parse MGS failover nodes from provided NID list and add + * them to existing MGC import + */ +static bool lustre_add_mgc_failnodes(struct obd_device *obd, char *ptr) { - size_t niduuid_len = strlen(nidstr) + strlen(LUSTRE_MGC_OBDNAME) + - NIDUUID_SUFFIX_MAX_LEN; - - LASSERT(niduuid); + struct obd_import *imp = obd->u.cli.cl_import; + char node[LNET_NIDSTR_SIZE]; + struct lnet_nid nid; + int rc; + bool large_nids = false; - /* See comment in niduuid_create() */ - if (niduuid_len > UUID_MAX) { - nidstr += niduuid_len - UUID_MAX; - niduuid_len = strlen(LUSTRE_MGC_OBDNAME) + - strlen(nidstr) + NIDUUID_SUFFIX_MAX_LEN; - } + LASSERT(imp); - OBD_ALLOC(*niduuid, niduuid_len); - if (!*niduuid) - return -ENOMEM; + /* Add any failover MGS NIDs */ + while (ptr) { + int count = 0; - snprintf(*niduuid, niduuid_len, "%s%s", LUSTRE_MGC_OBDNAME, nidstr); - return 0; -} + while (class_parse_nid_quiet(ptr, &nid, &ptr) == 0) { + large_nids |= !nid_is_nid4(&nid); -static inline void mgc_niduuid_destroy(char **niduuid) -{ - if (*niduuid) { - char *tmp = strchr(*niduuid, '_'); + /* New failover node */ + if (!count) /* construct node UUID from primary NID */ + libcfs_nidstr_r(&nid, node, LNET_NIDSTR_SIZE); - /* If the "_%x" suffix hasn't been added yet then the size - * calculation below should still be correct - */ - if (tmp) - *tmp = '\0'; - - OBD_FREE(*niduuid, strlen(*niduuid) + NIDUUID_SUFFIX_MAX_LEN); + rc = class_add_uuid(node, &nid); + if (rc) { + libcfs_nidstr_r(&nid, node, LNET_NIDSTR_SIZE); + CWARN("%s: can't add failover NID %s, rc = %d\n", + obd->obd_name, node, rc); + } else { + count++; + } + if (*ptr == ':') + break; + } + /* if new peer mapping was created */ + if (count > 0) { + struct obd_uuid uuid; + + obd_str2uuid(&uuid, node); + rc = obd_add_conn(imp, &uuid, 0); + if (rc) + CWARN("%s: can't add failover peer %s, rc = %d\n", + obd->obd_name, node, rc); + } else { + /* at ":/fsname" */ + break; + } } - *niduuid = NULL; + + return large_nids; } /** @@ -256,10 +270,10 @@ int lustre_start_mgc(struct super_block *sb) uuid_t uuidc; struct lnet_nid nid; char nidstr[LNET_NIDSTR_SIZE]; - char *mgcname = NULL, *niduuid = NULL, *mgssec = NULL; + char *mgcname = NULL, *mgssec = NULL; bool large_nids = false; - char *ptr, *niduuid_suffix; - int rc = 0, i = 0, j; + char *ptr; + int rc = 0, i = 0; size_t len; ENTRY; @@ -306,8 +320,7 @@ int lustre_start_mgc(struct super_block *sb) libcfs_nidstr_r(&nid, nidstr, sizeof(nidstr)); len = strlen(LUSTRE_MGC_OBDNAME) + strlen(nidstr) + 1; OBD_ALLOC(mgcname, len); - rc = mgc_niduuid_create(&niduuid, nidstr); - if (rc || mgcname == NULL) + if (!mgcname) GOTO(out_free, rc = -ENOMEM); snprintf(mgcname, len, "%s%s", LUSTRE_MGC_OBDNAME, nidstr); @@ -357,6 +370,8 @@ int lustre_start_mgc(struct super_block *sb) } } + lustre_add_mgc_failnodes(obd, ptr); + recov_bk = 0; /* * If we are restarting the MGS, don't try to keep the MGC's @@ -386,9 +401,8 @@ int lustre_start_mgc(struct super_block *sb) /* Add the primary NIDs for the MGS */ i = 0; - niduuid_suffix = niduuid + strlen(niduuid); - snprintf(niduuid_suffix, NIDUUID_SUFFIX_MAX_LEN, "_%x", i); if (IS_SERVER(lsi)) { + /* All mgsnode are listed in lmd_mgs at this moment */ ptr = lsi->lsi_lmd->lmd_mgs; CDEBUG(D_MOUNT, "mgs NIDs %s.\n", ptr); if (IS_MGS(lsi)) { @@ -397,16 +411,11 @@ int lustre_start_mgc(struct super_block *sb) while ((rc = LNetGetId(i++, &id, true)) != -ENOENT) { rc = do_lcfg_nid(mgcname, &id.nid, - LCFG_ADD_UUID, - niduuid); + LCFG_ADD_UUID, nidstr); } } else { - /* Use mgsnode= nids */ - /* mount -o mgsnode=nid */ - if (lsi->lsi_lmd->lmd_mgs) { - ptr = lsi->lsi_lmd->lmd_mgs; - } else if (class_find_param(ptr, PARAM_MGSNODE, - &ptr) != 0) { + /* Target must have at least one mgsnode */ + if (!ptr) { CERROR("No MGS NIDs given.\n"); GOTO(out_free, rc = -EINVAL); } @@ -417,8 +426,7 @@ int lustre_start_mgc(struct super_block *sb) */ while (class_parse_nid(ptr, &nid, &ptr) == 0) { rc = do_lcfg_nid(mgcname, &nid, - LCFG_ADD_UUID, - niduuid); + LCFG_ADD_UUID, nidstr); if (rc == 0) ++i; /* Stop at the first failover NID */ @@ -430,8 +438,7 @@ int lustre_start_mgc(struct super_block *sb) /* Use NIDs from mount line: uml1,1@elan:uml2,2@elan:/lustre */ ptr = lsi->lsi_lmd->lmd_dev; while (class_parse_nid(ptr, &nid, &ptr) == 0) { - rc = do_lcfg_nid(mgcname, &nid, LCFG_ADD_UUID, - niduuid); + rc = do_lcfg_nid(mgcname, &nid, LCFG_ADD_UUID, nidstr); if (rc == 0) ++i; /* Stop at the first failover NID */ @@ -443,7 +450,6 @@ int lustre_start_mgc(struct super_block *sb) CERROR("No valid MGS NIDs found.\n"); GOTO(out_free, rc = -EINVAL); } - lsi->lsi_lmd->lmd_mgs_failnodes = 1; /* Random uuid for MGC allows easier reconnects */ OBD_ALLOC_PTR(uuid); @@ -456,46 +462,18 @@ int lustre_start_mgc(struct super_block *sb) /* Start the MGC */ rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME, (char *)uuid->uuid, LUSTRE_MGS_OBDNAME, - niduuid, NULL, lsi->lsi_lmd->lmd_nidnet); + nidstr, NULL, lsi->lsi_lmd->lmd_nidnet); if (rc) GOTO(out_free, rc); - /* Add any failover MGS NIDs */ - i = 1; - while (ptr && ((*ptr == ':' || - class_find_param(ptr, PARAM_MGSNODE, &ptr) == 0))) { - /* New failover node */ - snprintf(niduuid_suffix, NIDUUID_SUFFIX_MAX_LEN, "_%x", i); - j = 0; - while (class_parse_nid_quiet(ptr, &nid, &ptr) == 0) { - if (!nid_is_nid4(&nid)) - large_nids = true; - - rc = do_lcfg_nid(mgcname, &nid, LCFG_ADD_UUID, - niduuid); - if (rc == 0) - ++j; - if (*ptr == ':') - break; - } - if (j > 0) { - rc = do_lcfg(mgcname, 0, LCFG_ADD_CONN, - niduuid, NULL, NULL, NULL); - if (rc == 0) - ++i; - } else { - /* at ":/fsname" */ - break; - } - } - lsi->lsi_lmd->lmd_mgs_failnodes = i; - obd = class_name2obd(mgcname); if (!obd) { CERROR("Can't find mgcobd %s\n", mgcname); GOTO(out_free, rc = -ENOTCONN); } + large_nids = lustre_add_mgc_failnodes(obd, ptr); + rc = obd_set_info_async(NULL, obd->obd_self_export, strlen(KEY_MGSSEC), KEY_MGSSEC, strlen(mgssec), mgssec, NULL); @@ -551,7 +529,6 @@ out_free: OBD_FREE_PTR(uuid); OBD_FREE_PTR(data); OBD_FREE(mgcname, len); - mgc_niduuid_destroy(&niduuid); RETURN(rc); } @@ -561,9 +538,7 @@ SERVER_ONLY int lustre_stop_mgc(struct super_block *sb) { struct lustre_sb_info *lsi = s2lsi(sb); struct obd_device *obd; - char *niduuid = NULL, *niduuid_suffix; - char nidstr[LNET_NIDSTR_SIZE]; - int i, rc = 0; + int rc = 0; ENTRY; @@ -574,16 +549,6 @@ SERVER_ONLY int lustre_stop_mgc(struct super_block *sb) RETURN(-ENOENT); lsi->lsi_mgc = NULL; - /* Reconstruct the NID uuid from the obd_name */ - strscpy(nidstr, &obd->obd_name[0] + strlen(LUSTRE_MGC_OBDNAME), - sizeof(nidstr)); - - rc = mgc_niduuid_create(&niduuid, nidstr); - if (rc) - RETURN(-ENOMEM); - - niduuid_suffix = niduuid + strlen(niduuid); - mutex_lock(&mgc_start_lock); LASSERT(atomic_read(&obd->u.cli.cl_mgc_refcount) > 0); if (!atomic_dec_and_test(&obd->u.cli.cl_mgc_refcount)) { @@ -616,20 +581,10 @@ SERVER_ONLY int lustre_stop_mgc(struct super_block *sb) if (rc) GOTO(out, rc); - for (i = 0; i < lsi->lsi_lmd->lmd_mgs_failnodes; i++) { - snprintf(niduuid_suffix, NIDUUID_SUFFIX_MAX_LEN, "_%x", i); - rc = do_lcfg(LUSTRE_MGC_OBDNAME, 0, LCFG_DEL_UUID, - niduuid, NULL, NULL, NULL); - if (rc) - CERROR("del MDC UUID %s failed: rc = %d\n", - niduuid, rc); - } out: /* class_import_put will get rid of the additional connections */ mutex_unlock(&mgc_start_lock); - mgc_niduuid_destroy(&niduuid); - RETURN(rc); } SERVER_ONLY_EXPORT_SYMBOL(lustre_stop_mgc); diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 7699250..2e8b86d 100755 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -12052,6 +12052,37 @@ test_155() { } run_test 155 "gap in seq allocation from ofd after restarting" +test_160() { + ((OST1_VERSION >= $(version_code 2.16.55) )) || + skip "need OST >= 2.16.55 to have MGC with all failovers" + + stopall + reformat + + local mgs_nid=$(do_facet mgs $LCTL list_nids | head -1) + local failover_nid="192.168.252.160@${NETTYPE}" + local mgs_nodes=$mgs_nid:$failover_nid + local tmp_mnt="$TMP/lmount" + local count; + + start_mgsmds + start_ost + + stack_trap "cleanup; reformat" + + do_facet mgs "mkdir -p $tmp_mnt" + do_facet mgs "$MOUNT_CMD $mgs_nodes:/$FSNAME $tmp_mnt" || + error "Fail to mount local client on MGS" + do_facet mgs "umount $tmp_mnt" + + do_facet mgs "$LCTL get_param mgc.MGC${mgs_nid}.import" + count=$(do_facet mgs "$LCTL get_param mgc.MGC${mgs_nid}.import" | + grep -c "$failover_nid") + (( count > 0 )) || + error "MGC misses failover MGS nid" +} +run_test 160 "MGC updates failnodes from all participants" + cleanup_200() { local modopts=$1 stopall -- 1.8.3.1 From c92cc0c9062bd4c99983c766d4e5e73622e0ea05 Mon Sep 17 00:00:00 2001 From: Oleg Drokin Date: Wed, 11 Jun 2025 18:49:10 -0400 Subject: [PATCH 03/16] LU-19099 llite: Add vfstrace debug prints for ll_fallocate Currently ll_fallocate() does not have any vfstrace prints, but it's important for debugging related issues. Fixes: 48457868a02a ("LU-3606 fallocate: Implement fallocate preallocate operation") Test-Parameters: trivial Change-Id: I165a7208a59b055756db416063c6695cc7f2d8e4 Signed-off-by: Oleg Drokin Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59718 Reviewed-by: Andreas Dilger Reviewed-by: Arshad Hussain Reviewed-by: Emoly Liu Tested-by: jenkins Tested-by: Maloo --- lustre/llite/file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 5b6073d..fdd0ebc 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -6588,6 +6588,9 @@ static long ll_fallocate(struct file *filp, int mode, loff_t offset, loff_t len) struct inode *inode = file_inode(filp); int rc; + CDEBUG(D_VFSTRACE, "VFS Op: "DNAME", mode %x, offset %lld, len %lld\n", + encode_fn_file(filp), mode, offset, len); + if (offset < 0 || len <= 0) RETURN(-EINVAL); /* -- 1.8.3.1 From 30c3fe01d93128ff636de1d3c12217a77b9fa623 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Fri, 25 Apr 2025 05:49:03 +0300 Subject: [PATCH 04/16] LU-18602 osc: don't block async enqueues don't block async enqueue RPCs trying to resend, otherwise the client can get into a deadlock awaiting for RPC pinning object-inodes at umount: CPU: 0 PID: 9863 Comm: umount Call Trace: dump_stack+0x6e/0xa0 lbug_with_loc.cold.4+0x5/0x63 [libcfs] lov_delete_composite+0x45b/0x680 [lov] lov_object_delete+0xc1/0x260 [lov] lu_object_free.isra.5+0x76/0x190 [obdclass] cl_inode_fini+0xeb/0x250 [lustre] ll_clear_inode+0x269/0x620 [lustre] ll_delete_inode+0x3b/0x140 [lustre] evict+0xbc/0x180 dispose_list+0x38/0x60 evict_inodes+0x12e/0x170 generic_shutdown_super+0x2d/0xf0 kill_anon_super+0x9/0x20 deactivate_locked_super+0x24/0x60 cleanup_mnt+0x36/0x70 Signed-off-by: Alex Zhuravlev Change-Id: Id1c6c71414b7387b9b0b191b65e34cf65388b57f Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/57599 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexey Lyashkov Reviewed-by: Oleg Drokin Reviewed-by: Andreas Dilger --- lustre/osc/osc_request.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index ab34b8c..cf5e0a7 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3246,6 +3246,8 @@ int osc_enqueue_base(struct obd_export *exp, struct ldlm_res_id *res_id, */ aa->oa_lvb = NULL; aa->oa_flags = NULL; + /* don't block async enqueue RPCs trying to resend */ + req->rq_no_delay = req->rq_no_resend = 1; } req->rq_interpret_reply = osc_enqueue_interpret; -- 1.8.3.1 From 7055e949d3f5fc8318577dab9967ad74cac832c6 Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Sat, 8 Mar 2025 13:33:28 -0500 Subject: [PATCH 05/16] LU-16518 lst: fix switch-case unannotated fall-through (2) Fix more unannotated fall-through errors reported by Clang, by moving the default case to the end of the switch-case statement. Test-Parameters: trivial Signed-off-by: Timothy Day Change-Id: I12fb3bf709f2edd6ef03f58c122255319dc91049 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/58345 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lnet/selftest/framework.c | 4 ++-- lnet/selftest/rpc.c | 4 ++-- lnet/selftest/selftest.h | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lnet/selftest/framework.c b/lnet/selftest/framework.c index 9bec7f5..765ce67 100644 --- a/lnet/selftest/framework.c +++ b/lnet/selftest/framework.c @@ -1293,8 +1293,6 @@ sfw_handle_server_rpc(struct srpc_server_rpc *rpc) } switch (sv->sv_id) { - default: - LBUG(); case SRPC_SERVICE_TEST: rc = sfw_add_test(rpc); break; @@ -1323,6 +1321,8 @@ sfw_handle_server_rpc(struct srpc_server_rpc *rpc) rc = sfw_remove_session(&request->msg_body.rmsn_reqst, &reply->msg_body.rmsn_reply); break; + default: + LASSERTF(0, "sv_id bad %u\n", sv->sv_id); } if (sfw_data.fw_session != NULL) diff --git a/lnet/selftest/rpc.c b/lnet/selftest/rpc.c index d2893b8..615c25e 100644 --- a/lnet/selftest/rpc.c +++ b/lnet/selftest/rpc.c @@ -1219,8 +1219,6 @@ srpc_send_rpc(struct swi_workitem *wi) spin_unlock(&rpc->crpc_lock); switch (wi->swi_state) { - default: - LBUG(); case SWI_STATE_NEWBORN: LASSERT(!srpc_event_pending(rpc)); @@ -1304,6 +1302,8 @@ srpc_send_rpc(struct swi_workitem *wi) wi->swi_state = SWI_STATE_DONE; srpc_client_rpc_done(rpc, rc); return; + default: + LASSERTF(0, "swi_state bad %u\n", wi->swi_state); } if (rc != 0) { diff --git a/lnet/selftest/selftest.h b/lnet/selftest/selftest.h index 1467e926..a12f306 100644 --- a/lnet/selftest/selftest.h +++ b/lnet/selftest/selftest.h @@ -632,8 +632,6 @@ swi_state2str(int state) { #define STATE2STR(x) case x: return #x switch (state) { - default: - LBUG(); STATE2STR(SWI_STATE_NEWBORN); STATE2STR(SWI_STATE_REPLY_SUBMITTED); STATE2STR(SWI_STATE_REPLY_SENT); @@ -642,6 +640,9 @@ swi_state2str(int state) STATE2STR(SWI_STATE_REPLY_RECEIVED); STATE2STR(SWI_STATE_BULK_STARTED); STATE2STR(SWI_STATE_DONE); + default: + LASSERTF(0, "state bad %u\n", state); + return NULL; } #undef STATE2STR } -- 1.8.3.1 From c0103f630aa5128f3bce161218401be9d88816a3 Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Sat, 29 Mar 2025 19:29:40 -0400 Subject: [PATCH 06/16] LU-18687 doc: move man *.5 pages to Documentation/man5 Consolidate all of the man pages into the top level Documentation directory. Move all of the Lustre man pages (from 5) to Docmentation/. Test-Parameters: trivial Signed-off-by: Timothy Day Change-Id: Idac247218406378398fdbf7f84e779ed87c42eef Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/58589 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Lijing Chen Reviewed-by: Oleg Drokin --- Documentation/man5/Makefile.am | 4 +++- {lustre/doc => Documentation/man5}/ldev.conf.5 | 0 {lustre/doc => Documentation/man5}/nids.5 | 0 lustre/doc/Makefile.am | 2 -- 4 files changed, 3 insertions(+), 3 deletions(-) rename {lustre/doc => Documentation/man5}/ldev.conf.5 (100%) rename {lustre/doc => Documentation/man5}/nids.5 (100%) diff --git a/Documentation/man5/Makefile.am b/Documentation/man5/Makefile.am index 6e53ab5..52bca27 100644 --- a/Documentation/man5/Makefile.am +++ b/Documentation/man5/Makefile.am @@ -4,7 +4,9 @@ # This file is part of Lustre, http://www.lustre.org/ # -MANFILES = +MANFILES = \ + ldev.conf.5 \ + nids.5 if MANPAGES if UTILS diff --git a/lustre/doc/ldev.conf.5 b/Documentation/man5/ldev.conf.5 similarity index 100% rename from lustre/doc/ldev.conf.5 rename to Documentation/man5/ldev.conf.5 diff --git a/lustre/doc/nids.5 b/Documentation/man5/nids.5 similarity index 100% rename from lustre/doc/nids.5 rename to Documentation/man5/nids.5 diff --git a/lustre/doc/Makefile.am b/lustre/doc/Makefile.am index a2b6a79..e7a6cf1 100644 --- a/lustre/doc/Makefile.am +++ b/lustre/doc/Makefile.am @@ -43,7 +43,6 @@ MANFILES = \ lctl-pool_new.8 \ lctl-set_param.8 \ ldev.8 \ - ldev.conf.5 \ lfs-rm_entry.8 \ lfs-rmentry.8 \ lgss_sk.8 \ @@ -55,7 +54,6 @@ MANFILES = \ lustre_routes_conversion.8 \ lustre_rsync.8 \ mount.lustre.8 \ - nids.5 \ plot-llstat.8 \ routerstat.8 -- 1.8.3.1 From 74a7749450c4f43a0ee041b0e81b53dcd90ad0cd Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Sun, 1 Jun 2025 16:39:04 -0400 Subject: [PATCH 07/16] LU-18813 osd-wbcfs: make dt_last_seq_get() optional Allow osd-wbcfs to leave this unimplemented for now. Once we track object mapping internally, we can implement this. Fixes: 373b76b345b5 ("LU-17658 fid: check on disk sequence before allocating to osp") Test-Parameters: trivial Signed-off-by: Timothy Day Change-Id: Iea4401db0c7656fd56c43f6e0d296cabce89e864 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59502 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Li Dongyang Reviewed-by: Qian Yingjin Reviewed-by: Oleg Drokin --- lustre/include/dt_object.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lustre/include/dt_object.h b/lustre/include/dt_object.h index c3b05b0..21d70f5 100644 --- a/lustre/include/dt_object.h +++ b/lustre/include/dt_object.h @@ -3054,7 +3054,10 @@ static inline int dt_last_seq_get(const struct lu_env *env, { LASSERT(dev); LASSERT(dev->dd_ops); - LASSERT(dev->dd_ops->dt_last_seq_get); + + if (!dev->dd_ops->dt_last_seq_get) + return -EINVAL; + return dev->dd_ops->dt_last_seq_get(env, dev, seq); } -- 1.8.3.1 From 40585fc40f8da3deb9fe2d18d6accb812729d38b Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Tue, 15 Apr 2025 17:38:09 +0000 Subject: [PATCH 08/16] LU-18162 mdc: convert Metadata Client to LU device Convert MDC to use LU device init/fini rather than the legacy OBD API. Signed-off-by: Timothy Day Change-Id: Ic48accf1104d2845707b44519c32c5ced56ae8ef Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/58809 Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- lustre/include/lustre_osc.h | 8 ------ lustre/mdc/mdc_dev.c | 59 ++++++++++++++++++++++++++++++++++++--------- lustre/mdc/mdc_internal.h | 1 + lustre/mdc/mdc_request.c | 28 +-------------------- lustre/obdclass/lu_object.c | 12 +++++++-- lustre/osc/osc_dev.c | 15 +++++------- 6 files changed, 66 insertions(+), 57 deletions(-) diff --git a/lustre/include/lustre_osc.h b/lustre/include/lustre_osc.h index 44b763b..55027da 100644 --- a/lustre/include/lustre_osc.h +++ b/lustre/include/lustre_osc.h @@ -595,14 +595,6 @@ bool osc_page_gang_lookup(const struct lu_env *env, struct cl_io *io, bool osc_discard_cb(const struct lu_env *env, struct cl_io *io, void **pvec, int count, void *cbdata); -/* osc_dev.c */ -int osc_device_init(const struct lu_env *env, struct lu_device *d, - const char *name, struct lu_device *next); -struct lu_device *osc_device_fini(const struct lu_env *env, - struct lu_device *d); -struct lu_device *osc_device_free(const struct lu_env *env, - struct lu_device *d); - /* osc_object.c */ int osc_object_init(const struct lu_env *env, struct lu_object *obj, const struct lu_object_conf *conf); diff --git a/lustre/mdc/mdc_dev.c b/lustre/mdc/mdc_dev.c index ba5719e..67a7451 100644 --- a/lustre/mdc/mdc_dev.c +++ b/lustre/mdc/mdc_dev.c @@ -1676,21 +1676,36 @@ static const struct lu_device_operations mdc_lu_ops = { .ldo_recovery_complete = NULL, }; +static struct lu_device *mdc_device_free(const struct lu_env *env, + struct lu_device *lu) +{ + struct obd_device *obd = lu->ld_obd; + struct client_obd *cli = &obd->u.cli; + struct osc_device *osc = lu2osc_dev(lu); + + LASSERT(cli->cl_mod_rpcs_in_flight == 0); + cl_device_fini(lu2cl_dev(lu)); + osc_cleanup_common(obd); + OBD_FREE_PTR(osc); + + return NULL; +} + static struct lu_device *mdc_device_alloc(const struct lu_env *env, struct lu_device_type *t, struct lustre_cfg *cfg) { struct lu_device *d; - struct osc_device *oc; + struct osc_device *osc; struct obd_device *obd; int rc; - OBD_ALLOC_PTR(oc); - if (oc == NULL) + OBD_ALLOC_PTR(osc); + if (osc == NULL) RETURN(ERR_PTR(-ENOMEM)); - cl_device_init(&oc->osc_cl, t); - d = osc2lu_dev(oc); + cl_device_init(&osc->osc_cl, t); + d = osc2lu_dev(osc); d->ld_ops = &mdc_lu_ops; /* Setup MDC OBD */ @@ -1700,19 +1715,41 @@ static struct lu_device *mdc_device_alloc(const struct lu_env *env, rc = mdc_setup(obd, cfg); if (rc < 0) { - osc_device_free(env, d); + mdc_device_free(env, d); RETURN(ERR_PTR(rc)); } - oc->osc_exp = obd->obd_self_export; - oc->osc_stats.os_init = ktime_get_real(); + osc->osc_exp = obd->obd_self_export; + osc->osc_stats.os_init = ktime_get_real(); RETURN(d); } +static int mdc_device_init(const struct lu_env *env, struct lu_device *d, + const char *name, struct lu_device *next) +{ + RETURN(0); +} + +static struct lu_device *mdc_device_fini(const struct lu_env *env, + struct lu_device *lu) +{ + struct obd_device *obd = lu->ld_obd; + + ENTRY; + + osc_precleanup_common(obd); + mdc_changelog_cdev_finish(obd); + mdc_llog_finish(obd); + lprocfs_free_md_stats(obd); + ptlrpc_lprocfs_unregister_obd(obd); + + RETURN(NULL); +} + static const struct lu_device_type_operations mdc_device_type_ops = { .ldto_device_alloc = mdc_device_alloc, - .ldto_device_free = osc_device_free, - .ldto_device_init = osc_device_init, - .ldto_device_fini = osc_device_fini + .ldto_device_free = mdc_device_free, + .ldto_device_init = mdc_device_init, + .ldto_device_fini = mdc_device_fini }; struct lu_device_type mdc_device_type = { diff --git a/lustre/mdc/mdc_internal.h b/lustre/mdc/mdc_internal.h index f412576..cfe1f00 100644 --- a/lustre/mdc/mdc_internal.h +++ b/lustre/mdc/mdc_internal.h @@ -89,6 +89,7 @@ int mdc_resource_cancel_unused(struct obd_export *exp, const struct lu_fid *fid, int mdc_fid_alloc(const struct lu_env *env, struct obd_export *exp, struct lu_fid *fid, struct md_op_data *op_data); int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg); +void mdc_llog_finish(struct obd_device *obd); struct obd_client_handle; diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 42edc22..79c4f79 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -46,8 +46,6 @@ #define REQUEST_MINOR 244 -static int mdc_cleanup(struct obd_device *obd); - static inline int mdc_queue_wait(struct ptlrpc_request *req) { struct client_obd *cli = &req->rq_import->imp_obd->u.cli; @@ -2985,7 +2983,7 @@ static int mdc_llog_init(struct obd_device *obd) RETURN(0); } -static void mdc_llog_finish(struct obd_device *obd) +void mdc_llog_finish(struct obd_device *obd) { struct llog_ctxt *ctxt; @@ -3067,32 +3065,8 @@ static int mdc_init_ea_size(struct obd_export *exp, __u32 easize, RETURN(0); } -static int mdc_precleanup(struct obd_device *obd) -{ - ENTRY; - osc_precleanup_common(obd); - - mdc_changelog_cdev_finish(obd); - mdc_llog_finish(obd); - lprocfs_free_md_stats(obd); - ptlrpc_lprocfs_unregister_obd(obd); - - RETURN(0); -} - -static int mdc_cleanup(struct obd_device *obd) -{ - struct client_obd *cli = &obd->u.cli; - - LASSERT(cli->cl_mod_rpcs_in_flight == 0); - return osc_cleanup_common(obd); -} - static const struct obd_ops mdc_obd_ops = { .o_owner = THIS_MODULE, - .o_setup = mdc_setup, - .o_precleanup = mdc_precleanup, - .o_cleanup = mdc_cleanup, .o_add_conn = client_import_add_conn, .o_del_conn = client_import_del_conn, .o_connect = client_connect_import, diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index 61b6a01..604dc94 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -1335,15 +1335,23 @@ void lu_stack_fini(const struct lu_env *env, struct lu_device *top) lu_site_purge(env, site, ~0); for (scan = top; scan != NULL; scan = next) { - next = ldto_device_fini(env, scan); + if (strcmp(scan->ld_type->ldt_name, LUSTRE_MDC_NAME) == 0) + next = NULL; + else + next = ldto_device_fini(env, scan); + lu_device_put(scan); } /* purge again. */ lu_site_purge(env, site, ~0); - for (scan = top; scan != NULL; scan = next) + for (scan = top; scan != NULL; scan = next) { + if (strcmp(scan->ld_type->ldt_name, LUSTRE_MDC_NAME) == 0) + break; + next = ldto_device_free(env, scan); + } } EXPORT_SYMBOL(lu_stack_fini); diff --git a/lustre/osc/osc_dev.c b/lustre/osc/osc_dev.c index c329fbe..1e8cf74 100644 --- a/lustre/osc/osc_dev.c +++ b/lustre/osc/osc_dev.c @@ -149,22 +149,20 @@ static const struct lu_device_operations osc_lu_ops = { .ldo_recovery_complete = NULL }; -int osc_device_init(const struct lu_env *env, struct lu_device *d, - const char *name, struct lu_device *next) +static int osc_device_init(const struct lu_env *env, struct lu_device *d, + const char *name, struct lu_device *next) { RETURN(0); } -EXPORT_SYMBOL(osc_device_init); -struct lu_device *osc_device_fini(const struct lu_env *env, - struct lu_device *d) +static struct lu_device *osc_device_fini(const struct lu_env *env, + struct lu_device *d) { return NULL; } -EXPORT_SYMBOL(osc_device_fini); -struct lu_device *osc_device_free(const struct lu_env *env, - struct lu_device *d) +static struct lu_device *osc_device_free(const struct lu_env *env, + struct lu_device *d) { struct osc_device *oc = lu2osc_dev(d); @@ -172,7 +170,6 @@ struct lu_device *osc_device_free(const struct lu_env *env, OBD_FREE_PTR(oc); return NULL; } -EXPORT_SYMBOL(osc_device_free); static struct lu_device *osc_device_alloc(const struct lu_env *env, struct lu_device_type *t, -- 1.8.3.1 From d9b87ff95b7f52f7347f2fd9d51d33a5833cd9e1 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Mon, 19 May 2025 11:44:18 -0700 Subject: [PATCH 09/16] LU-18668 kernel: update RHEL 9.6 [5.14.0-570.17.1.el9_6] Update RHEL 9.6 kernel to 5.14.0-570.17.1.el9_6 for Lustre client. Test-Parameters: trivial env=SANITY_EXCEPT="17p" \ mdtcount=4 mdscount=2 clientdistro=el9.6 testlist=sanity Test-Parameters: optional clientdistro=el9.6 testgroup=full-part-1 Test-Parameters: optional clientdistro=el9.6 testgroup=full-part-2 Test-Parameters: optional clientdistro=el9.6 testgroup=full-part-3 Change-Id: Iac6973ac636953c1e64a60433ae72c0f692a24ca Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59292 Reviewed-by: Yang Sheng Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin Tested-by: Maloo Tested-by: jenkins --- lustre/ChangeLog | 2 +- lustre/kernel_patches/targets/5.14-rhel9.6.target.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index e9c9b34..615ab64 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -43,7 +43,7 @@ TBD Whamcloud 5.10.0-153.19.0.95.oe2203sp2 (openEuler 22.03 LTS SP2) * ldiskfs needs an ldiskfs patch series for that kernel, ZFS does not * Client primary kernels built and tested during release cycle: - 5.14.0-570.16.1.el9 (RHEL9.6) + 5.14.0-570.17.1.el9 (RHEL9.6) 5.14.0-503.40.1.el9 (RHEL9.5) 5.14.0-427.42.1.el9 (RHEL9.4) 5.14.0-362.24.1.el9 (RHEL9.3) diff --git a/lustre/kernel_patches/targets/5.14-rhel9.6.target.in b/lustre/kernel_patches/targets/5.14-rhel9.6.target.in index e19e98b..9e34ebe 100644 --- a/lustre/kernel_patches/targets/5.14-rhel9.6.target.in +++ b/lustre/kernel_patches/targets/5.14-rhel9.6.target.in @@ -1,5 +1,5 @@ lnxmaj="5.14.0" -lnxrel="570.16.1.el9_6" +lnxrel="570.17.1.el9_6" KERNEL_SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm SERIES="" -- 1.8.3.1 From ffac8cd8571145c1ccf766fc61dc342182c48a68 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Mon, 19 May 2025 11:51:33 -0700 Subject: [PATCH 10/16] LU-19029 kernel: update RHEL 8.10 [4.18.0-553.52.1.el8_10] Update RHEL 8.10 kernel to 4.18.0-553.52.1.el8_10. Test-Parameters: trivial fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testlist=sanity Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testlist=sanity Test-Parameters: optional fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-part-1 Test-Parameters: optional fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-part-2 Test-Parameters: optional fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-part-3 Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-zfs-part-1 Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-zfs-part-2 Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-zfs-part-3 Change-Id: I0d5a2872050a92e1bf8e8b9438ab2bd1a4a21636 Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59293 Reviewed-by: Oleg Drokin Reviewed-by: Alex Deiter Reviewed-by: Yang Sheng Tested-by: jenkins Tested-by: Maloo --- lustre/ChangeLog | 4 ++-- lustre/kernel_patches/targets/4.18-rhel8.10.target.in | 2 +- lustre/kernel_patches/which_patch | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 615ab64..89d27d2 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -6,7 +6,7 @@ TBD Whamcloud 5.14.0-503.40.1.el9 (RHEL9.5) 5.14.0-427.42.1.el9 (RHEL9.4) 5.14.0-362.24.1.el9 (RHEL9.3) - 4.18.0-553.51.1.el8 (RHEL8.10) + 4.18.0-553.52.1.el8 (RHEL8.10) 4.18.0-513.24.1.el8 (RHEL8.9) 4.18.0-477.27.1.el8 (RHEL8.8) * Other server kernels known to build and work at some point (others may also work): @@ -47,7 +47,7 @@ TBD Whamcloud 5.14.0-503.40.1.el9 (RHEL9.5) 5.14.0-427.42.1.el9 (RHEL9.4) 5.14.0-362.24.1.el9 (RHEL9.3) - 4.18.0-553.51.1.el8 (RHEL8.10) + 4.18.0-553.52.1.el8 (RHEL8.10) 4.18.0-513.24.1.el8 (RHEL8.9) 4.18.0-477.27.1.el8 (RHEL8.8) 5.15.0-88 (Ubuntu 22.04) diff --git a/lustre/kernel_patches/targets/4.18-rhel8.10.target.in b/lustre/kernel_patches/targets/4.18-rhel8.10.target.in index 5216019..9afce66 100644 --- a/lustre/kernel_patches/targets/4.18-rhel8.10.target.in +++ b/lustre/kernel_patches/targets/4.18-rhel8.10.target.in @@ -1,5 +1,5 @@ lnxmaj="4.18.0" -lnxrel="553.51.1.el8_10" +lnxrel="553.52.1.el8_10" KERNEL_SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm SERIES=4.18-rhel8.10.series diff --git a/lustre/kernel_patches/which_patch b/lustre/kernel_patches/which_patch index 68efae3..059aff7 100644 --- a/lustre/kernel_patches/which_patch +++ b/lustre/kernel_patches/which_patch @@ -20,7 +20,7 @@ PATCH SERIES FOR SERVER KERNELS: 4.18-rhel8.7.series 4.18.0-425.10.1.el8 (RHEL 8.7) 4.18-rhel8.8.series 4.18.0-477.27.1.el8 (RHEL 8.8) 4.18-rhel8.9.series 4.18.0-513.24.1.el8 (RHEL 8.9) -4.18-rhel8.10.series 4.18.0-553.51.1.el8 (RHEL 8.10) +4.18-rhel8.10.series 4.18.0-553.52.1.el8 (RHEL 8.10) 5.14-rhel9.1.series 5.14.0-162.23.1.el9 (RHEL 9.1) 5.14-rhel9.2.series 5.14.0-284.30.1.el9 (RHEL 9.2) 5.14-rhel9.3.series 5.14.0-362.24.1.el9 (RHEL 9.3) -- 1.8.3.1 From 5cf0f551ebda15c4253b02dc703aca76121ea7f0 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Wed, 21 May 2025 00:25:03 -0700 Subject: [PATCH 11/16] LU-19035 kernel: update RHEL 8.10 [4.18.0-553.53.1.el8_10] Update RHEL 8.10 kernel to 4.18.0-553.53.1.el8_10. Test-Parameters: trivial fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testlist=sanity Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testlist=sanity Test-Parameters: optional fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-part-1 Test-Parameters: optional fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-part-2 Test-Parameters: optional fstype=ldiskfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-part-3 Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-zfs-part-1 Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-zfs-part-2 Test-Parameters: optional fstype=zfs mdtcount=4 mdscount=2 \ clientdistro=el8.10 serverdistro=el8.10 testgroup=full-dne-zfs-part-3 Change-Id: Ide3ddc9dd8716e24cfb5bbbcba75237ac58041ba Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59341 Reviewed-by: Oleg Drokin Reviewed-by: Alex Deiter Reviewed-by: Yang Sheng Tested-by: jenkins Tested-by: Maloo --- lustre/ChangeLog | 4 ++-- lustre/kernel_patches/targets/4.18-rhel8.10.target.in | 2 +- lustre/kernel_patches/which_patch | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 89d27d2..bd99c28 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -6,7 +6,7 @@ TBD Whamcloud 5.14.0-503.40.1.el9 (RHEL9.5) 5.14.0-427.42.1.el9 (RHEL9.4) 5.14.0-362.24.1.el9 (RHEL9.3) - 4.18.0-553.52.1.el8 (RHEL8.10) + 4.18.0-553.53.1.el8 (RHEL8.10) 4.18.0-513.24.1.el8 (RHEL8.9) 4.18.0-477.27.1.el8 (RHEL8.8) * Other server kernels known to build and work at some point (others may also work): @@ -47,7 +47,7 @@ TBD Whamcloud 5.14.0-503.40.1.el9 (RHEL9.5) 5.14.0-427.42.1.el9 (RHEL9.4) 5.14.0-362.24.1.el9 (RHEL9.3) - 4.18.0-553.52.1.el8 (RHEL8.10) + 4.18.0-553.53.1.el8 (RHEL8.10) 4.18.0-513.24.1.el8 (RHEL8.9) 4.18.0-477.27.1.el8 (RHEL8.8) 5.15.0-88 (Ubuntu 22.04) diff --git a/lustre/kernel_patches/targets/4.18-rhel8.10.target.in b/lustre/kernel_patches/targets/4.18-rhel8.10.target.in index 9afce66..f31cc00 100644 --- a/lustre/kernel_patches/targets/4.18-rhel8.10.target.in +++ b/lustre/kernel_patches/targets/4.18-rhel8.10.target.in @@ -1,5 +1,5 @@ lnxmaj="4.18.0" -lnxrel="553.52.1.el8_10" +lnxrel="553.53.1.el8_10" KERNEL_SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm SERIES=4.18-rhel8.10.series diff --git a/lustre/kernel_patches/which_patch b/lustre/kernel_patches/which_patch index 059aff7..53dbe2a 100644 --- a/lustre/kernel_patches/which_patch +++ b/lustre/kernel_patches/which_patch @@ -20,7 +20,7 @@ PATCH SERIES FOR SERVER KERNELS: 4.18-rhel8.7.series 4.18.0-425.10.1.el8 (RHEL 8.7) 4.18-rhel8.8.series 4.18.0-477.27.1.el8 (RHEL 8.8) 4.18-rhel8.9.series 4.18.0-513.24.1.el8 (RHEL 8.9) -4.18-rhel8.10.series 4.18.0-553.52.1.el8 (RHEL 8.10) +4.18-rhel8.10.series 4.18.0-553.53.1.el8 (RHEL 8.10) 5.14-rhel9.1.series 5.14.0-162.23.1.el9 (RHEL 9.1) 5.14-rhel9.2.series 5.14.0-284.30.1.el9 (RHEL 9.2) 5.14-rhel9.3.series 5.14.0-362.24.1.el9 (RHEL 9.3) -- 1.8.3.1 From 0a0293fbd2b99507bc5b60dd24c81756b2b9436b Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Sat, 24 May 2025 07:56:10 +0700 Subject: [PATCH 12/16] LU-19049 lutf: Debian 13: swig 4.3, python 3.13.3 Update the config/ac_python_devel.m4 to handle distutils being removed. Handle swig 4.3 api change, SWIG_Python_AppendOutput() takes a 3rd argument 'is_null' Signed-off-by: Shaun Tancheff Change-Id: I35968b6928f682f5a70731e316db5e171fad00be Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59401 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Yang Sheng Reviewed-by: Oleg Drokin --- autoMakefile.am | 2 +- config/ac_python_devel.m4 | 489 ++++++++++++++-------- config/lustre-build.m4 | 8 +- lustre/tests/lutf/swig_templates/typemap.template | 4 + 4 files changed, 318 insertions(+), 185 deletions(-) diff --git a/autoMakefile.am b/autoMakefile.am index b707807..6488905 100644 --- a/autoMakefile.am +++ b/autoMakefile.am @@ -615,7 +615,7 @@ dkms-debs: undef.h debs_common ../lustre-*-utils_$${VER}_*.deb \ ../lustre_$${VER}.dsc ../lustre_$${VER}_*.changes \ ../lustre_$${VER}.tar.gz ../lustre-*-modules-dkms_$${VER}_*.deb \ - ../lustre-*-dbgsym_$${VER}_*.ddeb ../lustre_$${VER}_*.buildinfo \ + ../lustre-*-dbgsym_$${VER}_*.*deb ../lustre_$${VER}_*.buildinfo \ debs/ EXTRA_DIST += debian/* diff --git a/config/ac_python_devel.m4 b/config/ac_python_devel.m4 index 4e896de..935056c 100644 --- a/config/ac_python_devel.m4 +++ b/config/ac_python_devel.m4 @@ -4,7 +4,7 @@ # # SYNOPSIS # -# AX_PYTHON_DEVEL([version]) +# AX_PYTHON_DEVEL([version[,optional]]) # # DESCRIPTION # @@ -23,6 +23,11 @@ # version number. Don't use "PYTHON_VERSION" for this: that environment # variable is declared as precious and thus reserved for the end-user. # +# By default this will fail if it does not detect a development version of +# python. If you want it to continue, set optional to true, like +# AX_PYTHON_DEVEL([], [true]). The ax_python_devel_found variable will be +# "no" if it fails. +# # This macro should work for all versions of Python >= 2.1.0. As an end # user, you can disable the check for the python version by setting the # PYTHON_NOVERSIONCHECK environment variable to something else than the @@ -67,276 +72,396 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 21 +#serial 37 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL]) AC_DEFUN([AX_PYTHON_DEVEL],[ + # Get whether it's optional + if test -z "$2"; then + ax_python_devel_optional=false + else + ax_python_devel_optional=$2 + fi + ax_python_devel_found=yes + # # Allow the use of a (user set) custom python version # - #AC_ARG_VAR([PYTHON_VERSION],[The installed Python - # version to use, for example '2.3'. This string - # will be appended to the Python interpreter - # canonical name.]) + AC_ARG_VAR([PYTHON_VERSION],[The installed Python + version to use, for example '2.3'. This string + will be appended to the Python interpreter + canonical name.]) - AC_PATH_PROGS([PYTHON],[python3 python]) - #AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]]) + AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]]) if test -z "$PYTHON"; then - #AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path]) - AC_MSG_WARN([Cannot find python in your system path]) - PYTHON_VERSION_CHECK="no" - AC_SUBST([PYTHON_VERSION_CHECK]) + AC_MSG_WARN([Cannot find python$PYTHON_VERSION in your system path]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up, python development not available]) + fi + ax_python_devel_found=no PYTHON_VERSION="" fi - PYTHON_VERSION_CHECK="yes" - # - # Check for a version of Python >= 2.1.0 - # - AC_MSG_CHECKING([for a version of Python >= '3.6.0']) - ac_supports_python_ver=`$PYTHON -c "import sys; \ + if test $ax_python_devel_found = yes; then + # + # Check for a version of Python >= 2.1.0 + # + AC_MSG_CHECKING([for a version of Python >= '2.1.0']) + ac_supports_python_ver=`$PYTHON -c "import sys; \ ver = sys.version.split ()[[0]]; \ - print (ver >= '3.0')"` - if test "$ac_supports_python_ver" != "True"; then + print (ver >= '2.1.0')"` + if test "$ac_supports_python_ver" != "True"; then if test -z "$PYTHON_NOVERSIONCHECK"; then AC_MSG_RESULT([no]) - PYTHON_VERSION_CHECK="no" - AC_SUBST([PYTHON_VERSION_CHECK]) AC_MSG_WARN([ This version of the AC@&t@_PYTHON_DEVEL macro doesn't work properly with versions of Python before -3.6.0. You may need to re-run configure, setting the +2.1.0. You may need to re-run configure, setting the variables PYTHON_CPPFLAGS, PYTHON_LIBS, PYTHON_SITE_PKG, PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand. Moreover, to disable this check, set PYTHON_NOVERSIONCHECK to something else than an empty string. ]) + if ! $ax_python_devel_optional; then + AC_MSG_FAILURE([Giving up]) + fi + ax_python_devel_found=no + PYTHON_VERSION="" else AC_MSG_RESULT([skip at user request]) fi - else + else AC_MSG_RESULT([yes]) + fi fi - # - # if the macro parameter ``version'' is set, honour it - # - if test -n "$1" ; then + if test $ax_python_devel_found = yes; then + # + # If the macro parameter ``version'' is set, honour it. + # A Python shim class, VPy, is used to implement correct version comparisons via + # string expressions, since e.g. a naive textual ">= 2.7.3" won't work for + # Python 2.7.10 (the ".1" being evaluated as less than ".3"). + # + if test -n "$1"; then AC_MSG_CHECKING([for a version of Python $1]) - ac_supports_python_ver=`$PYTHON -c "import sys; \ - ver = sys.version.split ()[[0]]; \ + cat << EOF > ax_python_devel_vpy.py +class VPy: + def vtup(self, s): + return tuple(map(int, s.strip().replace("rc", ".").split("."))) + def __init__(self): + import sys + self.vpy = tuple(sys.version_info)[[:3]] + def __eq__(self, s): + return self.vpy == self.vtup(s) + def __ne__(self, s): + return self.vpy != self.vtup(s) + def __lt__(self, s): + return self.vpy < self.vtup(s) + def __gt__(self, s): + return self.vpy > self.vtup(s) + def __le__(self, s): + return self.vpy <= self.vtup(s) + def __ge__(self, s): + return self.vpy >= self.vtup(s) +EOF + ac_supports_python_ver=`$PYTHON -c "import ax_python_devel_vpy; \ + ver = ax_python_devel_vpy.VPy(); \ print (ver $1)"` + rm -rf ax_python_devel_vpy*.py* __pycache__/ax_python_devel_vpy*.py* if test "$ac_supports_python_ver" = "True"; then - AC_MSG_RESULT([yes]) + AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) - PYTHON_VERSION_CHECK="no" - AC_SUBST([PYTHON_VERSION_CHECK]) AC_MSG_WARN([this package requires Python $1. If you have it installed, but it isn't the default Python interpreter in your system path, please pass the PYTHON_VERSION variable to configure. See ``configure --help'' for reference. ]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up]) + fi + ax_python_devel_found=no PYTHON_VERSION="" fi + fi fi - # - # Check if you have distutils, else fail - # - if test "yes" = "$PYTHON_VERSION_CHECK" ; then + if test $ax_python_devel_found = yes; then + # + # Check if you have distutils, else fail + # + AC_MSG_CHECKING([for the sysconfig Python package]) + ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1` + if test $? -eq 0; then + AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="import sysconfig" + else + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([for the distutils Python package]) - ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` + ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1` if test $? -eq 0; then AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="from distutils import sysconfig" else - AC_MSG_RESULT([no]) - PYTHON_VERSION_CHECK="no" - AC_SUBST([PYTHON_VERSION_CHECK]) AC_MSG_WARN([cannot import Python module "distutils". - Please check your Python installation. The error was: - $ac_distutils_result]) +Please check your Python installation. The error was: +$ac_sysconfig_result]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up]) + fi + ax_python_devel_found=no PYTHON_VERSION="" fi + fi + fi - # - # Check for Python include path - # - AC_MSG_CHECKING([for Python include path]) - if test -z "$PYTHON_CPPFLAGS"; then - python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc ());"` - plat_python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc (plat_specific=1));"` - if test -n "${python_path}"; then - if test "${plat_python_path}" != "${python_path}"; then - python_path="-I$python_path -I$plat_python_path" - else - python_path="-I$python_path" - fi + if test $ax_python_devel_found = yes; then + # + # Check for Python include path + # + AC_MSG_CHECKING([for Python include path]) + if test -z "$PYTHON_CPPFLAGS"; then + if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then + # sysconfig module has different functions + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('include'));"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('platinclude'));"` + else + # old distutils way + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc ());"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc (plat_specific=1));"` + fi + if test -n "${python_path}"; then + if test "${plat_python_path}" != "${python_path}"; then + python_path="-I$python_path -I$plat_python_path" + else + python_path="-I$python_path" fi - PYTHON_CPPFLAGS=$python_path fi - AC_MSG_RESULT([$PYTHON_CPPFLAGS]) - AC_SUBST([PYTHON_CPPFLAGS]) + PYTHON_CPPFLAGS=$python_path + fi + AC_MSG_RESULT([$PYTHON_CPPFLAGS]) + AC_SUBST([PYTHON_CPPFLAGS]) - # - # Check for Python library path - # - AC_MSG_CHECKING([for Python library path]) - if test -z "$PYTHON_LIBS"; then - # (makes two attempts to ensure we've got a version number - # from the interpreter) - ac_python_version=`cat<]], - [[Py_Initialize();]]) - ],[pythonexists=yes],[pythonexists=no]) - AC_LANG_POP([C]) - # turn back to default flags - CPPFLAGS="$ac_save_CPPFLAGS" - LIBS="$ac_save_LIBS" - LDFLAGS="$ac_save_LDFLAGS" + # + # libraries which must be linked in when embedding + # + AC_MSG_CHECKING(python extra libraries) + if test -z "$PYTHON_EXTRA_LIBS"; then + PYTHON_EXTRA_LIBS=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + conf = sysconfig.get_config_var; \ + print (conf('LIBS') + ' ' + conf('SYSLIBS'))"` + fi + AC_MSG_RESULT([$PYTHON_EXTRA_LIBS]) + AC_SUBST(PYTHON_EXTRA_LIBS) - AC_MSG_RESULT([$pythonexists]) + # + # linking flags needed when embedding + # + AC_MSG_CHECKING(python extra linking flags) + if test -z "$PYTHON_EXTRA_LDFLAGS"; then + PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + conf = sysconfig.get_config_var; \ + print (conf('LINKFORSHARED'))"` + # Hack for macos, it sticks this in here. + PYTHON_EXTRA_LDFLAGS=`echo $PYTHON_EXTRA_LDFLAGS | sed 's/CoreFoundation.*$/CoreFoundation/'` + fi + AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS]) + AC_SUBST(PYTHON_EXTRA_LDFLAGS) - if test ! "x$pythonexists" = "xyes"; then - PYTHON_VERSION_CHECK="no" - AC_SUBST([PYTHON_VERSION_CHECK]) - AC_MSG_WARN([ - Could not link test program to Python. Maybe the main Python library has been - installed in some non-standard library path. If so, pass it to configure, - via the LIBS environment variable. - Example: ./configure LIBS="-L/usr/non-standard-path/python/lib" - ============================================================================ - ERROR! - You probably have to install the development version of the Python package - for your distribution. The exact name of this package varies among them. - ============================================================================ - ]) - PYTHON_VERSION="" - fi + # + # final check to see if everything compiles alright + # + AC_MSG_CHECKING([consistency of all components of python development environment]) + # save current global flags + ac_save_LIBS="$LIBS" + ac_save_LDFLAGS="$LDFLAGS" + ac_save_CPPFLAGS="$CPPFLAGS" + LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS" + LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS" + CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS" + AC_LANG_PUSH([C]) + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[#include ]], + [[Py_Initialize();]]) + ],[pythonexists=yes],[pythonexists=no]) + AC_LANG_POP([C]) + # turn back to default flags + CPPFLAGS="$ac_save_CPPFLAGS" + LIBS="$ac_save_LIBS" + LDFLAGS="$ac_save_LDFLAGS" + + AC_MSG_RESULT([$pythonexists]) + + if test ! "x$pythonexists" = "xyes"; then + AC_MSG_WARN([ + Could not link test program to Python. Maybe the main Python library has been + installed in some non-standard library path. If so, pass it to configure, + via the LIBS environment variable. + Example: ./configure LIBS="-L/usr/non-standard-path/python/lib" + ============================================================================ + ERROR! + You probably have to install the development version of the Python package + for your distribution. The exact name of this package varies among them. + ============================================================================ + ]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up]) + fi + ax_python_devel_found=no + PYTHON_VERSION="" + fi fi + # # all done! # diff --git a/config/lustre-build.m4 b/config/lustre-build.m4 index f5e4366..00eb989 100644 --- a/config/lustre-build.m4 +++ b/config/lustre-build.m4 @@ -251,11 +251,15 @@ AC_MSG_RESULT([$enable_utils]) # Build LNet Unit Test Framework? # AC_DEFUN([LB_CONFIG_LUTF], [ -AX_PYTHON_DEVEL() +# Python development libs are optional, disable LUTF is not available +# if you have python 2 and python3 and python defaults to 2 you can enable +# python3 here by setting PYTHON_VERSION=3 before calling configure, example: +# $ PYTHON_VERSION=3 ./configure [options] +AX_PYTHON_DEVEL([>= '3.6'], [true]) AS_IF([test "x$enable_dist" != xno], [ enable_lutf="yes" ], [ - AS_IF([test "x$PYTHON_VERSION_CHECK" = xno], [ + AS_IF([test "x$ax_python_devel_found" = xno], [ enable_lutf="no" ], [ AX_PKG_SWIG(2.0, [ enable_lutf="yes" ], diff --git a/lustre/tests/lutf/swig_templates/typemap.template b/lustre/tests/lutf/swig_templates/typemap.template index f956dbd..7249dfb 100755 --- a/lustre/tests/lutf/swig_templates/typemap.template +++ b/lustre/tests/lutf/swig_templates/typemap.template @@ -120,7 +120,11 @@ if (*$1 == NULL) goto fail; obj = PyUnicode_FromString(*$1); +#if SWIG_VERSION >= 0x040300 /* 4.3.0 or later */ + $result = SWIG_Python_AppendOutput($result,obj,0); +#else $result = SWIG_Python_AppendOutput($result,obj); +#endif %} /* The malloc'ed pointer is no longer needed, so make sure it is freed. */ -- 1.8.3.1 From 39f1380c20bde95b7582916d48cd00406f7a6a13 Mon Sep 17 00:00:00 2001 From: Emoly Liu Date: Mon, 26 May 2025 16:31:41 +0800 Subject: [PATCH 13/16] LU-18924 hsm: fix the crash cause by huge max_requests Some variables in function mdt_coordinator() should be unsigned type to avoid memory allocation crash caused by huge parameter mdt.*.hsm.max_requests. To avoid such a failure earlier, the sum of MDT max_requests is limited to 1/8 of total memory. If it is bigger than this limit, it will be recalculated by this limit and a useful warning message with memory information and the limit will be printed. Also, sanity-hsm.sh test_40 is modified to verify this patch and stack_trap is added to test_50 and test_100 to restore the default max_requests value correctly. Test-Parameters: testlist=sanity-hsm env=ONLY=40,ONLY_REPEAT=20 Test-Parameters: testlist=sanity-hsm Signed-off-by: Emoly Liu Signed-off-by: Etienne AUJAMES Change-Id: I3f6f9722c2af34a4632dc1620ad191774b8ed403 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/58793 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Guillaume Courrier Reviewed-by: Oleg Drokin --- lustre/mdt/mdt_coordinator.c | 103 +++++++++++++++++++++++++++++++++-------- lustre/tests/sanity-hsm.sh | 17 ++++++- lustre/tests/test-framework.sh | 4 +- 3 files changed, 101 insertions(+), 23 deletions(-) diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c index 52e12a7..ff154b3 100644 --- a/lustre/mdt/mdt_coordinator.c +++ b/lustre/mdt/mdt_coordinator.c @@ -115,8 +115,8 @@ struct hsm_scan_data { u32 hsd_start_cat_idx; u32 hsd_start_rec_idx; int hsd_action_count; - int hsd_request_len; /* array alloc len */ - int hsd_request_count; /* array used count */ + u64 hsd_request_len; /* array alloc len */ + u64 hsd_request_count; /* array used count */ struct hsm_scan_request *hsd_request; }; @@ -632,7 +632,6 @@ static int mdt_coordinator(void *data) struct coordinator *cdt = &mdt->mdt_coordinator; struct hsm_scan_data hsd = { NULL }; time64_t last_housekeeping = 0; - size_t request_sz = 0; int rc; ENTRY; @@ -721,21 +720,20 @@ static int mdt_coordinator(void *data) * we need to allocate a new buffer */ struct hsm_scan_request *tmp = NULL; - int max_requests = cdt->cdt_max_requests; - OBD_ALLOC_LARGE(tmp, max_requests * - sizeof(struct hsm_scan_request)); + u64 max_requests = cdt->cdt_max_requests; + + OBD_ALLOC_PTR_ARRAY_LARGE(tmp, max_requests); if (!tmp) { - CERROR("Failed to resize request buffer, " - "keeping it at %d\n", - hsd.hsd_request_len); + CERROR("%s: error resizing buffer to %llu, keep %llu: rc = %d\n", + mdt_obd_name(mdt), max_requests, + hsd.hsd_request_len, -ENOMEM); } else { if (hsd.hsd_request != NULL) - OBD_FREE_LARGE(hsd.hsd_request, - request_sz); + OBD_FREE_PTR_ARRAY_LARGE( + hsd.hsd_request, + hsd.hsd_request_len); hsd.hsd_request_len = max_requests; - request_sz = hsd.hsd_request_len * - sizeof(struct hsm_scan_request); hsd.hsd_request = tmp; } } @@ -749,7 +747,7 @@ static int mdt_coordinator(void *data) if (rc < 0) goto clean_cb_alloc; - CDEBUG(D_HSM, "found %d requests to send\n", + CDEBUG(D_HSM, "found %llu requests to send\n", hsd.hsd_request_count); if (list_empty(&cdt->cdt_agents)) { @@ -809,7 +807,7 @@ clean_cb_alloc: } if (hsd.hsd_request != NULL) - OBD_FREE_LARGE(hsd.hsd_request, request_sz); + OBD_FREE_PTR_ARRAY_LARGE(hsd.hsd_request, hsd.hsd_request_len); fail_to_start: mdt_hsm_cdt_cleanup(mdt); @@ -1064,6 +1062,65 @@ int hsm_init_ucred(struct lu_ucred *uc) RETURN(0); } +#define HAI_DATA_SIZE_EST (128) +#define HAI_SIZE_EST (sizeof(struct hsm_action_item) + HAI_DATA_SIZE_EST) +#define HSM_ACTIVE_REQ_SIZE_EST (sizeof(struct cdt_agent_req) + \ + sizeof(struct hsm_mem_req_rec) + \ + HAI_DATA_SIZE_EST) +/* mdt_coordinatoor prealloc: max_requests * sizeof(struct hsm_scan_request) */ +#define HSM_SCAN_REQ_SIZE (sizeof(struct hsm_scan_request)) + +/* The memory footprint estimation is the sum of the memory needed to build hal + * requests and the one needed to cache the active requests. + */ +#define HSM_REQ_MEM_FOOTPRINT_EST (HSM_SCAN_REQ_SIZE + HSM_ACTIVE_REQ_SIZE_EST) + +static u64 max_requests_total; +static DEFINE_SPINLOCK(max_requests_total_lock); + +/* Limit total max_requests to 1/8 total memory */ +static int mdt_hsm_max_requests_update(struct coordinator *cdt, u64 new) +{ + u64 max_ram = cfs_totalram_pages() * PAGE_SIZE / 8; + int rc = 0; + + if (new == cdt->cdt_max_requests) + return 0; + + spin_lock(&max_requests_total_lock); + if (new < cdt->cdt_max_requests) { + LASSERT(max_requests_total >= cdt->cdt_max_requests - new); + max_requests_total -= cdt->cdt_max_requests - new; + cdt->cdt_max_requests = new; + } else if (new > cdt->cdt_max_requests) { + u64 max_ram_reqs = max_ram / HSM_REQ_MEM_FOOTPRINT_EST; + u64 to_add = new - cdt->cdt_max_requests; + struct mdt_device *mdt = container_of(cdt, typeof(*mdt), + mdt_coordinator); + + if (to_add > max_ram_reqs || + max_requests_total > max_ram_reqs - to_add) { + rc = -ENOMEM; + LCONSOLE_WARN("%s: No more memory to set HSM max_requests=%llu (max request memory: %lluMB, current total %llu/%llu): rc = %d\n", + mdt_obd_name(mdt), new, max_ram >> 20, + max_requests_total, max_ram_reqs, rc); + to_add = max_ram_reqs - max_requests_total; + } + + max_requests_total += to_add; + cdt->cdt_max_requests += to_add; + + /* no memory available for a new MDT -> allow 1 more request */ + if (!cdt->cdt_max_requests) { + max_requests_total++; + cdt->cdt_max_requests++; + } + } + spin_unlock(&max_requests_total_lock); + + return rc; +} + /** * initialize coordinator struct * \param mdt [IN] device @@ -1124,10 +1181,12 @@ int mdt_hsm_cdt_init(struct mdt_device *mdt) cdt->cdt_default_archive_id = 1; cdt->cdt_grace_delay = 60; cdt->cdt_loop_period = 10; - cdt->cdt_max_requests = 3; cdt->cdt_policy = CDT_DEFAULT_POLICY; cdt->cdt_active_req_timeout = 3600; + cdt->cdt_max_requests = 0; + mdt_hsm_max_requests_update(cdt, 3); + /* by default do not remove archives on last unlink */ cdt->cdt_remove_archive_on_last_unlink = false; cdt->cdt_idle = true; @@ -1152,6 +1211,8 @@ int mdt_hsm_cdt_fini(struct mdt_device *mdt) struct coordinator *cdt = &mdt->mdt_coordinator; ENTRY; + mdt_hsm_max_requests_update(cdt, 0); + lu_context_exit(cdt->cdt_env.le_ses); lu_context_fini(cdt->cdt_env.le_ses); @@ -2285,11 +2346,13 @@ static ssize_t max_requests_store(struct kobject *kobj, struct attribute *attr, rc = kstrtoull(buffer, 0, &val); if (rc) return rc; + if (!val) + return -EINVAL; + rc = mdt_hsm_max_requests_update(cdt, val); + if (rc) + return rc; - if (val != 0) - cdt->cdt_max_requests = val; - - return val ? count : -EINVAL; + return count; } LUSTRE_RW_ATTR(max_requests); diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index 1a7002a..d41e213 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -2981,6 +2981,7 @@ test_40() { local p="" local fid="" local max_requests=$(get_hsm_param max_requests) + local huge_num=$((2**60)) stack_trap "set_hsm_param max_requests $max_requests" EXIT # Increase the number of HSM request that can be performed in @@ -2988,7 +2989,18 @@ test_40() { # also limits the number of requests per seconds that can be # performed, so we pick a decent number. But we also need to keep # that number low because the copytool has no rate limit and will - # fail some requests if if gets too many at once. + # fail some requests if it gets too many at once. + if (( $MDS1_VERSION >= $(version_code v2_16_56-40) )); then + set_hsm_param max_requests $huge_num && + error "set max_requests=$huge_num should failed" || + echo "set max_requests=$huge_num failed with $?" + + tmp_reqs=$(get_hsm_param max_requests) + (( $tmp_reqs < $huge_num && $tmp_reqs > $max_requests )) || + error "Should set max_requests to be a reasonable value" + do_facet mds1 "dmesg | tail -n 5 | grep 'to set HSM max_requests='" || + true + fi set_hsm_param max_requests 300 for i in $(seq 1 $file_count); do @@ -3047,6 +3059,7 @@ test_50() { local dir=$DIR/$tdir local batch_max=50 + stack_trap "set_hsm_param max_requests $(get_hsm_param max_requests)" set_hsm_param max_requests 1000000 mkdir $dir || error "mkdir $dir failed" df -i $MOUNT @@ -3655,6 +3668,8 @@ double_verify_reset_hsm_param() { local val=$(get_hsm_param $p) local save=$val local val2=$(($val * 2)) + + stack_trap "set_hsm_param $p $save" set_hsm_param $p $val2 val=$(get_hsm_param $p) [[ $val == $val2 ]] || diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 34a5d96..04ec0a3 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -12088,8 +12088,8 @@ mdts_set_param() { # if $arg include -P option, run 1 set_param per MDT on the MGS # else, run set_param on each MDT [[ $arg = *"-P"* ]] && facet=mgs - do_facet $facet $LCTL set_param $arg mdt.${MDT[$idx]}.$key$value - [[ $? != 0 ]] && rc=1 + do_facet $facet $LCTL set_param $arg mdt.${MDT[$idx]}.$key$value || + rc=$? done return $rc } -- 1.8.3.1 From 1f8a750e3f591117f6e24952298dbde0cb49ba04 Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Mon, 2 Jun 2025 10:41:59 +0200 Subject: [PATCH 14/16] LU-19079 nodemap: reserve cmds for gss identification Declare 2 new values in enum lcfg_command_type: LCFG_NODEMAP_GSS_IDENTIFY = 0x00ce065 LCFG_NODEMAP_LOOKUP_SHA = 0x00ce066 LCFG_NODEMAP_GSS_IDENTIFY is for a new nodemap property that would be named gss_identification. And LCFG_NODEMAP_LOOKUP_SHA is to be able to lookup a nodemap from the sha256 of its name. Declare a new value in enum nm_flag2_bits: NM_FL2_GSS_IDENTIFY = 0x8 This is to store on disk the value of the future gss_identification property. Reserve sanity-sec test_79 for testing the gss identification feature. Test-Parameters: trivial Signed-off-by: Sebastien Buisson Change-Id: I2e2648f2eeb0956d7cb0793865b3344d1e8ed5a0 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59514 Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- lustre/include/uapi/linux/lustre/lustre_cfg.h | 2 ++ lustre/include/uapi/linux/lustre/lustre_disk.h | 1 + lustre/ptlrpc/wiretest.c | 6 ++++++ lustre/tests/sanity-sec.sh | 6 ++++++ lustre/utils/wirecheck.c | 3 +++ lustre/utils/wiretest.c | 6 ++++++ 6 files changed, 24 insertions(+) diff --git a/lustre/include/uapi/linux/lustre/lustre_cfg.h b/lustre/include/uapi/linux/lustre/lustre_cfg.h index 7802326..e436318 100644 --- a/lustre/include/uapi/linux/lustre/lustre_cfg.h +++ b/lustre/include/uapi/linux/lustre/lustre_cfg.h @@ -129,6 +129,8 @@ enum lcfg_command_type { LCFG_NODEMAP_DENY_MOUNT = 0x00ce060, /**< deny mount */ LCFG_NODEMAP_RAISE_PRIVS = 0x00ce061, /**< sub-nm raise privs */ LCFG_NODEMAP_SET_CAPS = 0x00ce063, /**< user capabilities */ + LCFG_NODEMAP_GSS_IDENTIFY = 0x00ce065, /**< gss identification */ + LCFG_NODEMAP_LOOKUP_SHA = 0x00ce066, /**< lookup nm sha */ }; struct lustre_cfg_bufs { diff --git a/lustre/include/uapi/linux/lustre/lustre_disk.h b/lustre/include/uapi/linux/lustre/lustre_disk.h index 4f111d8..2310291 100644 --- a/lustre/include/uapi/linux/lustre/lustre_disk.h +++ b/lustre/include/uapi/linux/lustre/lustre_disk.h @@ -261,6 +261,7 @@ enum nm_flag2_bits { NM_FL2_READONLY_MOUNT = 0x1, NM_FL2_DENY_MOUNT = 0x2, NM_FL2_FILESET_USE_IAM = 0x4, + NM_FL2_GSS_IDENTIFY = 0x8, }; /* Nodemap records, uses 32 byte record length. diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index 07da401..e45d40b 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -6563,6 +6563,8 @@ void lustre_assert_wire_constants(void) (unsigned int)NM_FL2_DENY_MOUNT); LASSERTF(NM_FL2_FILESET_USE_IAM == 0x00000004UL, "found 0x%.8xUL\n", (unsigned)NM_FL2_FILESET_USE_IAM); + LASSERTF(NM_FL2_GSS_IDENTIFY == 0x00000008UL, "found 0x%.8xUL\n", + (unsigned int)NM_FL2_GSS_IDENTIFY); LASSERTF(NODEMAP_UID == 0, "found %lld\n", (long long)NODEMAP_UID); LASSERTF(NODEMAP_GID == 1, "found %lld\n", @@ -7207,6 +7209,10 @@ void lustre_assert_wire_constants(void) (unsigned)LCFG_NODEMAP_RAISE_PRIVS); LASSERTF(LCFG_NODEMAP_SET_CAPS == 0x00ce063UL, "found 0x%.8xUL\n", (unsigned)LCFG_NODEMAP_SET_CAPS); + LASSERTF(LCFG_NODEMAP_GSS_IDENTIFY == 0x000ce065UL, "found 0x%.8xUL\n", + (unsigned)LCFG_NODEMAP_GSS_IDENTIFY); + LASSERTF(LCFG_NODEMAP_LOOKUP_SHA == 0x000ce066UL, "found 0x%.8xUL\n", + (unsigned)LCFG_NODEMAP_LOOKUP_SHA); #endif /* HAVE_SERVER_SUPPORT */ LASSERTF(PORTALS_CFG_TYPE == 1, "found %lld\n", (long long)PORTALS_CFG_TYPE); diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index be7d518..e023d71 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -8647,6 +8647,12 @@ test_77() { } run_test 77 "root offsetting" +test_79() { + # reserve test_79 + skip "not implemented yet" +} +#run_test 79 "ssk for nodemap identification" + log "cleanup: ======================================================" sec_unsetup() { diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 00e3a30..fe5c87c 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -3102,6 +3102,7 @@ static void check_nodemap_key(void) CHECK_VALUE_X(NM_FL2_READONLY_MOUNT); CHECK_VALUE_X(NM_FL2_DENY_MOUNT); CHECK_VALUE_X(NM_FL2_FILESET_USE_IAM); + CHECK_VALUE_X(NM_FL2_GSS_IDENTIFY); CHECK_VALUE(NODEMAP_UID); CHECK_VALUE(NODEMAP_GID); @@ -3403,6 +3404,8 @@ check_lustre_cfg(void) CHECK_VALUE_X(LCFG_NODEMAP_DENY_MOUNT); CHECK_VALUE_X(LCFG_NODEMAP_RAISE_PRIVS); CHECK_VALUE_X(LCFG_NODEMAP_SET_CAPS); + CHECK_VALUE_X(LCFG_NODEMAP_GSS_IDENTIFY); + CHECK_VALUE_X(LCFG_NODEMAP_LOOKUP_SHA); printf("#endif /* HAVE_SERVER_SUPPORT */\n"); #endif /* !HAVE_NATIVE_LINUX_CLIENT */ CHECK_VALUE(PORTALS_CFG_TYPE); diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index 07679c9..450ce92 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -6608,6 +6608,8 @@ void lustre_assert_wire_constants(void) (unsigned int)NM_FL2_DENY_MOUNT); LASSERTF(NM_FL2_FILESET_USE_IAM == 0x00000004UL, "found 0x%.8xUL\n", (unsigned)NM_FL2_FILESET_USE_IAM); + LASSERTF(NM_FL2_GSS_IDENTIFY == 0x00000008UL, "found 0x%.8xUL\n", + (unsigned int)NM_FL2_GSS_IDENTIFY); LASSERTF(NODEMAP_UID == 0, "found %lld\n", (long long)NODEMAP_UID); LASSERTF(NODEMAP_GID == 1, "found %lld\n", @@ -7252,6 +7254,10 @@ void lustre_assert_wire_constants(void) (unsigned)LCFG_NODEMAP_RAISE_PRIVS); LASSERTF(LCFG_NODEMAP_SET_CAPS == 0x00ce063UL, "found 0x%.8xUL\n", (unsigned)LCFG_NODEMAP_SET_CAPS); + LASSERTF(LCFG_NODEMAP_GSS_IDENTIFY == 0x000ce065UL, "found 0x%.8xUL\n", + (unsigned)LCFG_NODEMAP_GSS_IDENTIFY); + LASSERTF(LCFG_NODEMAP_LOOKUP_SHA == 0x000ce066UL, "found 0x%.8xUL\n", + (unsigned)LCFG_NODEMAP_LOOKUP_SHA); #endif /* HAVE_SERVER_SUPPORT */ LASSERTF(PORTALS_CFG_TYPE == 1, "found %lld\n", (long long)PORTALS_CFG_TYPE); -- 1.8.3.1 From 85f386d63e0a942e9a59d48179949f89470d9158 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Mon, 26 May 2025 22:02:22 -0600 Subject: [PATCH 15/16] LU-19062 llapi: add layout pattern string functions Add llapi_lov_pattern_string() to print arbitrary pattern flags to a string rather than layout2name() which only can print specific hard-coded combinations of patterns. Add llapi_lov_string_pattern() to convert layout pattern names to flags. Add enum lov_pattern that holds LOV_PATTERN constants, and use it. Add description of patterns to lfs-getstripe.1. Restore "-L, --layout" argument listing to lfs-setstripe.1. Fixes: b6deb420a8 ("LU-17370 utils: simplify lfs help text") Signed-off-by: Andreas Dilger Change-Id: Ie21c7c75c685f3a15ac23e83562a12a3ea2540e5 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59530 Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Zhenyu Xu Reviewed-by: Oleg Drokin --- Documentation/man1/lfs-getstripe.1 | 20 +++++- Documentation/man1/lfs-setstripe.1 | 2 +- Documentation/man3/Makefile.am | 2 + Documentation/man3/llapi_lov_pattern_string.3 | 31 ++++++++++ Documentation/man3/llapi_lov_string_pattern.3 | 35 +++++++++++ lustre/include/lustre/lustreapi.h | 23 ++++--- lustre/include/uapi/linux/lustre/lustre_idl.h | 34 +++++------ lustre/include/uapi/linux/lustre/lustre_user.h | 40 ++++++------ lustre/tests/sanity-hsm.sh | 2 +- lustre/tests/sanity.sh | 12 ++-- lustre/utils/lfs.c | 27 +-------- lustre/utils/liblustreapi.c | 35 ++++------- lustre/utils/liblustreapi_hsm.c | 4 +- lustre/utils/liblustreapi_layout.c | 84 +++++++++++++++++++++++++- 14 files changed, 240 insertions(+), 111 deletions(-) create mode 100644 Documentation/man3/llapi_lov_pattern_string.3 create mode 100644 Documentation/man3/llapi_lov_string_pattern.3 diff --git a/Documentation/man1/lfs-getstripe.1 b/Documentation/man1/lfs-getstripe.1 index 0fb2559..8a5770a 100644 --- a/Documentation/man1/lfs-getstripe.1 +++ b/Documentation/man1/lfs-getstripe.1 @@ -197,14 +197,28 @@ Print OST and MDT indexes in hexadecimal rather than decimal. Show only the file layout, which is one of: .RS 1.2i .TP +.B bad +Files that have an invalid layout and are inaccessible. +.TP +.B foreign +Files that have a foreign (non-Lustre/free format) component. +Such files may be inaccessible without external software integration. +.TP +.B hole +Files that are missing a data object, +possibly due to OST failure and layout reconstruction by LFSCK. +.TP +.B mdt +Files that store the first data component on the MDT holding the inode. +.TP +.B overstriped +Files have more than one stripe per OST for improved lock scalability. +.TP .B raid0 Traditional Lustre RAID-0 striping format. .TP .B released HSM-archived files that are not resident in the filesystem. -.TP -.B mdt -Files that have the first data component on an MDT. .RE .TP .BR -m ", " --mdt ", " --mdt-index diff --git a/Documentation/man1/lfs-setstripe.1 b/Documentation/man1/lfs-setstripe.1 index 08c1bc5..37d7fc9 100644 --- a/Documentation/man1/lfs-setstripe.1 +++ b/Documentation/man1/lfs-setstripe.1 @@ -187,7 +187,7 @@ of allows the MDS to choose the starting index and it is strongly recommended, as this allows space and load balancing to be done by the MDS as needed. .TP - +.BR -L ", " --layout " \fILAYOUT_TYPE" The type of layout for that component, which can be one of: .RS .B raid0\fR - stripe the file data across diff --git a/Documentation/man3/Makefile.am b/Documentation/man3/Makefile.am index ebf7cc3..eaeaa27 100644 --- a/Documentation/man3/Makefile.am +++ b/Documentation/man3/Makefile.am @@ -79,6 +79,8 @@ LIBMAN = \ llapi_layout_stripe_count_set.3 \ llapi_layout_stripe_size_get.3 \ llapi_layout_stripe_size_set.3 \ + llapi_lov_pattern_string.3 \ + llapi_lov_string_pattern.3 \ llapi_open_by_fid.3 \ llapi_open_by_fid_at.3 \ llapi_param_get_paths.3 \ diff --git a/Documentation/man3/llapi_lov_pattern_string.3 b/Documentation/man3/llapi_lov_pattern_string.3 new file mode 100644 index 0000000..07ce0f9 --- /dev/null +++ b/Documentation/man3/llapi_lov_pattern_string.3 @@ -0,0 +1,31 @@ +.TH LLAPI_LOV_STRING_PATTERN 3 2025-06-04 "Lustre User API" "Lustre Library Functions" +.SH NAME +llapi_lov_string_pattern \- convert pattern name string to LOV_PATTERN mask +.SH SYNOPSIS +.nf +.B #include +.PP +.BI "int llapi_lov_string_pattern(const char *" string ", enum lov_pattern *" pattern ); +.fi +.SH DESCRIPTION +.B llapi_lov_string_pattern() +converts the comma-separated list of LOV pattern names in +.I string +to a mask of +.B LOV_PATTERN_* +flags returned in +.IR pattern . +.SH RETURN VALUES +.BR llapi_lov_string_pattern( ) +returns 0 on success, or +.B -EINVAL +on failure (in which case, errno is set appropriately). +.SH AVAILABILITY +.BR llapi_lov_string_pattern () +is part of the +.BR lustre (7) +user application interface library since release 2.17.0 +.SH SEE ALSO +.BR llapi_lov_pattern_string (3), +.BR lustre (7), +.BR lustreapi (7) diff --git a/Documentation/man3/llapi_lov_string_pattern.3 b/Documentation/man3/llapi_lov_string_pattern.3 new file mode 100644 index 0000000..8ecd295 --- /dev/null +++ b/Documentation/man3/llapi_lov_string_pattern.3 @@ -0,0 +1,35 @@ +.TH LLAPI_LOV_PATTERN_STRING 3 2025-06-04 "Lustre User API" "Lustre Library Functions" +.SH NAME +llapi_lov_pattern_string \- convert LOV_PATTERN mask to string +.SH SYNOPSIS +.nf +.B #include +.PP +.BI "int llapi_lov_pattern_string(enum lov_pattern " pattern ", char *" buf \ + ", size_t " buflen ");" +.fi +.SH DESCRIPTION +.B llapi_lov_pattern_string() +converts the +.I pattern +mask to a comma-separated string of pattern names +in the +.I buf +buffer, up to +.I buflen +bytes in length. +.SH RETURN VALUES +.B llapi_lov_pattern_string() +returns a pointer to the +.B buf +string on success, or NULL on failure +(currently only in case of buffer overflow). +.SH AVAILABILITY +.BR llapi_lov_pattern_string () +is part of the +.BR lustre (7) +user application interface library since release 2.17.0 +.SH SEE ALSO +.BR llapi_lov_string_pattern (3), +.BR lustre (7), +.BR lustreapi (7) diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 9e876f0..6d7d5ba 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -128,7 +128,7 @@ struct llapi_stripe_param { unsigned long long lsp_stripe_size; char *lsp_pool; int lsp_stripe_offset; - int lsp_stripe_pattern; + enum lov_pattern lsp_stripe_pattern; /* Number of stripes. Size of lsp_osts[] if lsp_specific is true.*/ int lsp_stripe_count; bool lsp_is_specific; @@ -147,7 +147,7 @@ enum { LLAPI_MIGRATION_VERBOSE = 0x0008, }; -__u32 llapi_pattern_to_lov(uint64_t pattern); +enum lov_pattern llapi_pattern_to_lov(uint64_t pattern); int llapi_file_open_param(const char *name, int flags, mode_t mode, const struct llapi_stripe_param *param); @@ -155,16 +155,18 @@ int llapi_file_is_encrypted(int fd); int llapi_file_create_foreign(const char *name, mode_t mode, __u32 type, __u32 flags, char *foreign_lov); int llapi_file_create(const char *name, unsigned long long stripe_size, - int stripe_offset, int stripe_count, int stripe_pattern); + int stripe_offset, int stripe_count, + enum lov_pattern stripe_pattern); int llapi_file_open(const char *name, int flags, int mode, unsigned long long stripe_size, int stripe_offset, - int stripe_count, int stripe_pattern); + int stripe_count, enum lov_pattern stripe_pattern); int llapi_file_create_pool(const char *name, unsigned long long stripe_size, int stripe_offset, int stripe_count, - int stripe_pattern, char *pool_name); + enum lov_pattern stripe_pattern, char *pool_name); int llapi_file_open_pool(const char *name, int flags, int mode, unsigned long long stripe_size, int stripe_offset, - int stripe_count, int stripe_pattern, char *pool_name); + int stripe_count, enum lov_pattern stripe_pattern, + char *pool_name); int llapi_poollist(const char *name); int llapi_get_poolbuf(const char *name, char **buf, char ***poolist, int *poolcount); @@ -492,7 +494,7 @@ int llapi_dir_create(const char *name, mode_t mode, int llapi_dir_create_foreign(const char *name, mode_t mode, __u32 type, __u32 flags, const char *value); int llapi_dir_create_pool(const char *name, int flags, int stripe_offset, - int stripe_count, int stripe_pattern, + int stripe_count, enum lov_pattern stripe_pattern, const char *poolname); int llapi_direntry_remove(char *dname); int llapi_unlink_foreign(char *dname); @@ -713,8 +715,8 @@ int llapi_hsm_action_get_dfid(const struct hsm_copyaction_private *hcp, int llapi_hsm_action_get_fd(const struct hsm_copyaction_private *hcp); int llapi_hsm_import(const char *dst, int archive, const struct stat *st, unsigned long long stripe_size, int stripe_offset, - int stripe_count, int stripe_pattern, char *pool_name, - struct lu_fid *newfid); + int stripe_count, enum lov_pattern stripe_pattern, + char *pool_name, struct lu_fid *newfid); /* HSM user interface */ struct hsm_user_request *llapi_hsm_user_request_alloc(int itemcount, @@ -1209,6 +1211,9 @@ int llapi_layout_flags_set(struct llapi_layout *layout, uint32_t flags); int llapi_layout_flags_get(struct llapi_layout *layout, uint32_t *flags); const char *llapi_layout_flags_string(uint32_t flags); __u16 llapi_layout_string_flags(char *string); +char *llapi_lov_pattern_string(enum lov_pattern pattern, char *buf, + size_t buflen); +int llapi_lov_string_pattern(const char *string, enum lov_pattern *pattern); /** * llapi_layout_mirror_count_get() - Get mirror count from the header of diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index 382aeab..3ef0a83 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -1241,14 +1241,13 @@ struct lov_ost_data_v1 { /* per-stripe data structure (little-endian)*/ #define lov_mds_md lov_mds_md_v1 struct lov_mds_md_v1 { /* LOV EA mds/wire data (little-endian) */ - __u32 lmm_magic; /* magic number = LOV_MAGIC_V1 */ - __u32 lmm_pattern; /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */ - struct ost_id lmm_oi; /* LOV object ID */ - __u32 lmm_stripe_size; /* size of stripe in bytes */ - /* lmm_stripe_count used to be __u32 */ - __u16 lmm_stripe_count; /* num stripes in use for this object */ - __u16 lmm_layout_gen; /* layout generation number */ - struct lov_ost_data_v1 lmm_objects[]; /* per-stripe data */ + __u32 lmm_magic; /* LOV_MAGIC_V1 */ + enum lov_pattern lmm_pattern; /* LOV_PATTERN_RAID0, ... */ + struct ost_id lmm_oi; /* LOV object ID */ + __u32 lmm_stripe_size; /* size of stripe in bytes */ + __u16 lmm_stripe_count; /* OST stripes in layout */ + __u16 lmm_layout_gen; /* layout generation number */ + struct lov_ost_data_v1 lmm_objects[]; /* per-stripe data */ }; #define MAX_MD_SIZE_OLD (sizeof(struct lov_mds_md) + \ @@ -1299,15 +1298,14 @@ struct lov_mds_md_v1 { /* LOV EA mds/wire data (little-endian) */ #define XATTR_JOB_MAX_LEN 13 struct lov_mds_md_v3 { /* LOV EA mds/wire data (little-endian) */ - __u32 lmm_magic; /* magic number = LOV_MAGIC_V3 */ - __u32 lmm_pattern; /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */ - struct ost_id lmm_oi; /* LOV object ID */ - __u32 lmm_stripe_size; /* size of stripe in bytes */ - /* lmm_stripe_count used to be __u32 */ - __u16 lmm_stripe_count; /* num stripes in use for this object */ - __u16 lmm_layout_gen; /* layout generation number */ - char lmm_pool_name[LOV_MAXPOOLNAME + 1]; /* must be 32bit aligned */ - struct lov_ost_data_v1 lmm_objects[]; /* per-stripe data */ + __u32 lmm_magic; /* LOV_MAGIC_V3 */ + enum lov_pattern lmm_pattern; /* LOV_PATTERN_RAID0, ... */ + struct ost_id lmm_oi; /* LOV object ID */ + __u32 lmm_stripe_size; /* size of stripe in bytes */ + __u16 lmm_stripe_count; /* OST stripes in layout */ + __u16 lmm_layout_gen; /* layout generation number */ + char lmm_pool_name[LOV_MAXPOOLNAME + 1]; + struct lov_ost_data_v1 lmm_objects[]; /* per-stripe data */ }; static inline __u32 lov_mds_md_size(__u16 stripes, __u32 lmm_magic) @@ -2518,7 +2516,7 @@ struct lov_desc { __u32 ld_tgt_count; /* how many OBD's */ __u32 ld_active_tgt_count; /* how many active */ __s32 ld_default_stripe_count; /* how many objects are used */ - __u32 ld_pattern; /* default PATTERN_RAID0 */ + enum lov_pattern ld_pattern; /* default LOV_PATTERN_RAID0 */ __u64 ld_default_stripe_size; /* in bytes */ __s64 ld_default_stripe_offset; /* starting OST index */ __u32 ld_padding_0; /* unused */ diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 8215779..17409f2 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -785,29 +785,31 @@ enum ll_file_flags { #define LMV_USER_MAGIC_V0 0x0CD20CD0 /* old default lmv magic*/ #define LMV_USER_MAGIC_SPECIFIC 0x0CD40CD0 -#define LOV_PATTERN_NONE 0x000 -#define LOV_PATTERN_RAID0 0x001 -#define LOV_PATTERN_RAID1 0x002 -#define LOV_PATTERN_MDT 0x100 -#define LOV_PATTERN_OVERSTRIPING 0x200 -#define LOV_PATTERN_FOREIGN 0x400 -#define LOV_PATTERN_COMPRESS 0x800 - -/* combine exclusive patterns as a bad pattern */ -#define LOV_PATTERN_BAD (LOV_PATTERN_RAID1 | LOV_PATTERN_MDT | \ - LOV_PATTERN_FOREIGN) - -#define LOV_PATTERN_F_MASK 0xffff0000 -#define LOV_PATTERN_F_HOLE 0x40000000 /* there is hole in LOV EA */ -#define LOV_PATTERN_F_RELEASED 0x80000000 /* HSM released file */ -#define LOV_PATTERN_DEFAULT 0xffffffff +enum lov_pattern { + LOV_PATTERN_NONE = 0x000, + LOV_PATTERN_RAID0 = 0x001, + LOV_PATTERN_RAID1 = 0x002, + LOV_PATTERN_MDT = 0x100, + LOV_PATTERN_OVERSTRIPING = 0x200, + LOV_PATTERN_FOREIGN = 0x400, + LOV_PATTERN_COMPRESS = 0x800, + + /* combine exclusive patterns as a bad pattern */ + LOV_PATTERN_BAD = (LOV_PATTERN_RAID1 | LOV_PATTERN_MDT | + LOV_PATTERN_FOREIGN), + + LOV_PATTERN_F_MASK = 0xffff0000, + LOV_PATTERN_F_HOLE = 0x40000000, /* hole in LOV EA objects */ + LOV_PATTERN_F_RELEASED = 0x80000000, /* HSM released file */ + LOV_PATTERN_DEFAULT = 0xffffffff +}; #define LOV_OFFSET_DEFAULT ((__u16)-1) #define LMV_OFFSET_DEFAULT ((__u32)-1) -static inline bool lov_pattern_supported(__u32 pattern) +static inline bool lov_pattern_supported(enum lov_pattern pattern) { - __u32 pattern_base = pattern & ~LOV_PATTERN_F_RELEASED; + enum lov_pattern pattern_base = pattern & ~LOV_PATTERN_F_RELEASED; return pattern_base == LOV_PATTERN_RAID0 || pattern_base == (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING) || @@ -818,7 +820,7 @@ static inline bool lov_pattern_supported(__u32 pattern) * having many extra checks on lov_pattern_supported, we have this separate * check for non-released, non-readonly, non-DOM components */ -static inline bool lov_pattern_supported_normal_comp(__u32 pattern) +static inline bool lov_pattern_supported_normal_comp(enum lov_pattern pattern) { return pattern == LOV_PATTERN_RAID0 || pattern == (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING); diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index d41e213..2ec3842 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -1014,7 +1014,7 @@ test_11a() { echo -n "Verifying released pattern: " local PTRN=$($LFS getstripe -L $f) echo $PTRN - [[ $PTRN == released ]] || error "Is not released" + [[ $PTRN =~ released ]] || error "Is not released" local fid=$(path2fid $f) echo "Verifying new fid $fid in archive" diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 40cd85c..60fbb47 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -24045,10 +24045,10 @@ test_228c() { run_test 228c "NOT shrink the last entry in OI index node to recycle idle leaf" test_229() { # LU-2482, LU-3448 - [ $PARALLEL == "yes" ] && skip "skip parallel run" - [ $OSTCOUNT -lt 2 ] && skip_env "needs >= 2 OSTs" - [ $MDS1_VERSION -lt $(version_code 2.4.53) ] && - skip "No HSM $(lustre_build_version $SINGLEMDS) MDS < 2.4.53" + [[ $PARALLEL == "yes" ]] && skip "skip parallel run" + (( $OSTCOUNT >= 2 )) || skip_env "needs >= 2 OSTs" + (( $MDS1_VERSION >= $(version_code 2.4.50.0-53-ga61ff5914b) )) || + skip "need MDS >= 2.4.50.53 for HSM released file" rm -f $DIR/$tfile @@ -24059,11 +24059,11 @@ test_229() { # LU-2482, LU-3448 $LFS getstripe -v $DIR/$tfile local pattern=$($LFS getstripe -L $DIR/$tfile) - [ X"$pattern" = X"released" ] || error "pattern error ($pattern)" + [[ "$pattern" =~ "released" ]] || error "pattern error ($pattern)" local stripe_count=$($LFS getstripe -c $DIR/$tfile) || error "getstripe" - [ $stripe_count -eq 2 ] || error "stripe count not 2 ($stripe_count)" + (( $stripe_count == 2 )) || error "stripe count not 2 ($stripe_count)" stat $DIR/$tfile || error "failed to stat released file" chown $RUNAS_ID $DIR/$tfile || diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index d97df53..b844c15 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -5171,31 +5171,6 @@ static inline int gid2name(char **name, unsigned int id) return 0; } -static int name2layout(__u32 *layout, char *name) -{ - char *ptr, *layout_name; - - *layout = 0; - for (ptr = name; ; ptr = NULL) { - layout_name = strtok(ptr, ","); - if (!layout_name) - break; - if (strcmp(layout_name, "released") == 0) - *layout |= LOV_PATTERN_F_RELEASED; - else if (strcmp(layout_name, "raid0") == 0) - *layout |= LOV_PATTERN_RAID0; - else if (strcmp(layout_name, "mdt") == 0) - *layout |= LOV_PATTERN_MDT; - else if (strcmp(layout_name, "overstriping") == 0) - *layout |= LOV_PATTERN_OVERSTRIPING; - else if (strcmp(layout_name, "foreign") == 0) - *layout |= LOV_PATTERN_FOREIGN; - else - return -1; - } - return 0; -} - static int name2attrs(char *name, __u64 *attrs, __u64 *neg_attrs) { char *ptr, *attr_name = name; @@ -6317,7 +6292,7 @@ static int lfs_find(int argc, char **argv) param.fp_lazy = 1; break; case 'L': - ret = name2layout(¶m.fp_layout, optarg); + ret = llapi_lov_string_pattern(optarg, ¶m.fp_layout); if (ret) goto err; param.fp_exclude_layout = !!neg_opt; diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index e24f1f6..2685944 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -667,7 +667,8 @@ int llapi_file_is_encrypted(int fd) int llapi_file_open_pool(const char *name, int flags, int mode, unsigned long long stripe_size, int stripe_offset, - int stripe_count, int stripe_pattern, char *pool_name) + int stripe_count, enum lov_pattern stripe_pattern, + char *pool_name) { const struct llapi_stripe_param param = { .lsp_stripe_size = stripe_size, @@ -681,7 +682,7 @@ int llapi_file_open_pool(const char *name, int flags, int mode, int llapi_file_open(const char *name, int flags, int mode, unsigned long long stripe_size, int stripe_offset, - int stripe_count, int stripe_pattern) + int stripe_count, enum lov_pattern stripe_pattern) { return llapi_file_open_pool(name, flags, mode, stripe_size, stripe_offset, stripe_count, @@ -790,7 +791,8 @@ out_err: } int llapi_file_create(const char *name, unsigned long long stripe_size, - int stripe_offset, int stripe_count, int stripe_pattern) + int stripe_offset, int stripe_count, + enum lov_pattern stripe_pattern) { int fd; @@ -806,7 +808,7 @@ int llapi_file_create(const char *name, unsigned long long stripe_size, int llapi_file_create_pool(const char *name, unsigned long long stripe_size, int stripe_offset, int stripe_count, - int stripe_pattern, char *pool_name) + enum lov_pattern stripe_pattern, char *pool_name) { int fd; @@ -1184,7 +1186,7 @@ out: } int llapi_dir_create_pool(const char *name, int mode, int stripe_offset, - int stripe_count, int stripe_pattern, + int stripe_count, enum lov_pattern stripe_pattern, const char *pool_name) { const struct llapi_stripe_param param = { @@ -2733,23 +2735,6 @@ int sattr_cache_get_defaults(const char *const fsname, return 0; } -static char *layout2name(__u32 layout_pattern) -{ - if (layout_pattern & LOV_PATTERN_F_RELEASED) - return "released"; - else if (layout_pattern & LOV_PATTERN_FOREIGN) - return "foreign"; - else if (layout_pattern == LOV_PATTERN_MDT) - return "mdt"; - else if (layout_pattern == LOV_PATTERN_RAID0) - return "raid0"; - else if (layout_pattern == - (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING)) - return "raid0,overstriped"; - else - return "unknown"; -} - enum lov_dump_flags { LDF_IS_DIR = 0x0001, LDF_IS_RAW = 0x0002, @@ -2927,13 +2912,17 @@ static void lov_dump_user_lmm_header(struct lov_user_md *lum, char *path, } if ((verbose & VERBOSE_PATTERN)) { + char buf[128]; + llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); if (verbose & ~VERBOSE_PATTERN) llapi_printf(LLAPI_MSG_NORMAL, "%s%spattern: ", space, prefix); if (lov_pattern_supported(lum->lmm_pattern)) llapi_printf(LLAPI_MSG_NORMAL, "%s", - layout2name(lum->lmm_pattern)); + llapi_lov_pattern_string(lum->lmm_pattern, + buf, sizeof(buf)) ?: + "overflow"); else llapi_printf(LLAPI_MSG_NORMAL, "%x", lum->lmm_pattern); separator = (!yaml && is_dir) ? " " : "\n"; diff --git a/lustre/utils/liblustreapi_hsm.c b/lustre/utils/liblustreapi_hsm.c index 3bfbed4..aa9699a 100644 --- a/lustre/utils/liblustreapi_hsm.c +++ b/lustre/utils/liblustreapi_hsm.c @@ -1421,8 +1421,8 @@ int llapi_hsm_action_get_fd(const struct hsm_copyaction_private *hcp) */ int llapi_hsm_import(const char *dst, int archive, const struct stat *st, unsigned long long stripe_size, int stripe_offset, - int stripe_count, int stripe_pattern, char *pool_name, - struct lu_fid *newfid) + int stripe_count, enum lov_pattern stripe_pattern, + char *pool_name, struct lu_fid *newfid) { struct hsm_user_import hui; int fd; diff --git a/lustre/utils/liblustreapi_layout.c b/lustre/utils/liblustreapi_layout.c index 5871354..193cdd9 100644 --- a/lustre/utils/liblustreapi_layout.c +++ b/lustre/utils/liblustreapi_layout.c @@ -727,11 +727,11 @@ out_layout: goto out; } -__u32 llapi_pattern_to_lov(uint64_t pattern) +enum lov_pattern llapi_pattern_to_lov(uint64_t llapi_pattern) { - __u32 lov_pattern; + enum lov_pattern lov_pattern; - switch (pattern) { + switch (llapi_pattern) { case LLAPI_LAYOUT_DEFAULT: lov_pattern = LOV_PATTERN_RAID0; break; @@ -1928,6 +1928,84 @@ __u16 llapi_layout_string_flags(char *string) return 0; } +static struct { + enum lov_pattern llpn_pattern; + const char *llpn_pattern_name; +} lov_pattern_names[] = { + { LOV_PATTERN_RAID0, "raid0" }, + { LOV_PATTERN_RAID1, "raid1" }, + { LOV_PATTERN_MDT, "mdt" }, + { LOV_PATTERN_OVERSTRIPING, "overstriped" }, /* getstripe */ + { LOV_PATTERN_OVERSTRIPING, "overstriping" }, /* setstripe compat */ + { LOV_PATTERN_BAD, "bad" }, + { LOV_PATTERN_FOREIGN, "foreign" }, + { LOV_PATTERN_COMPRESS, "compress" }, + { LOV_PATTERN_F_HOLE, "hole" }, + { LOV_PATTERN_F_RELEASED, "released" }, + { LOV_PATTERN_DEFAULT, "default" }, + { 0, NULL } +}; + +int llapi_lov_string_pattern(const char *string, enum lov_pattern *pattern) +{ + const char *p; + + *pattern = 0; + for (p = string; p != NULL; p = strchr(p, ',')) { + int i; + + while (*p == ',') + p++; + if (*p == '\0') + break; + for (i = 0; lov_pattern_names[i].llpn_pattern != 0; i++) { + int l; + + l = strlen(lov_pattern_names[i].llpn_pattern_name); + if (strncmp(p, lov_pattern_names[i].llpn_pattern_name, + l)) + continue; + *pattern |= lov_pattern_names[i].llpn_pattern; + break; + } + if (lov_pattern_names[i].llpn_pattern == 0) { + errno = EINVAL; + return -errno; + } + } + + return 0; +} + +char *llapi_lov_pattern_string(enum lov_pattern pattern, char *buf, + size_t buflen) +{ + char *p = buf; + int l = 0; + int i; + + p[0] = '\0'; + for (i = 0; lov_pattern_names[i].llpn_pattern && pattern; i++) { + if ((lov_pattern_names[i].llpn_pattern & pattern) == + lov_pattern_names[i].llpn_pattern) { + l += snprintf(p, buflen - l, "%s%s", buf[0] ? "," : "", + lov_pattern_names[i].llpn_pattern_name); + pattern &= ~lov_pattern_names[i].llpn_pattern; + if (l >= buflen) { + errno = EOVERFLOW; + return NULL; + } + p = buf + l; + } + } + + if (pattern) + snprintf(buf, buflen, "%sunknown_%x", buf[0] ? "," : "", + pattern); + + return buf; +} + /** * llapi_layout_mirror_count_is_valid() - Check the validity of mirror count. * @count: Mirror count value to be checked. -- 1.8.3.1 From 82f2bdb17aea86dbef296c60dd4eee691695036e Mon Sep 17 00:00:00 2001 From: Aurelien Degremont Date: Tue, 28 Jan 2025 15:08:07 +0100 Subject: [PATCH 16/16] LU-19098 hsm: don't print progname twice with lhsmtool Since Lustre 2.11, log message from llapi_error() prefixes the message with the current program short name. There is no more a need for lhsmtool_posix log fonctions (CT_xxxx) to also do the same. Remove the duplicate prog name and cmd_name. Test-Parameters: trivial testlist=sanity-hsm Signed-off-by: Aurelien Degremont Change-Id: I4ebdf01cd00d1544678cbad066e2c3a79ecfda38 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59680 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Arshad Hussain Reviewed-by: Oleg Drokin --- lustre/utils/lhsmtool_posix.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lustre/utils/lhsmtool_posix.c b/lustre/utils/lhsmtool_posix.c index ae6e608..7e02586 100644 --- a/lustre/utils/lhsmtool_posix.c +++ b/lustre/utils/lhsmtool_posix.c @@ -169,7 +169,6 @@ static int arc_fd = -1; static int err_major; static int err_minor; -static char cmd_name[PATH_MAX]; static char fs_name[MAX_OBD_NAME + 1]; static int pid_file_fd = -1; @@ -186,23 +185,23 @@ static inline double ct_now(void) #define CT_ERROR(_rc, _format, ...) \ llapi_error(LLAPI_MSG_ERROR, _rc, \ - "%f %s[%ld]: "_format, \ - ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__) + "%f [%ld]: "_format, \ + ct_now(), syscall(SYS_gettid), ## __VA_ARGS__) #define CT_DEBUG(_format, ...) \ llapi_error(LLAPI_MSG_DEBUG | LLAPI_MSG_NO_ERRNO, 0, \ - "%f %s[%ld]: "_format, \ - ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__) + "%f [%ld]: "_format, \ + ct_now(), syscall(SYS_gettid), ## __VA_ARGS__) #define CT_WARN(_format, ...) \ llapi_error(LLAPI_MSG_WARN | LLAPI_MSG_NO_ERRNO, 0, \ - "%f %s[%ld]: "_format, \ - ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__) + "%f [%ld]: "_format, \ + ct_now(), syscall(SYS_gettid), ## __VA_ARGS__) #define CT_TRACE(_format, ...) \ llapi_error(LLAPI_MSG_INFO | LLAPI_MSG_NO_ERRNO, 0, \ - "%f %s[%ld]: "_format, \ - ct_now(), cmd_name, syscall(SYS_gettid), ## __VA_ARGS__) + "%f [%ld]: "_format, \ + ct_now(), syscall(SYS_gettid), ## __VA_ARGS__) static void usage(const char *name, int rc) { @@ -257,7 +256,7 @@ static void usage(const char *name, int rc) " -u, --update-interval Interval between progress reports sent\n" " to Coordinator\n" " -v, --verbose Produce more verbose output\n", - cmd_name, cmd_name, cmd_name, cmd_name, cmd_name, cmd_name); + name, name, name, name, name, name); exit(rc); } @@ -410,7 +409,7 @@ repeat: opt.o_event_fifo = optarg; break; case 'h': - usage(argv[0], 0); + usage(basename(argv[0]), 0); case 'i': opt.o_action = CA_IMPORT; break; @@ -2446,10 +2445,10 @@ int main(int argc, char **argv) { int rc; - snprintf(cmd_name, sizeof(cmd_name), "%s", basename(argv[0])); rc = ct_parseopts(argc, argv); if (rc < 0) { - CT_WARN("try '%s --help' for more information", cmd_name); + CT_WARN("try '%s --help' for more information", + basename(argv[0])); return -rc; } -- 1.8.3.1