From b21345ead0e86a30cc4cea91ff95a3b96a0304aa Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Mon, 7 Aug 2023 17:01:02 +0300 Subject: [PATCH] EX-8041 utils: lpurge to use 2nd-stripe objects in some cases lpurge must be considering objects which are not first stripe (another lpurge can be waiting for thresholds, etc). Test-Parameters: mdscount=2 mdtcount=4 trivial testlist=hot-pools Signed-off-by: Alex Zhuravlev Change-Id: Iccc0b6af6f283287b41d6f16c875c56f83413e2e Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51884 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/src/lpurge.c | 65 +++++++++++++++++++++++++---------------------- lustre/tests/hot-pools.sh | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 30 deletions(-) diff --git a/lipe/src/lpurge.c b/lipe/src/lpurge.c index cd86269..c6deae1 100644 --- a/lipe/src/lpurge.c +++ b/lipe/src/lpurge.c @@ -646,15 +646,6 @@ static int lpurge_check_ost_object(struct lpurge_slot *ls, return 0; } - /* to avoid N OSTs to 1 MDT scalability issue we only consider - * objects which store 1st stripe - */ - if (attrs->loa_filter_fid.ff_parent.f_ver != 0) { - ls->ls_notfirst_objs++; - ls->ls_notfirst_used_kb += loa_used_kb(attrs); - return 0; - } - /* if the object has got ost_layout structure which encodes * whether the object has a mirror, then we can skip objects * with mirror_id=0 (no mirror) @@ -789,6 +780,14 @@ static int lpurge_lipe_callback(struct lipe_instance *instance, lo->lo_mirror_id = id; } + if (!is_mdt && attrs->loa_filter_fid.ff_parent.f_ver != 0) { + /* + * to avoid N OSTs to 1 MDT scalability issue we prefer + * objects which store 1st stripe + */ + ls->ls_notfirst_objs++; + ls->ls_notfirst_used_kb += loa_used_kb(attrs); + } lipe_list_add(&lo->lo_list, &ls->ls_obj_list); ls->ls_stored++; @@ -886,7 +885,7 @@ static bool last_non_stale_mirror(__u16 mirror_id, struct llapi_layout *layout) } static int -lpurge_mirror_delete(const struct lu_fid *fid, unsigned int mirror_id) +lpurge_mirror_delete(const struct lu_fid *_fid, unsigned int mirror_id) { char fid_buf[FID_LEN + 1]; char vname_buf[PATH_MAX]; @@ -894,6 +893,7 @@ lpurge_mirror_delete(const struct lu_fid *fid, unsigned int mirror_id) struct ll_ioc_lease *lil = NULL; struct llapi_layout *layout = NULL; int mdt_index = -1; + struct lu_fid fid; int fd = -1; int vfd = -1; int rc; @@ -904,13 +904,15 @@ lpurge_mirror_delete(const struct lu_fid *fid, unsigned int mirror_id) * after we open the file. */ memset(lov_xattr_buf, 0, sizeof(lov_xattr_buf)); - snprintf(lov_xattr_buf, sizeof(lov_xattr_buf), DFID, PFID(fid)); + fid = *_fid; + fid.f_ver = 0; + snprintf(lov_xattr_buf, sizeof(lov_xattr_buf), DFID, PFID(&fid)); rc = ioctl(open_by_fid_fd, IOC_MDC_GETFILESTRIPE, lov_xattr_buf); if (rc < 0) { rc = -errno; LX_DEBUG("cannot IOC_MDC_GETFILESTRIPE "DFID", rc = %d\n", - PFID(fid), rc); + PFID(&fid), rc); goto out; } @@ -932,13 +934,13 @@ lpurge_mirror_delete(const struct lu_fid *fid, unsigned int mirror_id) * lfs mirror split -d --mirror-id mirror_id $MOUNTPOINT/.lustre/fid/FID */ - snprintf(fid_buf, sizeof(fid_buf), DFID, PFID(fid)); + snprintf(fid_buf, sizeof(fid_buf), DFID, PFID(&fid)); fd = openat(open_by_fid_fd, fid_buf, O_RDWR); if (fd < 0) { rc = -errno; LX_DEBUG("cannot open "DFID" for split: %s\n", - PFID(fid), strerror(errno)); + PFID(&fid), strerror(errno)); goto out; } @@ -990,10 +992,10 @@ lpurge_mirror_delete(const struct lu_fid *fid, unsigned int mirror_id) out: if (rc < 0) LX_DEBUG("cannot delete mirror %u of "DFID": rc = %d\n", - mirror_id, PFID(fid), rc); + mirror_id, PFID(&fid), rc); else LX_DEBUG("deleted mirror %u of "DFID"\n", - mirror_id, PFID(fid)); + mirror_id, PFID(&fid)); llapi_layout_free(layout); @@ -1126,10 +1128,10 @@ static void lpurge_work_submit(struct lpurge_object *lo) static void lpurge_purge_slot(struct lpurge_slot *ls, long long target_kb) { - struct lpurge_object *lo; + struct lpurge_object *lo, *t; unsigned long long queued_kb, prev_used_kb, used_kb; long long purged_kb; - int i, rc; + int i, rc, priority; again: LX_DEBUG("releasing upto %llu KB: expect %lu KB from %lu objects\n", @@ -1141,21 +1143,24 @@ again: pthread_mutex_lock(&lpurge_work_lock); assert(lpurge_work_should_run); - while (!lipe_list_empty(&ls->ls_obj_list)) { - lo = lipe_list_entry(ls->ls_obj_list.next, struct lpurge_object, - lo_list); + /* first of all, try to release space claimed by first-stripe objects */ + for (priority = 0; priority < 2 && queued_kb < target_kb; priority++) { + lipe_list_for_each_entry_safe(lo, t, &ls->ls_obj_list, lo_list) { + + if ((!!lo->lo_fid.f_ver) != priority) + continue; - queued_kb += lo->lo_used_kb; - ls->ls_used_kb -= lo->lo_used_kb; - ls->ls_found--; - ls->ls_stored--; + queued_kb += lo->lo_used_kb; + ls->ls_used_kb -= lo->lo_used_kb; + ls->ls_found--; + ls->ls_stored--; - lipe_list_move_tail(&lo->lo_list, &lpurge_work_list); - stats.s_queued++; + lipe_list_move_tail(&lo->lo_list, &lpurge_work_list); + stats.s_queued++; - /* if current collection of objects may free target space, then stop */ - if (queued_kb >= target_kb) - break; + if (queued_kb >= target_kb) + break; + } } pthread_cond_broadcast(&lpurge_work_cond); diff --git a/lustre/tests/hot-pools.sh b/lustre/tests/hot-pools.sh index a2946a7..252f686 100755 --- a/lustre/tests/hot-pools.sh +++ b/lustre/tests/hot-pools.sh @@ -2346,6 +2346,55 @@ test_74() { } run_test 74 "ofd keepalive message" +test_76() { + local td=$DIR/$tdir + local tf=$td/$tfile + local ss=$((5*1024*1024)) + local fnr=5 + + init_hot_pools_env + + start_lamigo_service + check_lamigo_is_started || error "failed to start lamigo" + + mkdir $td || echo "can't create $td" + $LFS setstripe -p $LAMIGO_SRC -S $ss -o 0,1 $td || error "$LFS setstripe $td failed" + $LFS getstripe $td + + for ((i=0; i<$fnr; i++)); do + dd if=/dev/zero of=$tf-$i bs=$ss seek=1 count=1 || + error $"can't create $tf-$i" + du -hs $tf-$i + done + cancel_lru_locks osc + $LFS df -h + $LFS getstripe $tf-* | grep lcm_mirror_count + sleep $((LAMIGO_AGE * 2)) + verify_one_lamigo_param 0 replicated $fnr + + echo "before starting lpurge" + $LFS df | grep OST0001 + LPURGE_FREELO=99 LPURGE_FREEHI=100 start_lpurge_service + check_lpurge_is_started || error "failed to start lpurge" + stack_trap stop_lpurge_service + + local mcnt + for ((i=0; i < 20; i++)); do + mcnt=$($LFS getstripe $tf-* | grep lcm_mirror_count | + awk '{print $2}'|sort -u ) + [[ $mcnt == "1" ]] && break + sleep 2 + done + echo "with lpurge in action" + $LFS df | grep OST0001 + [[ $mcnt == "1" ]] || { + $LFS getstripe -v $tf-* + $LFS df + error "2nd stripes weren't deleted" + } +} +run_test 76 "lpurge to use 2nd-stripe objects" + complete_test $SECONDS check_and_cleanup_lustre exit_status -- 1.8.3.1