From 9ee5113beb77133745cff7000cb18df783a77196 Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Tue, 2 Aug 2022 15:41:52 +0300 Subject: [PATCH] LU-15938 llog: more checks in llog_reader Add more correctness checks and reports in llog_reader: - better report wrong record length and chunk skipping case - add tail check: tail id and len should be the same as in head - better report for gap in record indeces - test case with two corruption types: 1) llog has bits set in bitmap beyond file end 2) corruption in the middle Lustre-change: https://review.whamcloud.com/48112 Lustre-commit: 386ffcdbb4c9b89f798de4c83a51a3f020542c8b Signed-off-by: Mikhail Pershin Change-Id: I0c2af6ae2592c94e14e90ead12e28104409313b2 Reviewed-by: Andreas Dilger Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49214 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alex Zhuravlev --- lustre/tests/sanity.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++++ lustre/utils/llog_reader.c | 51 ++++++++++++++++++++++++++---------- 2 files changed, 101 insertions(+), 14 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index b9ee445..54ffa9d 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -8230,6 +8230,70 @@ test_60i() { } run_test 60i "llog: new record vs reader race" +test_60j() { + (( $MDS1_VERSION >= $(version_code 2.15.50) )) || + skip "need MDS version at least 2.15.50" + [[ $PARALLEL != "yes" ]] || skip "skip parallel run" + remote_mds_nodsh && skip "remote MDS with nodsh" + [[ "$mds1_FSTYPE" == "ldiskfs" ]] || skip "ldiskfs only test" + + changelog_users $SINGLEMDS | grep "^cl" && + skip "active changelog user" + + local llog_reader=$(do_facet $SINGLEMDS "which llog_reader 2> /dev/null") + + [[ -z $(do_facet $SINGLEMDS ls -d $llog_reader 2> /dev/null) ]] && + skip_env "missing llog_reader" + + mkdir_on_mdt0 $DIR/$tdir + + local f=$DIR/$tdir/$tfile + local mdt_dev + local tmpfile + local plain + + changelog_register || error "cannot register changelog user" + + # set changelog_mask to ALL + changelog_chmask "ALL" + changelog_clear + + createmany -o ${f}- 100 || error "createmany failed as $RUNAS_ID" + unlinkmany ${f}- 100 || error "unlinkmany failed" + + tmpfile="$(mktemp --tmpdir -u $tfile.XXXXXX)" + mdt_dev=$(facet_device $SINGLEMDS) + + do_facet $SINGLEMDS sync + plain=$(do_facet $SINGLEMDS "$DEBUGFS -c -R 'dump changelog_catalog \ + $tmpfile' $mdt_dev; $llog_reader $tmpfile" | + awk '{match($0,"path=([^ ]+)",a)}END{print a[1]}') + + stack_trap "do_facet $SINGLEMDS rm -f $tmpfile" + + # if $tmpfile is not on EXT3 filesystem for some reason + [[ ${plain:0:1} == 'O' ]] || + skip "path $plain is not in 'O/1/d/' format" + + size=$(do_facet $SINGLEMDS "$DEBUGFS -c -R 'dump $plain $tmpfile' \ + $mdt_dev; stat -c %s $tmpfile") + echo "Truncate llog from $size to $((size - size % 8192))" + size=$((size - size % 8192)) + do_facet $SINGLEMDS $TRUNCATE $tmpfile $size + errs=$(do_facet $SINGLEMDS "$llog_reader $tmpfile" | + grep -c 'in bitmap only') + (( $errs > 0 )) || error "llog_reader didn't find lost records" + + size=$((size - 9000)) + echo "Corrupt llog in the middle at $size" + do_facet $SINGLEMDS dd if=/dev/urandom of=$tmpfile bs=1 seek=$size \ + count=333 conv=notrunc + errs=$(do_facet $SINGLEMDS "$llog_reader $tmpfile" | + grep -c 'next chunk') + (( $errs > 0 )) || error "llog_reader didn't skip bad chunk" +} +run_test 60j "llog_reader reports corruptions" + test_61a() { [ $PARALLEL == "yes" ] && skip "skip parallel run" diff --git a/lustre/utils/llog_reader.c b/lustre/utils/llog_reader.c index 4e5ef64..146cd9d9 100644 --- a/lustre/utils/llog_reader.c +++ b/lustre/utils/llog_reader.c @@ -308,7 +308,8 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog, last_idx = 0; while (ptr < (file_buf + file_size)) { struct llog_rec_hdr *cur_rec; - int idx; + struct llog_rec_tail *cur_tail; + unsigned int idx, len, tail_idx, tail_len; unsigned long offset; offset = (unsigned long)ptr - (unsigned long)file_buf; @@ -320,31 +321,53 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog, } cur_rec = (struct llog_rec_hdr *)ptr; idx = __le32_to_cpu(cur_rec->lrh_index); - if (cur_rec->lrh_len == 0 || - cur_rec->lrh_len > (*llog)->llh_hdr.lrh_len) { - cur_rec->lrh_len = (*llog)->llh_hdr.lrh_len - - offset % (*llog)->llh_hdr.lrh_len; - printf("off %lu skip %u to next chunk.\n", offset, - cur_rec->lrh_len); - } else if (ext2_test_bit(idx, LLOG_HDR_BITMAP(*llog))) { + len = __le32_to_cpu(cur_rec->lrh_len); + + if (len == 0 || len + offset % (*llog)->llh_hdr.lrh_len > + (*llog)->llh_hdr.lrh_len) { + printf("error: rec #%d type=%x has wrong len=%u @%lu\n", + idx, cur_rec->lrh_type, len, offset); + errors++; + len = (*llog)->llh_hdr.lrh_len - + offset % (*llog)->llh_hdr.lrh_len; + printf("skip %u bytes to the next chunk at off %lu.\n", + len, offset + len); + ptr += len; + continue; + } + + if (ext2_test_bit(idx, LLOG_HDR_BITMAP(*llog))) { printf("rec #%d type=%x len=%u offset %lu\n", idx, - cur_rec->lrh_type, cur_rec->lrh_len, offset); + cur_rec->lrh_type, len, offset); recs_pr[i] = cur_rec; i++; } else { cur_rec->lrh_id = CANCELLED; if (cur_rec->lrh_type == LLOG_PAD_MAGIC && - ((offset + cur_rec->lrh_len) & 0x7) != 0) { + ((offset + len) & 0x7) != 0) { printf("error: rec #%d wrong padding len=%u offset %lu to 0x%lx\n", - idx, cur_rec->lrh_len, offset, - offset + cur_rec->lrh_len); + idx, len, offset, offset + len); errors++; } /* The header counts only set records */ } + cur_tail = (struct llog_rec_tail *)(ptr + len - sizeof(*cur_tail)); + tail_idx = __le32_to_cpu(cur_tail->lrt_index); + tail_len = __le32_to_cpu(cur_tail->lrt_len); + if (idx != tail_idx || len != tail_len) { + printf("error: rec #%d len=%u has tail #%d len=%u%s\n", + idx, len, tail_idx, tail_len, + cur_rec->lrh_id == CANCELLED ? " (unset)" : ""); + errors++; + } + + if (idx > last_idx + 1) + printf("error: rec #%d len=%u has gap in %d recs\n", + idx, len, idx - last_idx); + while (++last_idx < idx) { - printf("error: rec #%d is missing%s set in bitmap\n", + printf("error: -> rec #%d is lost%s set in bitmap\n", last_idx, ext2_test_bit(last_idx, LLOG_HDR_BITMAP(*llog)) ? " but" : ", not"); @@ -358,7 +381,7 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog, last_idx = idx; } - ptr += __le32_to_cpu(cur_rec->lrh_len); + ptr += len; if ((ptr - file_buf) > file_size) { printf("error: rec #%d is trimmed by EOF, offset %lu\n", idx, offset); -- 1.8.3.1