From d582837043897658b4a1c5bbead0623a881fb076 Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Tue, 5 Mar 2024 10:19:25 +0100 Subject: [PATCH] EX-7729 osc: Add counters for compressed data This patch is the first of two patches that add counters to track client/server-side data compression statistics. This patch add new compr_stats file in osc.*.compr_stats. From added counters: 1. Size of compressed/uncompressed chunks written/read by client to compressed files, in chunk/bytes; 2. Compressed page counter; Test-Parameters: trivial testlist=sanity-compr Signed-off-by: Vitaliy Kuznetsov Change-Id: I091153480e53309c641d39f271bef536296dc09e Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53737 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Artem Blagodarenko Reviewed-by: Andreas Dilger --- lustre/include/obd.h | 13 +++++++ lustre/ldlm/ldlm_lib.c | 12 ++++++ lustre/osc/lproc_osc.c | 59 ++++++++++++++++++++++++++++++ lustre/osc/osc_compress.c | 71 ++++++++++++++++++++++++++++++------ lustre/tests/sanity-compr.sh | 87 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 231 insertions(+), 11 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 9b80c01..d9c4dac 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -236,6 +236,19 @@ struct client_obd { /* grant consumed for dirty pages */ unsigned long cl_dirty_grant; + spinlock_t cl_compr_stats_lock; + __u64 cl_w_bytes_raw; /* original data written */ + __u64 cl_w_bytes_compr; /* compressed data size */ + __u64 cl_w_bytes_incompr; /* data not compressed */ + __u64 cl_w_chunks_compr; /* chunks could compress */ + __u64 cl_w_chunks_nocompr;/* chunks not compressed */ + __u64 cl_w_pages_compr; /* pages could compress */ + __u64 cl_w_pages_uncompr; /* pages not compressed */ + __u64 cl_r_bytes_raw; /* uncompressed data size */ + __u64 cl_r_bytes_compr; /* compressed data size */ + __u64 cl_r_chunks_compr; /* chunks were compressed */ + __u64 cl_r_pages_compr; /* pages were compressed */ + /* since we allocate grant by blocks, we don't know how many grant will * be used to add a page into cache. As a solution, we reserve maximum * grant before trying to dirty a page and unreserve the rest. diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 027e7bf..aad14f9 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -435,8 +435,20 @@ int client_obd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) INIT_LIST_HEAD(&cli->cl_loi_write_list); INIT_LIST_HEAD(&cli->cl_loi_read_list); spin_lock_init(&cli->cl_loi_list_lock); + spin_lock_init(&cli->cl_compr_stats_lock); atomic_set(&cli->cl_pending_w_pages, 0); atomic_set(&cli->cl_pending_r_pages, 0); + cli->cl_w_chunks_compr = 0; + cli->cl_w_chunks_nocompr = 0; + cli->cl_w_bytes_compr = 0; + cli->cl_w_bytes_raw = 0; + cli->cl_w_bytes_incompr = 0; + cli->cl_w_pages_compr = 0; + cli->cl_w_pages_uncompr = 0; + cli->cl_r_bytes_compr = 0; + cli->cl_r_bytes_raw = 0; + cli->cl_r_pages_compr = 0; + cli->cl_r_chunks_compr = 0; cli->cl_r_in_flight = 0; cli->cl_w_in_flight = 0; diff --git a/lustre/osc/lproc_osc.c b/lustre/osc/lproc_osc.c index a912b30..3e011b2 100644 --- a/lustre/osc/lproc_osc.c +++ b/lustre/osc/lproc_osc.c @@ -766,6 +766,63 @@ out: } LPROC_SEQ_FOPS(osc_compress_type_best); +static int osc_stats_compr_seq_show(struct seq_file *seq, void *v) +{ + struct obd_device *obd = seq->private; + struct client_obd *cli = &obd->u.cli; + + seq_printf(seq, "write_pages_compr: %llu\n", + cli->cl_w_pages_compr); + seq_printf(seq, "write_pages_uncompr: %llu\n", + cli->cl_w_pages_uncompr); + seq_printf(seq, "write_chunks_compr: %llu\n", + cli->cl_w_chunks_compr); + seq_printf(seq, "write_chunks_no_compr: %llu\n", + cli->cl_w_chunks_nocompr); + seq_printf(seq, "write_bytes_compr: %llu\n", + cli->cl_w_bytes_compr); + seq_printf(seq, "write_bytes_raw: %llu\n", + cli->cl_w_bytes_raw); + seq_printf(seq, "write_bytes_incompr: %llu\n", + cli->cl_w_bytes_incompr); + seq_printf(seq, "read_pages_compr: %llu\n", + cli->cl_r_pages_compr); + seq_printf(seq, "read_chunks_compr: %llu\n", + cli->cl_r_chunks_compr); + seq_printf(seq, "read_bytes_compr: %llu\n", + cli->cl_r_bytes_compr); + seq_printf(seq, "read_bytes_raw: %llu\n", + cli->cl_r_bytes_raw); + + return 0; +} + +static ssize_t osc_stats_compr_seq_write(struct file *file, + const char __user *buf, + size_t len, loff_t *off) +{ + struct seq_file *seq = file->private_data; + struct obd_device *obd = seq->private; + struct client_obd *cli = &obd->u.cli; + + spin_lock(&cli->cl_compr_stats_lock); + cli->cl_w_pages_compr = 0; + cli->cl_w_pages_uncompr = 0; + cli->cl_w_bytes_compr = 0; + cli->cl_w_bytes_raw = 0; + cli->cl_w_chunks_compr = 0; + cli->cl_w_chunks_nocompr = 0; + cli->cl_w_bytes_incompr = 0; + cli->cl_r_pages_compr = 0; + cli->cl_r_bytes_compr = 0; + cli->cl_r_bytes_raw = 0; + cli->cl_r_chunks_compr = 0; + spin_unlock(&cli->cl_compr_stats_lock); + + return len; +} +LPROC_SEQ_FOPS(osc_stats_compr); + LPROC_SEQ_FOPS_RO_TYPE(osc, connect_flags); LPROC_SEQ_FOPS_RO_TYPE(osc, server_uuid); LPROC_SEQ_FOPS_RO_TYPE(osc, timeouts); @@ -793,6 +850,8 @@ struct lprocfs_vars lprocfs_osc_obd_vars[] = { .fops = &osc_import_fops }, { .name = "state", .fops = &osc_state_fops }, + { .name = "stats_compr", + .fops = &osc_stats_compr_fops }, { .name = "pinger_recov", .fops = &osc_pinger_recov_fops }, { .name = "unstable_stats", diff --git a/lustre/osc/osc_compress.c b/lustre/osc/osc_compress.c index c39c997..6e4be50 100644 --- a/lustre/osc/osc_compress.c +++ b/lustre/osc/osc_compress.c @@ -114,9 +114,13 @@ int compress_request(struct client_obd *cli, struct obdo *oa, struct cl_page *clpage; struct crypto_comp *cc; enum ll_compr_type type; - unsigned int src_size; - unsigned int dst_size; - int chunk_count = 0; + unsigned int total_src_size = 0; + unsigned int total_dst_size = 0; + unsigned int total_uncompr_size = 0; + int compr_chunk_count = 0; + int chunks_no_compr = 0; + int compr_pages_count = 0; + int uncompr_pages_count = 0; int pages_per_chunk; int dest_buf_bits; int src_buf_bits; @@ -164,6 +168,8 @@ int compress_request(struct client_obd *cli, struct obdo *oa, bool chunk_unmergeable = false; bool compress_this = false; bool compressed = false; + unsigned int src_size; + unsigned int dst_size; __u64 chunk_len_bytes; int chunk_len = 1; int chunk_start; @@ -259,7 +265,7 @@ int compress_request(struct client_obd *cli, struct obdo *oa, */ if (!compressed) { sptlrpc_pool_put_pages(&dst, dest_buf_bits); - + chunks_no_compr++; GOTO(skip, compressed); } @@ -282,6 +288,12 @@ int compress_request(struct client_obd *cli, struct obdo *oa, cpga_i += ((dst_size - 1) >> PAGE_SHIFT) + 1; /* and move pga_i along past the end of this chunk */ pga_i += chunk_len; + compr_chunk_count++; + compr_pages_count += ((src_size - 1) >> PAGE_SHIFT) + 1; + total_dst_size += dst_size; + total_src_size += src_size; + } else { + chunks_no_compr++; } skip: /* if we didn't do compression here, so point this page in the @@ -298,13 +310,27 @@ skip: *cpg = *pg; pga_i++; cpga_i++; + uncompr_pages_count++; + total_uncompr_size += pg->count; + total_dst_size += pg->count; + total_src_size += pg->count; } } *page_count = cpga_i; + spin_lock(&cli->cl_compr_stats_lock); + cli->cl_w_pages_compr += compr_pages_count; + cli->cl_w_pages_uncompr += uncompr_pages_count; + cli->cl_w_bytes_compr += total_dst_size; + cli->cl_w_bytes_raw += total_src_size; + cli->cl_w_bytes_incompr += total_uncompr_size; + cli->cl_w_chunks_compr += compr_chunk_count; + cli->cl_w_chunks_nocompr += chunks_no_compr; + spin_unlock(&cli->cl_compr_stats_lock); + CDEBUG(D_SEC, "Compressed content: %i pages (%i chunks)\n", cpga_i, - chunk_count); + compr_chunk_count); out: if (cc) crypto_free_comp(cc); @@ -321,19 +347,23 @@ int decompress_request(struct osc_brw_async_args *aa, int page_count) struct brw_page **pga = aa->aa_ppga; struct osc_async_page *oap = NULL; struct ll_compr_hdr *llch = NULL; - char *obd_name = aa->aa_cli->cl_import->imp_obd->obd_name; + struct client_obd *cli = aa->aa_cli; + char *obd_name = cli->cl_import->imp_obd->obd_name; enum ll_compr_type type = LL_COMPR_TYPE_NONE; struct crypto_comp *cc = NULL; struct cl_page *clpage; int next_chunk_min = 0; unsigned int src_size; unsigned int dst_size; + unsigned int total_src_size = 0; + unsigned int total_dst_size = 0; + unsigned int decompr_chunk_count = 0; + unsigned int decompr_pages_count = 0; int pages_per_chunk; char *src = NULL; char *dst = NULL; int chunk_bits; int chunk_size; - int count = 0; int buf_bits; int rc = 0; int i = 0; @@ -371,6 +401,8 @@ int decompress_request(struct osc_brw_async_args *aa, int page_count) * compressed chunk - continue */ if (oap->oap_obj_off & (chunk_size - 1)) { + total_dst_size += pga[i]->count; + total_src_size += pga[i]->count; i++; continue; } @@ -384,6 +416,8 @@ int decompress_request(struct osc_brw_async_args *aa, int page_count) rc); GOTO(out, rc); } + total_dst_size += pga[i]->count; + total_src_size += pga[i]->count; i++; continue; } @@ -436,7 +470,7 @@ int decompress_request(struct osc_brw_async_args *aa, int page_count) } compressed_bytes = llch->llch_compr_size + sizeof(*llch); - compressed_pages = (compressed_bytes >> PAGE_SHIFT) + 1; + compressed_pages = ((compressed_bytes - 1) >> PAGE_SHIFT) + 1; CDEBUG(D_SEC, "compressed bytes %d compressed pages %d\n", compressed_bytes, compressed_pages); /* must be enough pages left in the RPC to hold the compressed @@ -482,6 +516,10 @@ int decompress_request(struct osc_brw_async_args *aa, int page_count) CDEBUG(D_SEC, "Decompressed size %u, pages %d\n", dst_size, decompressed_pages); + total_dst_size += dst_size; + total_src_size += compressed_bytes; + CDEBUG(D_SEC, "total_dst_size: %u, total_src_size: %u\n", + total_dst_size, total_src_size); /* must be enough pages left in the RPC to hold the decompressed * data, if not, the data from disk is probably corrupt */ @@ -499,10 +537,21 @@ int decompress_request(struct osc_brw_async_args *aa, int page_count) next_chunk_min = i + compressed_pages - 1; i += decompressed_pages; CDEBUG(D_SEC, "next chunk min %d\n", next_chunk_min); - count++; + decompr_pages_count += decompressed_pages; + decompr_chunk_count++; } - CDEBUG(D_SEC, - "RPC is %d pages, decompressed %d chunks\n", page_count, count); + CDEBUG(D_SEC, "RPC is %d pages, decompressed %d chunks\n", page_count, + decompr_chunk_count); + + spin_lock(&cli->cl_compr_stats_lock); + cli->cl_r_chunks_compr += decompr_chunk_count; + cli->cl_r_pages_compr += decompr_pages_count; + cli->cl_r_bytes_compr += total_src_size; + cli->cl_r_bytes_raw += total_dst_size; + spin_unlock(&cli->cl_compr_stats_lock); + CDEBUG(D_SEC, "cli->cl_r_bytes_raw: %llu\n", cli->cl_r_bytes_raw); + CDEBUG(D_SEC, "cli->cl_r_bytes_compr: %llu\n", cli->cl_r_bytes_compr); + out: if (cc) crypto_free_comp(cc); diff --git a/lustre/tests/sanity-compr.sh b/lustre/tests/sanity-compr.sh index d6ced48..b0f1bfc 100644 --- a/lustre/tests/sanity-compr.sh +++ b/lustre/tests/sanity-compr.sh @@ -1299,6 +1299,93 @@ test_1080() { } run_test 1080 "Compression header error tolerance" +test_1020() { + (( MDS1_VERSION >= $(version_code 2.14.0-ddn128) )) || + skip "Need MDS version at least 2.14.0-ddn128" + + test_mkdir -p $DIR/$tdir + local tf=$DIR/$tdir/$tfile + local hdf=$LUSTRE/tests/AMSR_E_L3_DailyOcean_V05_20111003.hdf + local tmp_hdf=$TMP/$tfile.hdf + local source=$tmp_hdf + local chunksize=128 + local wb_compr=0 # write_bytes_compr + local wb_raw=0 # write_bytes_raw + local w_chunks=0 # write_chunks_compr + local rb_raw=0 # read_bytes_raw + local rb_compr=0 # read_bytes_compr + local r_chunks=0 # read_chunks_compr + + # clear counters + lctl set_param osc.*.stats_compr 0 + + if [[ -f $hdf.bz2 ]] && type -p bzcat >/dev/null; then + bzcat $hdf.bz2 > $tmp_hdf + elif [[ -f $hdf.bz2 ]] && type -p bunzip2 >/dev/null; then + cp $hdf.bz2 $tmp_hdf.bz2 || error "cp $tmp_hdf.bz2" + bunzip2 $tmp_hdf.bz2 || error "bunzip2 $tmp_hdf.bz2" + else + skip_env "bunzip2 is not installed" + fi + + stack_trap "rm -f $tf; disable_compression" + enable_compression + + lctl set_param debug=sec debug_mb=256 + + # Disable readahead so reads are not expanded to full chinks + $LCTL set_param osc.*.rpc_stats=c + read_ahead_mb=$($LCTL get_param -n llite.*.max_read_ahead_mb) + $LCTL set_param llite.*.max_read_ahead_mb=0 + stack_trap "$LCTL set_param llite.*.max_read_ahead_mb=$read_ahead_mb" EXIT + + + # Simple compressed layout + (OST-0000 and stripe_count == 1) + $LFS setstripe -i 0 -c 1 -E -1 \ + -Z lz4:0 --compress-chunk=$chunksize $DIR/$tdir || + error "set a compress component in $DIR/$tdir failed" + + cp -a $source $tf || error "copy $hdf to $tf failed" + sync + cancel_lru_locks osc + + wb_compr=$(lctl get_param osc.$FSNAME-OST0000*.stats_compr | + awk '/write_bytes_compr:/ { print $2 }') + + wb_raw=$(lctl get_param osc.$FSNAME-OST0000*.stats_compr | + awk '/write_bytes_raw:/ { print $2 }') + + w_chunks=$(lctl get_param osc.$FSNAME-OST0000*.stats_compr | + awk '/write_chunks_compr:/ { print $2 }') + + dd if=$tf of=/dev/null bs=1M || error "read $tf via dd failed" + + rb_compr=$(lctl get_param osc.$FSNAME-OST0000*.stats_compr | + awk '/read_bytes_compr:/ { print $2 }') + + rb_raw=$(lctl get_param osc.$FSNAME-OST0000*.stats_compr | + awk '/read_bytes_raw:/ { print $2 }') + + r_chunks=$(lctl get_param osc.$FSNAME-OST0000*.stats_compr | + awk '/read_chunks_compr:/ { print $2 }') + + # Print stats for debug + du --block-size=1 $tf + lctl get_param osc.$FSNAME-OST0000*.stats_compr + + # Uncomment the check after adding the server part. + # (( rb_compr == wb_compr )) || + # error "compr size does not match. '$rb_compr' != '$wb_compr'" + + (( rb_raw == wb_raw )) || + error "original size does not match. '$rb_raw' != '$wb_raw'" + + # Uncomment the check after adding the server part. + # (( r_chunks == w_chunks )) || + # error "num of chunks does not match. '$r_chunks' != '$w_chunks'" +} +run_test 1020 "Сhecking compression counters" + complete_test $SECONDS check_and_cleanup_lustre declare -a logs=($ONLY) -- 1.8.3.1