From 5b998d803ffbd02ed306195d231e0a3478658574 Mon Sep 17 00:00:00 2001 From: Emoly Liu Date: Wed, 11 May 2022 20:12:00 +0800 Subject: [PATCH] LU-14736 utils: update leak-finder.pl for new format Update leak-finder.pl to handle some of the newer log formats, so that it produces more useful results. The changes include: - add option "--by_func" to sort leak logs by function name in ascending order; - add option "--summary" to print a summary report by the total number of leak bytes of each function in ascending order in YAML format, "- { func: , alloc_bytes: , leak_count: , leak_bytes: }," - define LIBCFS_MEM_MSG() to print alloc/free log in a uniform format, as follows, also define LIBCFS_ALLOC_POST() and LIBCFS_FREE_PRE(): mask:subs:cpu:epoch second.usec:?:pid:?: (filename:line:function_name()) alloc-type 'var_name': size at memory_address - correct some alloc/free debug messages in lnet part. Test-Parameters: trivial Signed-off-by: Andreas Dilger Signed-off-by: Emoly Liu Change-Id: Idee539f7aebbd49a52fe5254d292860c283ebbe5 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/43983 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Zhenyu Xu Reviewed-by: Oleg Drokin --- libcfs/include/libcfs/libcfs_private.h | 23 ++++++---- lnet/include/lnet/lib-lnet.h | 7 ++- lnet/klnds/gnilnd/gnilnd.h | 4 +- lnet/klnds/gnilnd/gnilnd_cb.c | 27 ++++++----- lnet/klnds/gnilnd/gnilnd_conn.c | 4 +- lnet/lnet/lib-md.c | 4 +- lnet/lnet/lib-me.c | 4 +- lnet/lnet/lib-ptl.c | 4 +- lnet/lnet/udsp.c | 4 +- lustre/include/obd_support.h | 20 ++++---- lustre/mdt/mdt_open.c | 2 +- lustre/obdclass/genops.c | 2 +- lustre/tests/leak_finder.pl | 84 +++++++++++++++++++++++++++++----- 13 files changed, 123 insertions(+), 66 deletions(-) diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h index a60f142..02fc165 100644 --- a/libcfs/include/libcfs/libcfs_private.h +++ b/libcfs/include/libcfs/libcfs_private.h @@ -156,20 +156,27 @@ do { \ ((mask) & GFP_ATOMIC)) != 0); \ } while (0) -#define LIBCFS_ALLOC_POST(ptr, size) \ +/* message format here needs to match regexp in lustre/tests/leak_finder.pl */ +#define LIBCFS_MEM_MSG(ptr, size, name) \ + CDEBUG(D_MALLOC, name " '" #ptr "': %d at %p.\n", (int)(size), ptr) + +#define LIBCFS_ALLOC_POST(ptr, size, name) \ do { \ if (unlikely((ptr) == NULL)) { \ CERROR("LNET: out of memory at %s:%d (tried to alloc '" \ #ptr "' = %d)\n", __FILE__, __LINE__, (int)(size)); \ CERROR("LNET: %lld total bytes allocated by lnet\n", \ - libcfs_kmem_read()); \ + libcfs_kmem_read()); \ } else { \ libcfs_kmem_inc((ptr), (size)); \ - CDEBUG(D_MALLOC, "alloc '" #ptr "': %d at %p (tot %lld).\n", \ - (int)(size), (ptr), libcfs_kmem_read()); \ + LIBCFS_MEM_MSG(ptr, size, name); \ } \ } while (0) +#define LIBCFS_FREE_PRE(ptr, size, name) \ + libcfs_kmem_dec((ptr), (size)); \ + LIBCFS_MEM_MSG(ptr, size, name) + /** * allocate memory with GFP flags @mask * The allocated memory is zeroed-out. @@ -179,7 +186,7 @@ do { \ LIBCFS_ALLOC_PRE((size), (mask)); \ (ptr) = (size) <= LIBCFS_VMALLOC_SIZE ? \ kzalloc((size), (mask)) : vzalloc(size); \ - LIBCFS_ALLOC_POST((ptr), (size)); \ + LIBCFS_ALLOC_POST((ptr), (size), "alloc"); \ } while (0) /** @@ -206,7 +213,7 @@ do { \ (ptr) = (size) <= LIBCFS_VMALLOC_SIZE ? \ cfs_cpt_malloc((cptab), (cpt), (size), (mask) | __GFP_ZERO) : \ cfs_cpt_vzalloc((cptab), (cpt), (size)); \ - LIBCFS_ALLOC_POST((ptr), (size)); \ + LIBCFS_ALLOC_POST((ptr), (size), "alloc"); \ } while (0) /** default numa allocator */ @@ -224,9 +231,7 @@ do { \ "%s:%d\n", s, __FILE__, __LINE__); \ break; \ } \ - libcfs_kmem_dec((ptr), s); \ - CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %lld).\n", \ - s, (ptr), libcfs_kmem_read()); \ + LIBCFS_FREE_PRE(ptr, size, "kfreed"); \ if (unlikely(s > LIBCFS_VMALLOC_SIZE)) \ libcfs_vfree_atomic(ptr); \ else \ diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index aaa24d9..671df0a 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -306,7 +306,7 @@ lnet_md_free(struct lnet_libmd *md) size = offsetof(struct lnet_libmd, md_kiov[md->md_niov]); if (size <= LNET_SMALL_MD_SIZE) { - CDEBUG(D_MALLOC, "slab-freed 'md' at %p.\n", md); + LIBCFS_MEM_MSG(md, size, "slab-freed"); kmem_cache_free(lnet_small_mds_cachep, md); } else { LIBCFS_FREE(md, size); @@ -478,15 +478,14 @@ lnet_rspt_alloc(int cpt) the_lnet.ln_counters[cpt]->lct_health.lch_rst_alloc++; lnet_net_unlock(cpt); } - CDEBUG(D_MALLOC, "rspt alloc %p\n", rspt); + LIBCFS_ALLOC_POST(rspt, sizeof(*rspt), "alloc"); return rspt; } static inline void lnet_rspt_free(struct lnet_rsp_tracker *rspt, int cpt) { - CDEBUG(D_MALLOC, "rspt free %p\n", rspt); - + LIBCFS_FREE_PRE(rspt, sizeof(*rspt), "free"); kmem_cache_free(lnet_rspt_cachep, rspt); lnet_net_lock(cpt); the_lnet.ln_counters[cpt]->lct_health.lch_rst_alloc--; diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h index 4ad528b..7c335e2 100644 --- a/lnet/klnds/gnilnd/gnilnd.h +++ b/lnet/klnds/gnilnd/gnilnd.h @@ -1004,13 +1004,13 @@ static inline void *kgnilnd_vzalloc(int size) else ret = __ll_vmalloc(size, __GFP_HIGHMEM | GFP_NOIO | __GFP_ZERO); - LIBCFS_ALLOC_POST(ret, size); + LIBCFS_ALLOC_POST(ret, size, "alloc"); return ret; } static inline void kgnilnd_vfree(void *ptr, int size) { - libcfs_kmem_dec(ptr, size); + LIBCFS_FREE_PRE(ptr, size, "vfree"); vfree(ptr); } diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c index fbcec23..3ed3dad 100644 --- a/lnet/klnds/gnilnd/gnilnd_cb.c +++ b/lnet/klnds/gnilnd/gnilnd_cb.c @@ -258,20 +258,22 @@ kgnilnd_free_tx(kgn_tx_t *tx) /* we only allocate this if we need to */ if (tx->tx_phys != NULL) { kmem_cache_free(kgnilnd_data.kgn_tx_phys_cache, tx->tx_phys); - CDEBUG(D_MALLOC, "slab-freed 'tx_phys': %lu at %p.\n", - GNILND_MAX_IOV * sizeof(gni_mem_segment_t), tx->tx_phys); + LIBCFS_MEM_MSG(tx->tx_phys, + GNILND_MAX_IOV * sizeof(gni_mem_segment_t), + "slab-freed"); } /* Only free the buffer if we used it */ if (tx->tx_buffer_copy != NULL) { + LIBCFS_MEM_MSG(tx->tx_buffer_copy, tx->tx_rdma_desc.length, + "vfreed"); kgnilnd_vfree(tx->tx_buffer_copy, tx->tx_rdma_desc.length); tx->tx_buffer_copy = NULL; - CDEBUG(D_MALLOC, "vfreed buffer2\n"); } #if 0 KGNILND_POISON(tx, 0x5a, sizeof(kgn_tx_t)); #endif - CDEBUG(D_MALLOC, "slab-freed 'tx': %lu at %p.\n", sizeof(*tx), tx); + LIBCFS_MEM_MSG(tx, sizeof(*tx), "slab-freed"); kmem_cache_free(kgnilnd_data.kgn_tx_cache, tx); } @@ -288,8 +290,7 @@ kgnilnd_alloc_tx (void) CERROR("failed to allocate tx\n"); return NULL; } - CDEBUG(D_MALLOC, "slab-alloced 'tx': %lu at %p.\n", - sizeof(*tx), tx); + LIBCFS_MEM_MSG(tx, sizeof(*tx), "slab-alloced"); /* setup everything here to minimize time under the lock */ tx->tx_buftype = GNILND_BUF_NONE; @@ -639,8 +640,9 @@ kgnilnd_setup_phys_buffer(kgn_tx_t *tx, int nkiov, struct bio_vec *kiov, GOTO(error, rc); } - CDEBUG(D_MALLOC, "slab-alloced 'tx->tx_phys': %lu at %p.\n", - GNILND_MAX_IOV * sizeof(gni_mem_segment_t), tx->tx_phys); + LIBCFS_MEM_MSG(tx->tx_phys, + GNILND_MAX_IOV * sizeof(gni_mem_segment_t), + "slab-alloced"); /* if loops changes, please change kgnilnd_cksum_kiov * and kgnilnd_setup_immediate_buffer */ @@ -722,8 +724,7 @@ kgnilnd_setup_phys_buffer(kgn_tx_t *tx, int nkiov, struct bio_vec *kiov, error: if (tx->tx_phys != NULL) { kmem_cache_free(kgnilnd_data.kgn_tx_phys_cache, tx->tx_phys); - CDEBUG(D_MALLOC, "slab-freed 'tx_phys': %lu at %p.\n", - sizeof(*tx->tx_phys), tx->tx_phys); + LIBCFS_MEM_MSG(tx->tx_phys, sizeof(*tx->tx_phys), "slab-freed"); tx->tx_phys = NULL; } return rc; @@ -1959,8 +1960,7 @@ kgnilnd_alloc_rx(void) CERROR("failed to allocate rx\n"); return NULL; } - CDEBUG(D_MALLOC, "slab-alloced 'rx': %lu at %p.\n", - sizeof(*rx), rx); + LIBCFS_MEM_MSG(rx, sizeof(*rx), "slab-alloced"); /* no memset to zero, we'll always fill all members */ return rx; @@ -2010,9 +2010,8 @@ kgnilnd_consume_rx(kgn_rx_t *rx) kgnilnd_release_msg(conn); } + LIBCFS_MEM_MSG(rx, sizeof(*rx), "slab-freed"); kmem_cache_free(kgnilnd_data.kgn_rx_cache, rx); - CDEBUG(D_MALLOC, "slab-freed 'rx': %lu at %p.\n", - sizeof(*rx), rx); } int diff --git a/lnet/klnds/gnilnd/gnilnd_conn.c b/lnet/klnds/gnilnd/gnilnd_conn.c index d95e15f..d7b6ef3 100644 --- a/lnet/klnds/gnilnd/gnilnd_conn.c +++ b/lnet/klnds/gnilnd/gnilnd_conn.c @@ -344,10 +344,8 @@ kgnilnd_free_fmablk_locked(kgn_device_t *dev, kgn_fma_memblock_t *fma_blk) * purgatory holds. While we have purgatory holds, we might check the conn * RX mailbox during the CLOSING process. It is possible that kgni might * try to look into the RX side for credits when sending the CLOSE msg too */ - CDEBUG(D_MALLOC, "fmablk %p free buffer %p mbox_size %d\n", - fma_blk, fma_blk->gnm_block, fma_blk->gnm_mbox_size); - if (fma_blk->gnm_state == GNILND_FMABLK_PHYS) { + LIBCFS_MEM_MSG(fma_blk->gnm_block, fma_blk->gnm_mbox_size, "free"); kmem_cache_free(kgnilnd_data.kgn_mbox_cache, fma_blk->gnm_block); } else { kgnilnd_vfree(fma_blk->gnm_block, fma_blk->gnm_blk_size); diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index 72b9aa7..9e200c9 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -180,9 +180,7 @@ lnet_md_build(const struct lnet_md *umd, int unlink) if (size <= LNET_SMALL_MD_SIZE) { lmd = kmem_cache_zalloc(lnet_small_mds_cachep, GFP_NOFS); if (lmd) { - CDEBUG(D_MALLOC, - "slab-alloced 'md' of size %u at %p.\n", - size, lmd); + LIBCFS_MEM_MSG(lmd, size, "slab-alloced"); } else { CDEBUG(D_MALLOC, "failed to allocate 'md' of size %u\n", size); diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c index 8d7c9ee..56b174d 100644 --- a/lnet/lnet/lib-me.c +++ b/lnet/lnet/lib-me.c @@ -89,7 +89,7 @@ LNetMEAttach(unsigned int portal, CDEBUG(D_MALLOC, "failed to allocate 'me'\n"); return ERR_PTR(-ENOMEM); } - CDEBUG(D_MALLOC, "slab-alloced 'me' at %p.\n", me); + LIBCFS_ALLOC_POST(me, sizeof(*me), "slab-alloced"); lnet_res_lock(mtable->mt_cpt); @@ -132,7 +132,7 @@ lnet_me_unlink(struct lnet_me *me) lnet_md_unlink(md); } - CDEBUG(D_MALLOC, "slab-freed 'me' at %p.\n", me); + LIBCFS_FREE_PRE(me, sizeof(*me), "slab-freed"); kmem_cache_free(lnet_mes_cachep, me); } diff --git a/lnet/lnet/lib-ptl.c b/lnet/lnet/lib-ptl.c index 4196424..40f7214 100644 --- a/lnet/lnet/lib-ptl.c +++ b/lnet/lnet/lib-ptl.c @@ -770,9 +770,7 @@ lnet_ptl_cleanup(struct lnet_portal *ptl) me_list)) != NULL) { CERROR("Active ME %p on exit\n", me); list_del(&me->me_list); - CDEBUG(D_MALLOC, - "slab-freed 'me' at %p in cleanup.\n", - me); + LIBCFS_FREE_PRE(me, sizeof(*me), "slab-freed"); kmem_cache_free(lnet_mes_cachep, me); } } diff --git a/lnet/lnet/udsp.c b/lnet/lnet/udsp.c index bce643b..185d27f 100644 --- a/lnet/lnet/udsp.c +++ b/lnet/lnet/udsp.c @@ -1091,7 +1091,7 @@ lnet_udsp_alloc(void) INIT_LIST_HEAD(&udsp->udsp_rte.ud_addr_range); INIT_LIST_HEAD(&udsp->udsp_rte.ud_net_id.udn_net_num_range); - CDEBUG(D_MALLOC, "udsp alloc %p\n", udsp); + LIBCFS_ALLOC_POST(udsp, sizeof(*udsp), "alloc"); return udsp; } @@ -1124,7 +1124,7 @@ lnet_udsp_free(struct lnet_udsp *udsp) lnet_udsp_nid_descr_free(&udsp->udsp_dst); lnet_udsp_nid_descr_free(&udsp->udsp_rte); - CDEBUG(D_MALLOC, "udsp free %p\n", udsp); + LIBCFS_FREE_PRE(udsp, sizeof(*udsp), "kfreed"); kmem_cache_free(lnet_udsp_cachep, udsp); } diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 7b0e6ed..5a88c3d 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -805,16 +805,16 @@ static inline void obd_memory_sub(long size) #define OBD_DEBUG_MEMUSAGE (1) #if OBD_DEBUG_MEMUSAGE -#define OBD_ALLOC_POST(ptr, size, name) \ - obd_memory_add(size); \ - CDEBUG(D_MALLOC, name " '" #ptr "': %d at %p.\n", \ - (int)(size), ptr) - -#define OBD_FREE_PRE(ptr, size, name) \ - LASSERT(ptr); \ - obd_memory_sub(size); \ - CDEBUG(D_MALLOC, name " '" #ptr "': %d at %p.\n", \ - (int)(size), ptr); +/* message format here needs to match regexp in lustre/tests/leak_finder.pl */ +#define OBD_ALLOC_POST(ptr, size, name) \ + obd_memory_add(size); \ + LIBCFS_MEM_MSG(ptr, size, name) + +/* message format here needs to match regexp in lustre/tests/leak_finder.pl */ +#define OBD_FREE_PRE(ptr, size, name) \ + LASSERT(ptr); \ + obd_memory_sub(size); \ + LIBCFS_MEM_MSG(ptr, size, name) #else /* !OBD_DEBUG_MEMUSAGE */ diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c index 00a75e3d..7cbe4c9 100644 --- a/lustre/mdt/mdt_open.c +++ b/lustre/mdt/mdt_open.c @@ -103,7 +103,7 @@ void mdt_mfd_free(struct mdt_file_data *mfd) { LASSERT(refcount_read(&mfd->mfd_open_handle.h_ref) == 1); LASSERT(list_empty(&mfd->mfd_list)); - OBD_FREE_PRE(mfd, sizeof(*mfd), "rcu"); + OBD_FREE_PRE(mfd, sizeof(*mfd), "kfree_rcu"); kfree_rcu(mfd, mfd_open_handle.h_rcu); } diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 66397e5..64a68f2 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -958,7 +958,7 @@ static void class_export_destroy(struct obd_export *exp) if (exp != obd->obd_self_export) class_decref(obd, "export", exp); - OBD_FREE_PRE(exp, sizeof(*exp), "rcu"); + OBD_FREE_PRE(exp, sizeof(*exp), "kfree_rcu"); kfree_rcu(exp, exp_handle.h_rcu); EXIT; } diff --git a/lustre/tests/leak_finder.pl b/lustre/tests/leak_finder.pl index c230e59..d2db8a5 100644 --- a/lustre/tests/leak_finder.pl +++ b/lustre/tests/leak_finder.pl @@ -16,8 +16,13 @@ my %cpu; my $start_time = 0; if (!defined($ARGV[0])) { - print "No log file specified\n"; - exit + print "No log file specified\n"; + print "Usage: leak_finder.pl [--option]\n"; + print " --by_func show leak logs by function name in ascending order.\n"; + print " --summary implies --by_func, print a summary report by \n"; + print " the number of total leak bytes of each function \n"; + print " in ascending order in YAML format.\n"; + exit } open(INFILE, $ARGV[0]); @@ -47,19 +52,22 @@ seek(INFILE, 0, 0); while ($line = ) { $debug_line++; my ($file, $func, $lno, $name, $size, $addr, $type); - if ($line =~ m/^(.*)\((.*):(\d+):(.*)\(\)\) (k|v|slab-)(.*) '(.*)': (\d+) at ([\da-f]+)/){ + # message format here needs to match OBD_ALLOC_POST()/OBD_FREE_PRE() + # mask:subs:cpu:epoch second.usec:?:pid:?:(filename:line:function_name()) + # alloc-type 'var_name': size at memory_address. + if ($line =~ m/^(.*)\((.*):(\d+):(.*)\(\)\) (k[m]?|v[m]?|slab-|)(alloc(ed)?|free[d]?(_rcu)?) '(.*)': (\d+) at ([\da-f]+)/) { @parsed = split(":", $1); if ($parsed[3] <= $start_time) { next; } - + $file = $2; $lno = $3; $func = $4; $type = $6; - $name = $7; - $size = $8; - $addr = $9; + $name = $9; + $size = $10; + $addr = $11; # we can't dump the log after portals has exited, so skip "leaks" # from memory freed in the portals module unloading. @@ -68,11 +76,17 @@ while ($line = ) { } printf("%8s %6d bytes at %s called %s (%s:%s:%d)\n", $type, $size, $addr, $name, $file, $func, $lno); + } elsif ($line =~ m/(alloc(ed)?|free[d]?).*at [0-9a-f]*/) { + # alloc/free line that didn't match regexp, notify user of missed line + print STDERR "Couldn't parse line $debug_line, script needs to be fixed:\n$line"; + next; } else { + # line not related to alloc/free, skip it silently + #print STDERR "Couldn't parse $line"; next; } - if (index($type, 'alloced') >= 0) { + if (index($type, 'alloc') >= 0) { if (defined($memory->{$addr})) { print STDOUT "*** Two allocs with the same address ($size bytes at $addr, $file:$func:$lno)\n"; print STDOUT " first malloc at $memory->{$addr}->{file}:$memory->{$addr}->{func}:$memory->{$addr}->{lno}, second at $file:$func:$lno\n"; @@ -109,15 +123,61 @@ while ($line = ) { } close(INFILE); -# Sort leak output by allocation time +my $aa; +my $bb; my @sorted = sort { - return $memory->{$a}->{debug_line} <=> $memory->{$b}->{debug_line}; + if (defined($ARGV[1])) { + if ($ARGV[1] eq "--by_func" or $ARGV[1] eq "--summary") { + # Sort leak output by source code position + $aa = "$memory->{$a}->{func}:$memory->{$a}->{lno}:$memory->{$a}->{name}:$memory->{$a}->{size}"; + $bb = "$memory->{$b}->{func}:$memory->{$b}->{lno}:$memory->{$b}->{name}:$memory->{$b}->{size}"; + $aa cmp $bb; + } + } else { + # Sort leak output by allocation time + $memory->{$a}->{debug_line} <=> $memory->{$b}->{debug_line}; + } } keys(%{$memory}); +$aa = ""; +$bb = ""; my $key; +my $leak_count = 1; +my @records; +my $leak_size = 0; +my $leak_func; +my $i = 0; foreach $key (@sorted) { - my ($oldname, $oldsize, $oldfile, $oldfunc, $oldlno) = $memory->{$key}; - print STDOUT "*** Leak: $memory->{$key}->{size} bytes allocated at $key ($memory->{$key}->{file}:$memory->{$key}->{func}:$memory->{$key}->{lno}, debug file line $memory->{$key}->{debug_line})\n"; + if (defined($ARGV[1]) and $ARGV[1] eq "--summary") { + $aa = "$memory->{$key}->{func}:$memory->{$key}->{lno}:$memory->{$key}->{name}:$memory->{$key}->{size}"; + if ($bb eq $aa) { + $leak_count++; + } elsif ($bb ne ""){ + $records[$i]->{func} = $leak_func; + $records[$i]->{size} = $leak_size; + $records[$i]->{count} = $leak_count; + $records[$i]->{total} = $leak_count * $leak_size;; + $bb = $aa; + $i++; + $leak_count = 1; + } else { + $bb = $aa; + } + $leak_func = "$memory->{$key}->{func}:$memory->{$key}->{lno}:$memory->{$key}->{name}"; + $leak_size = $memory->{$key}->{size}; + } else { + print STDOUT "*** Leak: $memory->{$key}->{size} bytes allocated at $key ($memory->{$key}->{file}:$memory->{$key}->{func}:$memory->{$key}->{lno}:$memory->{$key}->{name}, debug file line $memory->{$key}->{debug_line})\n"; + } } +if (defined($ARGV[1]) and $ARGV[1] eq "--summary") { + # print a summary report by total leak bytes in ASC order + my @sorted_records = sort { + $a->{total} <=> $b->{total}; + } @records; + foreach $key (@sorted_records) { + printf("- { func: %-58s, alloc_bytes: %-8d, leak_count: %-6d, leak_bytes: %-8d },\n", + $key->{func}, $key->{size}, $key->{count}, $key->{total}); + } +} print STDOUT "maximum used: $max, amount leaked: $total\n"; -- 1.8.3.1