Whamcloud - gitweb
LU-17000 contrib: script to prepare coverity builds 00/53400/4
authorTimothy Day <timday@amazon.com>
Sun, 10 Dec 2023 22:58:09 +0000 (22:58 +0000)
committerOleg Drokin <green@whamcloud.com>
Sun, 4 Feb 2024 08:29:08 +0000 (08:29 +0000)
Add script 'coverity-run' to semi-automate running
and submitting Coverity builds for Lustre. This
should make it much easier to reproducibly submit
builds to Coverity - and serve as an example of
how the Coverity build process works. It should
also provide more transparency in how builds are
being prepared for Coverity.

Add a Vagrantfile for the Vagrant VM used during
the build process.

Update in-tree Documentation.

Test-Parameters: trivial
Signed-off-by: Timothy Day <timday@amazon.com>
Change-Id: I050b10d9df0e4e4c1b8bcc91a3c296c11f27ffef
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53400
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Arshad Hussain <arshad.hussain@aeoncomputing.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
.gitignore
Documentation/clang-and-static-analysis.txt
Vagrantfile [new file with mode: 0644]
contrib/coverity/.gitignore [new file with mode: 0644]
contrib/coverity/coverity-run [new file with mode: 0755]

index ddf3e32..5423df6 100644 (file)
@@ -102,6 +102,12 @@ TAGS
 /.venv
 /venv
 
+# Vagrant artifacts
+.vagrant/
+
+# Coverity results
+*.tgz
+
 #
 # Top level generic files
 #
index 8230c8c..06b66d2 100644 (file)
@@ -121,7 +121,9 @@ a Coverity bug should ideally have a line in the commit message like:
 
 This makes it easier to track which bugs still need to be fixed. Currently, the Coverity
 Scan project is maintained in an adhoc manner. Hence, the build may be outdated. But it
-can be updated easily following the instructions on the site.
+can be updated easily following the instructions on the site or using the script in
+'contrib/coverity' in the Lustre tree. Running 'coverity-run list' will provide more
+details.
 
 4.2 References
 ==============
diff --git a/Vagrantfile b/Vagrantfile
new file mode 100644 (file)
index 0000000..588fecf
--- /dev/null
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: GPL-2.0
+
+#
+# This file is part of Lustre, http://www.lustre.org/
+#
+# contrib/coverity/Vagrantfile
+#
+# Vagrant definition for a CentOS VM to run a Lustre
+# build for Coverity.
+#
+# Author: Timothy Day <timday@amazon.com>
+#
+
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "centos/8"
+
+  # Customizations
+  config.vm.provider "libvirt" do |libvirt|
+    libvirt.machine_virtual_size = 40
+    libvirt.memory = 8192
+    libvirt.cpus = 4
+  end
+
+  config.vm.provision "shell", inline: <<-SHELL
+      # Volume Setup
+      sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
+      sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-*
+      dnf update -y
+      dnf install -y cloud-utils-growpart
+      growpart /dev/vda 1
+      xfs_growfs /dev/vda1
+
+      # Networking Setup
+      GIVEN_IP=$(ip address show eth0 | awk -F' ' '$1 == "inet" { print $2 }' | awk -F'/' '{ print $1 }')
+      grep $(hostname) /etc/hosts
+      sed -i "s/$(hostname) //g" /etc/hosts
+      echo "$GIVEN_IP $(hostname)" >> /etc/hosts
+      grep $(hostname) /etc/hosts
+
+      # Add repos
+      dnf config-manager --set-enabled powertools
+      dnf config-manager --add-repo=https://downloads.whamcloud.com/public/e2fsprogs/latest/el8
+
+      # General kernel tools
+      dnf groupinstall -y 'Development Tools'
+
+      # Debug info (needed for ldiskfs)
+      sudo dnf install -y --enablerepo=*debug* kernel-debuginfo
+
+      # Install Lustre/ZFS dependencies and tools
+      dnf install -y --nogpgcheck git libyaml-devel libnl3-devel libmount-devel \
+          wget ncurses-devel bc dwarves kernel kernel-devel openssl-devel \
+          binutils-devel lsof crash kexec-tools perf psmisc e2fsprogs-devel \
+          elfutils-libelf-devel libudev-devel libattr-devel libaio-devel libuuid-devel \
+          libblkid-devel libtirpc-devel libffi-devel ncompress python3-cffi python3-devel \
+          python3-packaging
+  SHELL
+end
diff --git a/contrib/coverity/.gitignore b/contrib/coverity/.gitignore
new file mode 100644 (file)
index 0000000..4456f24
--- /dev/null
@@ -0,0 +1,6 @@
+# Vagrant artifacts
+.vagrant/
+vagrant-ssh.config
+
+# Coverity results
+*.tgz
diff --git a/contrib/coverity/coverity-run b/contrib/coverity/coverity-run
new file mode 100755 (executable)
index 0000000..6e0db09
--- /dev/null
@@ -0,0 +1,184 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+#
+# This file is part of Lustre, http://www.lustre.org/
+#
+# contrib/coverity/coverity-run
+#
+# Setup a simple VM for running Coverity builds
+# to upload to https://scan.coverity.com/projects/lustre
+#
+# Author: Timothy Day <timday@amazon.com>
+#
+
+#
+# Output list of all commands
+#
+function cr_list() {
+       less -F <<EOF
+Usage: ${0##*/} [options]
+Helper for running Coverity builds for Lustre
+       setup         Create Vagrant VM.
+       build         Build Lustre and dependencies. The users should
+                     validate the everything builds correctly.
+       run           Run Coverity scan. You must provide the path
+                     to the Coverity tool tarball using the COV_PATH
+                     env variable. This can be downloaded from the
+                     Coverity website.
+       clean         Remove Vagrant artifacts.
+       all           Run all of the above steps in order.
+       list          List all possible commands.
+
+The commands should be run in the following order:
+
+       setup -> build -> run -> clean
+
+The tool currently has a dependency on Vagrant and the 'libvirt'
+provider. After the scan is run, there will be a tar file in the
+'contrib/coverity' directory. This should be uploaded to Coverity
+via the web portal using the Lustre version output during the
+'run' step as the software version.
+EOF
+       exit
+}
+
+#
+# Create VM using Vagrant.
+#
+function cr_setup() {
+       vagrant up
+
+       # Save/test ssh config
+       vagrant ssh-config > vagrant-ssh.config
+       ssh -F vagrant-ssh.config default "uname -r"
+
+       # Restart VM
+       vagrant reload
+}
+
+#
+# Perform a test build of ZFS and Lustre. User must
+# validate that correct modules get built.
+#
+function cr_build() {
+       # Build ZFS
+       ssh -F vagrant-ssh.config default <<EOF
+# Grab repo
+rm -rf ~/zfs
+git clone https://github.com/openzfs/zfs.git
+
+# Build
+cd ~/zfs
+git checkout -b coverity-run zfs-2.1.11
+sh autogen.sh
+./configure
+make -s -j\$(nproc)
+
+# Install
+sudo make install
+EOF
+
+       # Build Lustre
+       ssh -F vagrant-ssh.config default <<EOF
+# Grab repo
+rm -rf ~/lustre-release
+git clone git://git.whamcloud.com/fs/lustre-release.git
+
+# Build
+cd ~/lustre-release
+./autogen.sh
+./configure
+make -s -j\$(nproc)
+
+# Report
+echo "KERNEL MODULES BUILT:"
+find . -name *.ko
+EOF
+}
+
+#
+# Run coverity scan. 'setup' and 'build' must have been
+# run beforehand. Automatically copies the tar file that
+# should be uploaded to Coverity along with the Lustre
+# version.
+#
+# The build is run sequentially, due to limitations with
+# cov-build.
+#
+function cr_run() {
+       # Check if COV_PATH is defined
+       if [[ -z ${COV_PATH+x} ]]; then
+               echo "Provide the path to the Coverity tool tarball using COV_PATH."
+               exit
+       fi
+
+       # Copy build tool to VM
+       scp -F vagrant-ssh.config "$COV_PATH" default:~
+
+       # Run scan
+       ssh -F vagrant-ssh.config default <<EOF
+# Get coverity tool
+cd ~
+tar xf *.tar.gz
+
+# Clean and run
+cd ~/lustre-release
+make clean
+"\$(find ~ -name cov-build)" --dir cov-int make
+tar czvf "lustre-coverity-\$(date +"%m-%d-%Y").tgz" cov-int
+
+# Report
+echo "KERNEL MODULES BUILT:"
+find . -name *.ko
+echo "LUSTRE VERSION:"
+cat LUSTRE-VERSION-FILE
+EOF
+
+       # Grab scan results
+       scp -F vagrant-ssh.config default:~/lustre-release/*.tgz .
+}
+
+#
+# Destroy Vagrant VM.
+#
+function cr_clean() {
+       vagrant destroy
+       rm -f vagrant-ssh.config
+}
+
+#
+# Run all steps in the correct order.
+#
+function cr_all() {
+       # Check if COV_PATH is defined
+       if [[ -z ${COV_PATH+x} ]]; then
+               echo "Provide the path to the Coverity tool tarball using COV_PATH."
+               exit
+       fi
+
+       cr_setup
+       cr_build
+       cr_run
+       cr_clean
+}
+
+# Run as root or with sudo
+if [[ "$EUID" -ne 0 ]]; then
+       echo "Please run as root or with sudo."
+       exit
+fi
+
+# Process options
+for arg in "$@"; do
+       shift
+       case "$arg" in
+               setup) cr_setup;;
+               build) cr_build;;
+               run) cr_run;;
+               clean) cr_clean;;
+               all) cr_all;;
+               list) cr_list;;
+               *) cr_list;;
+       esac
+done