diff --git a/scripts/perf/Containerfile b/scripts/perf/Containerfile new file mode 100644 index 00000000..30021fa5 --- /dev/null +++ b/scripts/perf/Containerfile @@ -0,0 +1,44 @@ +FROM fedora:41 +LABEL description="Test compilers with llvm-test-suite" + +USER root +WORKDIR /root + +# Install deps to run test-suite +RUN dnf install -y \ + clang \ + cmake \ + coreutils \ + envsubst \ + fedora-packager \ + git \ + jq \ + jq \ + ninja-build \ + perf \ + python3-lit \ + python3-pandas \ + python3-pip \ + python3-scipy \ + python3-setuptools \ + python3-virtualenv \ + tcl \ + tcl-devel \ + tcl-tclreadline \ + tcl-tclxml \ + tcl-tclxml-devel \ + tcl-thread-devel \ + tcl-zlib \ + which + +# Clone test suite (in correct version for installed clang version) +# See https://llvm.org/docs/TestSuiteGuide.html +# RUN export VERSION=`clang --version | grep -ioP 'clang version\s\K[0-9\.]+'` \ +# && git clone --depth=1 --branch llvmorg-${VERSION} https://github.com/llvm/llvm-test-suite.git test-suite +RUN git clone --depth=1 https://github.com/llvm/llvm-test-suite.git test-suite + +RUN dnf install -y 'dnf-command(copr)' + +COPY entrypoint.sh /root/entrypoint.sh +USER root +ENTRYPOINT [ "/root/entrypoint.sh" ] diff --git a/scripts/perf/Makefile b/scripts/perf/Makefile new file mode 100644 index 00000000..2e5f7932 --- /dev/null +++ b/scripts/perf/Makefile @@ -0,0 +1,10 @@ +.PHONY: all +all: setup run + +.PHONY: setup +setup: + podman build -t evaluation . + +.PHONY: run +run: + podman run -it --rm evaluation diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 00000000..ea07c19e --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,27 @@ +# README + +This container setup allows you to compare the system llvm against "big-merge" and "pgo". + +# How to + +Just run `make` to build and run the container image. It takes a long time to complete. + +Then you'll be promted to a terminal in the container where you'll find these files: + +``` +~/results-system-vs-pgo.txt +~/results-system-vs-big-merge.txt +~/results-big-merge-vs-pgo.txt +``` + +The names speak for themselves. + +## How to change to OS + +If you want to change the version of the operating system, go to `Containerfile` and change the line that looks like this: `FROM fedora:40`. Change it to `FROM fedora:41` or something else. + +Then run `make` again. + +## How to change the date for which to compare results? + +Go to `entrypoint.sh` and change the line that defines `yyyymmdd` to the year-month-date of your liking. diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh new file mode 100755 index 00000000..34aac745 --- /dev/null +++ b/scripts/perf/entrypoint.sh @@ -0,0 +1,124 @@ +#!/usr/bin/bash + +set -x +set -e + +function configure_build_run { + # Configure the test suite + cmake \ + -DCMAKE_GENERATOR=Ninja \ + -DCMAKE_C_COMPILER=/usr/bin/clang \ + -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ + -DTEST_SUITE_BENCHMARKING_ONLY=ON \ + -DTEST_SUITE_COLLECT_STATS=ON \ + -DTEST_SUITE_USE_PERF=OFF \ + -DTEST_SUITE_SUBDIRS=CTMark \ + -DTEST_SUITE_RUN_BENCHMARKS=OFF \ + -C~/test-suite/cmake/caches/O3.cmake \ + ~/test-suite + + # Build the test-suite + ninja -j1 + + # Run the tests with lit: + lit -v -o results.json . || true +} + +# Query version information for given day +yyyymmdd=20240911 +git_rev=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-git-revision-${yyyymmdd}.txt) +git_rev_short="${git_rev:0:14}" +llvm_release=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-release-${yyyymmdd}.txt) +rpm_suffix="${llvm_release}~pre${yyyymmdd}.g${git_rev_short}" + +echo "git_rev=$git_rev" +echo "git_rev_short=$git_rev_short" +echo "llvm_release=$llvm_release" +echo "rpm_suffix=$rpm_suffix" + +###################################################################################### +# PGO +###################################################################################### + +# Install and enable the repository that provides the PGO LLVM Toolchain +# See https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo +dnf copr enable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} +repo_file=$(dnf repoinfo --json *llvm-snapshots-pgo* | jq -r ".[0].repo_file_path") +distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file +cat /tmp/new_repo_file > $repo_file +dnf -y install \ + clang-${rpm_suffix} \ + clang-${rpm_suffix} \ + clang-libs-${rpm_suffix} \ + clang-resource-filesystem-${rpm_suffix} \ + llvm-${rpm_suffix} \ + llvm-libs-${rpm_suffix} + +mkdir -pv ~/pgo +cd ~/pgo + +configure_build_run + +# Remove packages from that PGO repo and the repo itself +repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-pgo" | awk '{print $1}') +dnf -y remove $repo_pkgs_installed; +dnf copr disable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} + +###################################################################################### +# big-merge +###################################################################################### + +# Install and enable the repository that provides the big-merge LLVM Toolchain +dnf copr enable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} +repo_file=$(dnf repoinfo --json *llvm-snapshots-big-merge* | jq -r ".[0].repo_file_path") +distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file +cat /tmp/new_repo_file > $repo_file +dnf -y install \ + clang-${rpm_suffix} \ + clang-${rpm_suffix} \ + clang-libs-${rpm_suffix} \ + clang-resource-filesystem-${rpm_suffix} \ + llvm-${rpm_suffix} \ + llvm-libs-${rpm_suffix} + +mkdir -pv ~/big-merge +cd ~/big-merge + +configure_build_run +# Remove packages from that big-merge repo and the repo itself +repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-big-merge" | awk '{print $1}') +dnf -y remove $repo_pkgs_installed; +dnf copr disable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} + +###################################################################################### +# system llvm +###################################################################################### + +# Build with regular clang +dnf install -y clang clang-libs clang-resource-filesystem llvm llvm-libs +mkdir -pv ~/system +cd ~/system + +configure_build_run + +system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head -n1) + +/root/test-suite/utils/compare.py \ + --metric compile_time \ + --lhs-name ${system_llvm_release} \ + --rhs-name pgo-${yyyymmdd} \ + ~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true + +/root/test-suite/utils/compare.py \ + --metric compile_time \ + --lhs-name ${system_llvm_release} \ + --rhs-name big-merge-${yyyymmdd} \ + ~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true + +/root/test-suite/utils/compare.py \ + --metric compile_time \ + --lhs-name big-merge \ + --rhs-name pgo-${yyyymmdd} \ + ~/big-merge/results.json vs ~/pgo/results.json > ~/results-big-merge-vs-pgo.txt || true + +bash