compare the system llvm against "big-merge" and "pgo"

scripts/perf/Containerfile

@@ -0,0 +1,44 @@
+FROM fedora:41
+LABEL description="Test compilers with llvm-test-suite"
+
+USER root
+WORKDIR /root
+
+# Install deps to run test-suite
+RUN dnf install -y \
+        clang \
+        cmake \
+        coreutils \
+        envsubst \
+        fedora-packager \
+        git \
+        jq \
+        jq \
+        ninja-build \
+        perf \
+        python3-lit \
+        python3-pandas \
+        python3-pip \
+        python3-scipy \
+        python3-setuptools \
+        python3-virtualenv \
+        tcl \
+        tcl-devel \
+        tcl-tclreadline \
+        tcl-tclxml \
+        tcl-tclxml-devel \
+        tcl-thread-devel \
+        tcl-zlib \
+        which
+
+# Clone test suite (in correct version for installed clang version)
+# See https://llvm.org/docs/TestSuiteGuide.html
+# RUN export VERSION=`clang --version | grep -ioP 'clang version\s\K[0-9\.]+'` \
+#     && git clone --depth=1 --branch llvmorg-${VERSION} https://github.com/llvm/llvm-test-suite.git test-suite
+RUN git clone --depth=1 https://github.com/llvm/llvm-test-suite.git test-suite
+
+RUN dnf install -y 'dnf-command(copr)'
+
+COPY entrypoint.sh /root/entrypoint.sh
+USER root
+ENTRYPOINT [ "/root/entrypoint.sh" ]

scripts/perf/Makefile

@@ -0,0 +1,10 @@
+.PHONY: all
+all: setup run
+
+.PHONY: setup
+setup:
+	podman build -t evaluation .
+
+.PHONY: run
+run:
+	podman run -it --rm evaluation

scripts/perf/README.md

@@ -0,0 +1,27 @@
+# README
+
+This container setup allows you to compare the system llvm against "big-merge" and "pgo".
+
+# How to
+
+Just run `make` to build and run the container image. It takes a long time to complete.
+
+Then you'll be promted to a terminal in the container where you'll find these files:
+
+```
+~/results-system-vs-pgo.txt
+~/results-system-vs-big-merge.txt
+~/results-big-merge-vs-pgo.txt
+```
+
+The names speak for themselves.
+
+## How to change to OS
+
+If you want to change the version of the operating system, go to `Containerfile` and change the line that looks like this: `FROM fedora:40`. Change it to `FROM fedora:41` or something else.
+
+Then run `make` again.
+
+## How to change the date for which to compare results?
+
+Go to `entrypoint.sh` and change the line that defines `yyyymmdd` to the year-month-date of your liking.

scripts/perf/entrypoint.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/bash
+
+set -x
+set -e
+
+function configure_build_run {
+    # Configure the test suite
+    cmake \
+        -DCMAKE_GENERATOR=Ninja \
+        -DCMAKE_C_COMPILER=/usr/bin/clang \
+        -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \
+        -DTEST_SUITE_BENCHMARKING_ONLY=ON \
+        -DTEST_SUITE_COLLECT_STATS=ON \
+        -DTEST_SUITE_USE_PERF=OFF \
+        -DTEST_SUITE_SUBDIRS=CTMark \
+        -DTEST_SUITE_RUN_BENCHMARKS=OFF \
+        -C~/test-suite/cmake/caches/O3.cmake \
+        ~/test-suite
+
+    # Build the test-suite
+    ninja -j1
+
+    # Run the tests with lit:
+    lit -v -o results.json . || true
+}
+
+# Query version information for given day
+yyyymmdd=20240911
+git_rev=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-git-revision-${yyyymmdd}.txt)
+git_rev_short="${git_rev:0:14}"
+llvm_release=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-release-${yyyymmdd}.txt)
+rpm_suffix="${llvm_release}~pre${yyyymmdd}.g${git_rev_short}"
+
+echo "git_rev=$git_rev"
+echo "git_rev_short=$git_rev_short"
+echo "llvm_release=$llvm_release"
+echo "rpm_suffix=$rpm_suffix"
+
+######################################################################################
+# PGO
+######################################################################################
+
+# Install and enable the repository that provides the PGO LLVM Toolchain
+# See https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo
+dnf copr enable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd}
+repo_file=$(dnf repoinfo --json *llvm-snapshots-pgo* | jq -r ".[0].repo_file_path")
+distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file
+cat /tmp/new_repo_file > $repo_file
+dnf -y install \
+    clang-${rpm_suffix} \
+    clang-${rpm_suffix} \
+    clang-libs-${rpm_suffix} \
+    clang-resource-filesystem-${rpm_suffix} \
+    llvm-${rpm_suffix} \
+    llvm-libs-${rpm_suffix}
+
+mkdir -pv ~/pgo
+cd ~/pgo
+
+configure_build_run
+
+# Remove packages from that PGO repo and the repo itself
+repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-pgo" | awk '{print $1}')
+dnf -y remove $repo_pkgs_installed;
+dnf copr disable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd}
+
+######################################################################################
+# big-merge
+######################################################################################
+
+# Install and enable the repository that provides the big-merge LLVM Toolchain
+dnf copr enable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd}
+repo_file=$(dnf repoinfo --json *llvm-snapshots-big-merge* | jq -r ".[0].repo_file_path")
+distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file
+cat /tmp/new_repo_file > $repo_file
+dnf -y install \
+    clang-${rpm_suffix} \
+    clang-${rpm_suffix} \
+    clang-libs-${rpm_suffix} \
+    clang-resource-filesystem-${rpm_suffix} \
+    llvm-${rpm_suffix} \
+    llvm-libs-${rpm_suffix}
+
+mkdir -pv ~/big-merge
+cd ~/big-merge
+
+configure_build_run
+# Remove packages from that big-merge repo and the repo itself
+repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-big-merge" | awk '{print $1}')
+dnf -y remove $repo_pkgs_installed;
+dnf copr disable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd}
+
+######################################################################################
+# system llvm
+######################################################################################
+
+# Build with regular clang
+dnf install -y clang clang-libs clang-resource-filesystem llvm llvm-libs
+mkdir -pv ~/system
+cd ~/system
+
+configure_build_run
+
+system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head -n1)
+
+/root/test-suite/utils/compare.py \
+    --metric compile_time \
+    --lhs-name ${system_llvm_release} \
+    --rhs-name pgo-${yyyymmdd} \
+    ~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true
+
+/root/test-suite/utils/compare.py \
+    --metric compile_time \
+    --lhs-name ${system_llvm_release} \
+    --rhs-name big-merge-${yyyymmdd} \
+    ~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true
+
+/root/test-suite/utils/compare.py \
+    --metric compile_time \
+    --lhs-name big-merge \
+    --rhs-name pgo-${yyyymmdd} \
+    ~/big-merge/results.json vs ~/pgo/results.json > ~/results-big-merge-vs-pgo.txt || true
+
+bash