From ef7c70f1f4ee49fedc2cedb906f4efabcaf9d492 Mon Sep 17 00:00:00 2001 From: Volkan Date: Thu, 2 Mar 2023 13:10:49 +0100 Subject: [PATCH] feat: Introduce a metrics workflow (#786) This commit introduces a workflow to continuously run and collect benchmark data whenever the master branch receives a commit. The collected data is appended onto a JSON file located in the `metrics-data` RuSTy branch which in turn is used by our frontend[1] to display the data graphically to track RuSTys performance over time. [1] https://plc-lang.github.io/metrics/ --- .cargo/config.toml | 2 + .github/workflows/metrics.yml | 32 ++++ .gitignore | 7 +- scripts/build.sh | 51 ++++-- xtask/.gitignore | 1 + xtask/Cargo.lock | 294 ++++++++++++++++++++++++++++++++++ xtask/Cargo.toml | 13 ++ xtask/README.md | 12 ++ xtask/res/ARCH.drawio | 1 + xtask/res/ARCH.svg | 4 + xtask/res/sieve.c | 32 ++++ xtask/res/sieve.st | 40 +++++ xtask/src/main.rs | 34 ++++ xtask/src/metrics/mod.rs | 113 +++++++++++++ xtask/src/metrics/oscat.rs | 42 +++++ xtask/src/metrics/sieve.rs | 30 ++++ xtask/src/traits.rs | 40 +++++ 17 files changed, 731 insertions(+), 17 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 .github/workflows/metrics.yml create mode 100644 xtask/.gitignore create mode 100644 xtask/Cargo.lock create mode 100644 xtask/Cargo.toml create mode 100644 xtask/README.md create mode 100644 xtask/res/ARCH.drawio create mode 100644 xtask/res/ARCH.svg create mode 100644 xtask/res/sieve.c create mode 100644 xtask/res/sieve.st create mode 100644 xtask/src/main.rs create mode 100644 xtask/src/metrics/mod.rs create mode 100644 xtask/src/metrics/oscat.rs create mode 100644 xtask/src/metrics/sieve.rs create mode 100644 xtask/src/traits.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000..d8c2032b49 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --manifest-path ./xtask/Cargo.toml --" diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml new file mode 100644 index 0000000000..3bfe11537f --- /dev/null +++ b/.github/workflows/metrics.yml @@ -0,0 +1,32 @@ +name: Metrics + +on: + push: + branches: + - master + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + IMAGE_NAME: rust-llvm + IMAGE_VERSION: latest + +jobs: + metrics: + name: Metrics + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: master + + - name: Collect metrics + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$IMAGE_VERSION + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + docker pull $IMAGE_ID + ./scripts/build.sh --metrics --container --container-name=$IMAGE_ID --ci diff --git a/.gitignore b/.gitignore index 95c5ad2ab2..161369ec3a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,4 @@ *.o *.bc *.a -*.elf - -# Introduced with issue #679: -# We're assuming config.toml to be a local config file for now. -# This entry has to be removed if a global config file is needed in the near future. -.cargo/config.toml +*.elf \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index 24afa38deb..6780310497 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -5,6 +5,7 @@ vendor=0 offline=0 check=0 check_style=0 +metrics=0 build=0 doc=0 test=0 @@ -14,6 +15,7 @@ debug=0 container=0 assume_linux=0 junit=0 +ci=0 CONTAINER_NAME='rust-llvm' @@ -78,13 +80,18 @@ function run_build() { function run_check() { CARGO_OPTIONS=$(set_cargo_options) - log "Running cargo check" + log "Running cargo check" cargo check $CARGO_OPTIONS } +function run_metrics() { + log "Running cargo xtask metrics" + cargo xtask metrics +} + function run_doc() { CARGO_OPTIONS=$(set_cargo_options) - log "Running cargo doc" + log "Running cargo doc" cargo doc $CARGO_OPTIONS log "Building book" cd book && mdbook build && mdbook test @@ -92,9 +99,9 @@ function run_doc() { function run_check_style() { CARGO_OPTIONS=$(set_cargo_options) - log "Running cargo clippy" + log "Running cargo clippy" cargo clippy $CARGO_OPTIONS -- -Dwarnings - log "Running cargo fmt check" + log "Running cargo fmt check" cargo fmt -- --check } @@ -136,6 +143,8 @@ function set_offline() { function run_in_container() { container_engine=$(get_container_engine) params="" + options="" + if [[ $offline -ne 0 ]]; then params="$params --offline" fi @@ -148,6 +157,9 @@ function run_in_container() { if [[ $check_style -ne 0 ]]; then params="$params --check-style" fi + if [[ $metrics -ne 0 ]]; then + params="$params --metrics" + fi if [[ $build -ne 0 ]]; then params="$params --build" fi @@ -166,6 +178,9 @@ function run_in_container() { if [[ $doc -ne 0 ]]; then params="$params --doc" fi + if [[ $ci -ne 0 ]]; then + options="$options --env=CI_RUN=true" + fi volume_target="/build" unameOut="$(uname -s)" @@ -184,7 +199,7 @@ function run_in_container() { build_location=$(sanitize_path "$project_location") log "Sanitized Project location : $project_location" - command_to_run="$container_engine run -v $build_location:$volume_target $CONTAINER_NAME scripts/build.sh $params" + command_to_run="$container_engine run $options -v $build_location:$volume_target $CONTAINER_NAME scripts/build.sh $params" log "Running command : $command_to_run" eval "$command_to_run" } @@ -193,7 +208,7 @@ function run_in_container() { set -o errexit -o pipefail -o noclobber -o nounset OPTIONS=sorbvc -LONGOPTS=sources,offline,release,check,check-style,build,doc,test,junit,verbose,container,linux,container-name:,coverage +LONGOPTS=sources,offline,release,check,check-style,metrics,ci,build,doc,test,junit,verbose,container,linux,container-name:,coverage check_env # -activate quoting/enhanced mode (e.g. by writing out “--options”) @@ -239,6 +254,12 @@ while true; do --check) check=1 ;; + --metrics) + metrics=1 + ;; + --ci) + ci=1 + ;; -b|--build) build=1 ;; @@ -268,6 +289,10 @@ log "Moving to project level directory $project_location" cd "$project_location" +if [[ $ci -ne 0 ]]; then + export CI_RUN=true +fi + if [[ $container -ne 0 ]]; then log "Container Build" run_in_container @@ -288,19 +313,23 @@ if [[ $offline -ne 0 ]]; then fi if [[ $check -ne 0 ]]; then - run_check + run_check fi if [[ $check_style -ne 0 ]]; then - run_check_style + run_check_style +fi + +if [[ $metrics -ne 0 ]]; then + run_metrics fi if [[ $build -ne 0 ]]; then - run_build + run_build fi if [[ $test -ne 0 ]]; then - run_test + run_test fi if [[ $doc -ne 0 ]]; then @@ -308,7 +337,7 @@ if [[ $doc -ne 0 ]]; then fi if [[ $coverage -ne 0 ]]; then - run_coverage + run_coverage fi if [[ -d $project_location/target/ ]]; then diff --git a/xtask/.gitignore b/xtask/.gitignore new file mode 100644 index 0000000000..9f970225ad --- /dev/null +++ b/xtask/.gitignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock new file mode 100644 index 0000000000..ca5f04b535 --- /dev/null +++ b/xtask/Cargo.lock @@ -0,0 +1,294 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ntapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "proc-macro2" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727220a596b4ca0af040a07091e49f5c105ec8f2592674339a5bf35be592f76e" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xshell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962c039b3a7b16cf4e9a4248397c6585c07547412e7d6a6e035389a802dcfe90" +dependencies = [ + "xshell-macros", +] + +[[package]] +name = "xshell-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dbabb1cbd15a1d6d12d9ed6b35cc6777d4af87ab3ba155ea37215f20beab80c" + +[[package]] +name = "xtask" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "sysinfo", + "xshell", +] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000000..374b1e3c09 --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +xshell = "0.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sysinfo = "0.28" +anyhow = "1.0" \ No newline at end of file diff --git a/xtask/README.md b/xtask/README.md new file mode 100644 index 0000000000..0716eac56b --- /dev/null +++ b/xtask/README.md @@ -0,0 +1,12 @@ +# RuSTy xtasks + +This directory contains the following [xtasks](https://github.com/matklad/cargo-xtask): + +- `metrics`: Runs various benchmarks, collects them and (if ran by a CI) appends the collected data to + the [`metrics.json`](https://github.com/PLC-lang/rusty/blob/metrics-data/metrics.json) file. + The given JSON file is then used by our [metrics dashboard](https://plc-lang.github.io/metrics/) to track and display + RuSTys performance over time. Graphically the workflow looks as follows: + +
+ +
\ No newline at end of file diff --git a/xtask/res/ARCH.drawio b/xtask/res/ARCH.drawio new file mode 100644 index 0000000000..e6d511a221 --- /dev/null +++ b/xtask/res/ARCH.drawio @@ -0,0 +1 @@ +3VhNT+MwEP01kXYPVPls02MpBQ6gRRQJcUJO4iYG145sp1+/fu3ESRrSlrJQWvZU+2U8tt/MPI9qOMPp4oqBNLmlEcSGbUYLw7kwbNsyXV/+KGRZIJ5nFUDMUKSNamCMVrBcqdEMRZA3DAWlWKC0CYaUEBiKBgYYo/Om2YTi5q4piGELGIcAt9FHFImkQH27V+PXEMVJubPV7RdfpqA01jfhCYjofA1yRoYzZJSKYjRdDCFW5JW8FOsut3ytDsYgEfsseGCr0bh38fT8FN0nj6tZL3gYnDmFlxnAmb6wPqxYlgzASBKip5SJhMaUADyq0XNGMxJBtY0pZ7XNDaWpBC0JvkAhljq6IBNUQomYYv212FNttPVuGuI0YyHccaEyRwCLodhhZ1cRkKkL6RQKtpTrGMRAoFnzHEDnUFzZ1TTLgWb6A6yXyb1G+wNDcQwZ30j/DQhkWTUoAxjFRI5DyRBkEphBJpDM24H+MEVRVEQHcrQCQe5PxSeliIj8Rt654V1U9CsHcGFsKCq9uE7l9cBsT6s2u9q72bHdwtGysd3e5GvPd+omtduKVu31rN/0QCcTLnPibfCq830inq1w3mU8yaUqVwKuQvQ2sHXVqHjOEyTgOAV5cs+llm6qkA0hatXIVtb9Jjt2T8/ntaxZZYCTNUnzzANVgffDpcbeU2rco0pNOzfvM/KfyIz3jsyYVrd3CKHpHktneifxXEMSDVRrJaeEElggl0jdJXd5hBrbIn3fU2N2Kyq30i8KZZWZj5S9TrDkyu5ieYHzQD4E3ViNfl0hcZ0F0mYQCkTJ76O/EKf3RLgtZkOZCepRXQjAX9XjWjJ9ZO7ck+Ou/8OfV3fP0vc/WfqbNd42m/H0fLfpoji/XnUAqW+LypBiLJ9JpSoigfkVeYbV/K20ABKpk6ap1GVlTolqRLtgqvKeBDytIvXTO4D+zqI8ky1At2wB/jUhyoL+rkfebwVea1znhVPSChtPQKqG4RIjWbvMeV/ugqLKb4IKAOFrnNf+n0xIN1DjvChzy/sijez7nZ7XKCvHb8ukb3e8tk5a1qGE0jqKUq41USEGnKOw0Ufl9EvtE22jHF4z+0LNtfxvEt3PBaxdIpdMahzMZW9Xp3UHYnh6fZZrVRn/Xrfgeh/m1FDklH8yFiJV/1XrjP4C \ No newline at end of file diff --git a/xtask/res/ARCH.svg b/xtask/res/ARCH.svg new file mode 100644 index 0000000000..95cd93475b --- /dev/null +++ b/xtask/res/ARCH.svg @@ -0,0 +1,4 @@ + + + +
Triggers
Triggers
Push to master
Push to master
Runs
Runs
Metrics Workflow
(GitHub Action)
Metrics Workflow...
cargo xtask metrics
cargo xtask metrics
Collects the results
and appends onto 
Collects the results...
metrics.json
metrics.json
Frontend
(GitHub Page)
Frontend...Text is not SVG - cannot display \ No newline at end of file diff --git a/xtask/res/sieve.c b/xtask/res/sieve.c new file mode 100644 index 0000000000..0b814d8e22 --- /dev/null +++ b/xtask/res/sieve.c @@ -0,0 +1,32 @@ +#include +#include + +unsigned long long SIEVE_SIZE = 500000000; +bool FLAGS[500000000] = {0}; + +void sieve() { + unsigned long long i, j = 0; + unsigned long long primes = 0; + + for (i = 2; i < SIEVE_SIZE; ++i) { + if (FLAGS[i] == 0) { + for (j = i * i; j < SIEVE_SIZE; j += i) { + if (j < SIEVE_SIZE) { + FLAGS[j] = 1; + } + } + } + } + + for (i = 2; i < SIEVE_SIZE; ++i) { + if (FLAGS[i] == 0) { + primes += 1; + } + } + + printf("Primes found : %lld", primes); +} + +int main(int argc, char **argv) { + sieve(); +} diff --git a/xtask/res/sieve.st b/xtask/res/sieve.st new file mode 100644 index 0000000000..56b4c982b9 --- /dev/null +++ b/xtask/res/sieve.st @@ -0,0 +1,40 @@ +VAR_GLOBAL + flags : ARRAY[1..500_000_000] OF BOOL; + sieve_size : ULINT := 500_000_000; +END_VAR + +// en.wikipedia.org/wiki/Sieve_of_Eratosthenes +FUNCTION sieve : DINT + VAR + i, j : ULINT; + primes : ULINT; + END_VAR + + FOR i := 2 TO sieve_size DO + IF flags[i] = 0 THEN + FOR j := i * i TO sieve_size BY i DO + IF j < sieve_size THEN + flags[j] := 1; + END_IF; + END_FOR; + END_IF; + END_FOR; + + FOR i := 2 TO sieve_size DO + IF flags[i] = 0 THEN + primes := primes + 1; + END_IF; + END_FOR; + + printf('Primes found: %lld (correct: %d)$N', primes, (primes - 1) = 26_355_867); +END_FUNCTION + +FUNCTION main : DINT + sieve(); +END_FUNCTION + +{external} +FUNCTION printf : DINT + VAR_INPUT {ref} format : STRING; END_VAR + VAR_INPUT args : ...; END_VAR +END_FUNCTION \ No newline at end of file diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000000..66135b2651 --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,34 @@ +use std::str::FromStr; + +use anyhow::anyhow; +use metrics::Metrics; +use xshell::Shell; + +mod metrics; +mod traits; + +enum Task { + Metrics, +} + +impl FromStr for Task { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "metrics" => Ok(Task::Metrics), + _ => Err(anyhow!("Unrecognized task '{s}'")), + } + } +} + +fn main() -> anyhow::Result<()> { + let sh = Shell::new()?; + let Some(arg) = std::env::args().nth(1) else { anyhow::bail!("No argument specified, try `xtask `") }; + + match Task::from_str(&arg)? { + Task::Metrics => Metrics::new(&sh)?.execute(&sh)?, + } + + Ok(()) +} diff --git a/xtask/src/metrics/mod.rs b/xtask/src/metrics/mod.rs new file mode 100644 index 0000000000..a7a6947fd2 --- /dev/null +++ b/xtask/src/metrics/mod.rs @@ -0,0 +1,113 @@ +use crate::metrics::{oscat::Oscat, sieve::Sieve}; +use crate::traits::Task; +use serde::Serialize; +use std::{ + collections::BTreeMap, + fs, + io::Write, + time::{SystemTime, UNIX_EPOCH}, +}; +use sysinfo::{CpuExt, System, SystemExt}; +use xshell::{cmd, Shell}; + +mod oscat; +mod sieve; + +#[derive(Serialize)] +pub struct Metrics { + /// Host information, see [`Host`]. + host: Host, + + /// Unix timestamp of when this xtask was called. + timestamp: u64, + + /// Commit hash on which the benchmark ran. + commit: String, + + /// Collected benchmarks, where the first tuple element describes the benchmark and the second + /// element is its raw wall-time value in milliseconds. + /// For example one such element might be `("oscat/aggressive", 8000)`, indicating an oscat build + /// with the `aggressive` optimization flag took 8000ms. + pub(crate) metrics: BTreeMap, +} + +#[derive(Serialize)] +struct Host { + os: String, + cpu: String, + mem: u64, +} + +impl Host { + fn new() -> Self { + let sys = System::new_all(); + + let os = sys.long_os_version().unwrap_or("n/a".to_string()); + let cpu = sys.global_cpu_info().brand().to_owned(); + let mem = sys.total_memory() / 1024; + + Self { os, cpu, mem } + } +} + +impl Metrics { + pub fn new(sh: &Shell) -> anyhow::Result { + // Needed because of "fatal: detected deubious ownership in repository at '/build'" error + cmd!(sh, "git config --global --add safe.directory /build").run()?; + + let host = Host::new(); + let commit = cmd!(sh, "git rev-parse HEAD").read()?; + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + let metrics = BTreeMap::new(); + + Ok(Self { host, timestamp, commit, metrics }) + } + + /// Starts the execution of various [`Task`]s, collecting bechmark data. + /// Additionally the data is pushed onto the `metrics` branch on rusty if the task + /// is executed within a CI enviroment, i.e. specified by the `CI_RUN` environment flag. + pub fn execute(&mut self, sh: &Shell) -> anyhow::Result<()> { + // Remove and re-create the folder in case of previous dry runs + sh.remove_path("./benchmark")?; + sh.create_dir("./benchmark")?; + + let tasks: Vec> = vec![Box::new(Oscat), Box::new(Sieve)]; + for task in tasks { + task.prepare(sh)?; + task.execute(sh, self)?; + } + + // Only commit and push IF we executed the task within a CI job + if std::env::var("CI_RUN").is_ok() { + self.finalize(sh)?; + } + + println!("{}", serde_json::to_string_pretty(self)?); + Ok(()) + } + + /// Appends the collected data to a JSON file, commiting and pushing it onto + /// the `metrics` branch hosted on RuSTy. Whoever the author of the last commit + /// on the RuSTy master branch is thereby also the author of this commit. + pub fn finalize(&self, sh: &Shell) -> anyhow::Result<()> { + let branch = "metrics-data"; + let filename = "metrics.json"; + let message = format!("Update {}", self.commit); + let user_name = cmd!(sh, "git log -1 --pretty=format:'%an'").read()?; + let user_mail = cmd!(sh, "git log -1 --pretty=format:'%ae'").read()?; + + cmd!(sh, "git pull").run()?; + cmd!(sh, "git config user.name \"{user_name}\"").run()?; + cmd!(sh, "git config user.email \"{user_mail}\"").run()?; + cmd!(sh, "git checkout {branch}").run()?; + + let mut file = fs::File::options().create(true).append(true).open(filename)?; + writeln!(file, "{}", serde_json::to_string(self)?)?; + + cmd!(sh, "git add {filename}").run()?; + cmd!(sh, "git commit -m {message}").run()?; + cmd!(sh, "git push origin {branch}").run()?; + + Ok(()) + } +} diff --git a/xtask/src/metrics/oscat.rs b/xtask/src/metrics/oscat.rs new file mode 100644 index 0000000000..095305fc89 --- /dev/null +++ b/xtask/src/metrics/oscat.rs @@ -0,0 +1,42 @@ +use crate::traits::{Benchmark, Task}; +use xshell::{cmd, Shell}; + +pub struct Oscat; +impl Task for Oscat { + fn prepare(&self, sh: &Shell) -> anyhow::Result<()> { + cmd!(sh, "git clone https://github.com/plc-lang/oscat ./benchmark/oscat").run()?; + cmd!(sh, "git clone https://github.com/plc-lang/standardfunctions ./benchmark/oscat/sf").run()?; + + cmd!(sh, "cargo b --release").run()?; + sh.copy_file("./target/release/rustyc", "./benchmark/oscat")?; + + sh.create_dir("./benchmark/oscat/lib").unwrap(); + sh.create_dir("./benchmark/oscat/include").unwrap(); + + let standardfunctions = sh.push_dir("./benchmark/oscat/sf"); + cmd!(sh, "cargo b --release").run()?; + std::mem::drop(standardfunctions); + + let oscat = sh.push_dir("./benchmark/oscat"); + sh.copy_file("sf/target/release/libiec61131std.so", "lib").unwrap(); + sh.read_dir("sf/iec61131-st/").unwrap().iter().for_each(|f| sh.copy_file(f, "include").unwrap()); + std::mem::drop(oscat); + + Ok(()) + } + + fn execute(&self, sh: &Shell, metrics: &mut super::Metrics) -> anyhow::Result<()> { + let _oscat = sh.push_dir("./benchmark/oscat"); + + for flag in ["none", "less", "default", "aggressive"] { + cmd!(sh, "./rustyc -O{flag} build").ignore_stderr().benchmark(metrics, "oscat", flag)?; + } + + cmd!(sh, "./rustyc check oscat.st") + .ignore_status() + .ignore_stderr() + .benchmark(metrics, "check", "oscat")?; + + Ok(()) + } +} diff --git a/xtask/src/metrics/sieve.rs b/xtask/src/metrics/sieve.rs new file mode 100644 index 0000000000..69c12a5a45 --- /dev/null +++ b/xtask/src/metrics/sieve.rs @@ -0,0 +1,30 @@ +use crate::traits::{Benchmark, Task}; +use xshell::cmd; + +pub struct Sieve; +impl Task for Sieve { + fn prepare(&self, sh: &xshell::Shell) -> anyhow::Result<()> { + cmd!(sh, "cargo b --release").run()?; + sh.copy_file("./target/release/rustyc", "./benchmark")?; + sh.copy_file("./xtask/res/sieve.st", "./benchmark")?; + sh.copy_file("./xtask/res/sieve.c", "./benchmark")?; + + Ok(()) + } + + fn execute(&self, sh: &xshell::Shell, metrics: &mut super::Metrics) -> anyhow::Result<()> { + let _path = sh.push_dir("./benchmark"); + + for flag in ["none", "less", "default", "aggressive"] { + cmd!(sh, "./rustyc --linker=clang -O{flag} sieve.st").run()?; + cmd!(sh, "./sieve").ignore_status().benchmark(metrics, "sieve-st", flag)?; + } + + for flag in ["0", "1", "2", "3"] { + cmd!(sh, "gcc -O{flag} sieve.c").run()?; + cmd!(sh, "./a.out").ignore_status().benchmark(metrics, "sieve-c", flag)?; + } + + Ok(()) + } +} diff --git a/xtask/src/traits.rs b/xtask/src/traits.rs new file mode 100644 index 0000000000..b93cec6833 --- /dev/null +++ b/xtask/src/traits.rs @@ -0,0 +1,40 @@ +use std::time::Instant; + +use xshell::{Cmd, Shell}; + +use super::Metrics; + +pub trait Task { + /// Prepares its environment to execute command(s). + fn prepare(&self, sh: &Shell) -> anyhow::Result<()>; + + /// Executes command(s), typically benchmarking it along the way. + fn execute(&self, sh: &Shell, metrics: &mut Metrics) -> anyhow::Result<()>; +} + +/// Trait Extension for the [`xshell::Cmd`] struct. +pub trait Benchmark { + const ITERATIONS_PER_BENCHMARK: u64 = 3; + + /// Benchmarks a command specified by the [`self`] argument measuring its wall-time, + /// collecting and inserting the data into the [`Metrics`] struct. The `name` thereby + /// specifies the to be benchmarked task (e.g. `rustyc`) whereas the `desc` argument + /// describes how the task ran (e.g. with the `-Oaggressive` flag). + fn benchmark(&self, metrics: &mut Metrics, name: &str, desc: &str) -> anyhow::Result<()>; +} + +impl<'sh> Benchmark for Cmd<'sh> { + fn benchmark(&self, metrics: &mut Metrics, name: &str, desc: &str) -> anyhow::Result<()> { + let mut elapsed_sum = 0; + for _ in 0..Self::ITERATIONS_PER_BENCHMARK { + let now = Instant::now(); + self.run()?; + let elapsed = now.elapsed(); + + elapsed_sum += elapsed.as_millis() as u64; + } + + metrics.metrics.insert(format!("{name}/{desc}"), elapsed_sum / Self::ITERATIONS_PER_BENCHMARK); + Ok(()) + } +}