From 5d98d296769defdaa371b79ab950262022b54cde Mon Sep 17 00:00:00 2001 From: Felipe Lima Date: Wed, 16 Oct 2024 15:37:48 -0700 Subject: [PATCH] fix(ruby): Enable ruby-sdk to be cross compiled ahead of time (#42) * fix(ruby): Enable rust-sdk to be cross compiled ahead of time * add github action to cross compile gems * remove comment * fix indentation * change working directory for ruby * fix cross-gem agai * yet another fix * another try * yep * manually run cross compile * fix indentation * more fixes * skip tags * adapt build from https://github.com/BoundaryML/baml/blob/canary/.github/workflows/build-ruby-release.reusable.yaml * fix path * Fix https://github.com/openssl/openssl/pull/25367\#issuecomment-2338141250 * chore: add "vendored" feature flag to eppo_core * chore(ruby): build gem for 3.0 version * code review comments * add fake publish step for testing * comment out needs * update cargo lockfile * fix file path * fix version * another try * another try * 20th time is the charm * fix paths * add ls for debugging * final try * move cross_gems and publish to publish.yml --------- Co-authored-by: Oleksii Shmalko --- .github/workflows/ci.yml | 41 ------- .github/workflows/publish.yml | 166 ++++++++++++++++++++++++++- .github/workflows/ruby.yml | 55 +++++++++ ruby-sdk/Cargo.lock | 12 +- ruby-sdk/Gemfile | 16 +-- ruby-sdk/Gemfile.lock | 8 +- ruby-sdk/Rakefile | 15 +-- ruby-sdk/build/x86-64_linux-setup.sh | 17 +++ ruby-sdk/ext/eppo_client/Cargo.toml | 2 +- 9 files changed, 256 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/ruby.yml create mode 100755 ruby-sdk/build/x86-64_linux-setup.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 342ad9b9..3bf28d77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,44 +26,3 @@ jobs: - run: cargo build --verbose --all-targets - run: cargo test --verbose - run: cargo doc --verbose - - - ruby_test: - runs-on: ${{ matrix.os }}-latest - strategy: - fail-fast: false - matrix: - os: [ubuntu, macos] - ruby: [3.0, 3.1, 3.2] - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Ruby & Rust - uses: oxidize-rb/actions/setup-ruby-and-rust@v1 - with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - cargo-cache: true - rubygems: '3.5.11' - - - run: npm ci - - - name: Override eppo_core for testing - run: | - mkdir -p ~/.cargo/ - echo "[patch.crates-io.eppo_core]" >> "${CARGO_HOME:-$HOME/.cargo}/config.toml" - echo "path = '$PWD/eppo_core'" >> "${CARGO_HOME:-$HOME/.cargo}/config.toml" - - - name: Install dependencies - run: bundle install - working-directory: ruby-sdk - - - name: Build - run: bundle exec rake build - working-directory: ruby-sdk - - - name: Run tests - run: npm run with-server test:ruby diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 200b42bd..711d3de0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -55,38 +55,196 @@ jobs: env: CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + cross_gems: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ruby-sdk + strategy: + fail-fast: false + matrix: + _: + - platform: x86_64-linux + # This is necessary because rb-sys-dock depends on manylinux2014, + # which is based on CentOS 7 which is EOL as of July 2024 Once + # rake-compiler-dock switches to manylinux_2_28 and rb-sys-dock + # picks that up, we can pick their updates up and then drop this. + # See https://github.com/oxidize-rb/rb-sys/issues/402 and + # https://github.com/rake-compiler/rake-compiler-dock/issues/122 + # for more details. + rb-sys-dock-setup: ./build/x86-64_linux-setup.sh + - platform: x86_64-linux-musl + - platform: aarch64-linux + - platform: aarch64-linux-musl + - platform: arm-linux + - platform: arm64-darwin + # - platform: x64-mingw32 + # - platform: x86-mingw-ucrt + steps: + - uses: actions/checkout@v4 + + - uses: oxidize-rb/actions/setup-ruby-and-rust@v1 + with: + ruby-version: "3.3" + bundler-cache: true + cargo-cache: true + cargo-vendor: true + cache-version: v2-${{ matrix._.platform }} + + - name: Set vars + id: vars + run: | + echo "rb-sys-version=$(bundle exec ruby -rrb_sys -e 'puts RbSys::VERSION')" >> $GITHUB_OUTPUT + + - uses: "ruby/setup-ruby@v1" + with: + ruby-version: "3.2" + bundler-cache: true + + - name: Configure environment + shell: bash + id: configure + run: | + : Configure environment + echo "RB_SYS_DOCK_UID=$(id -u)" >> $GITHUB_ENV + echo "RB_SYS_DOCK_GID=$(id -g)" >> $GITHUB_ENV + rb_sys_version="$((grep rb_sys Gemfile.lock | head -n 1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') || (gem info rb_sys --remote | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') || echo "latest")" + rb_sys_dock_cache_dir="$HOME/.cache/rb-sys-$rb_sys_version" + mkdir -p "$rb_sys_dock_cache_dir" + echo "RB_SYS_DOCK_CACHE_DIR=$rb_sys_dock_cache_dir" >> $GITHUB_ENV + echo "rb_sys_version=$rb_sys_version" >> $GITHUB_OUTPUT + + - name: Setup caching + uses: actions/cache@v4 + with: + path: | + ${{ env.RB_SYS_DOCK_CACHE_DIR }} + ruby-sdk/tmp/rb-sys-dock/${{ matrix._.platform }}/target + key: rb-sys-dock-v0-${{ matrix._.platform }}-${{ hashFiles('**/Gemfile.lock', '**/Cargo.lock') }} + save-always: true + restore-keys: | + rb-sys-dock-v0-${{ matrix._.platform }}- + + - name: Install cargo-cache + uses: oxidize-rb/actions/cargo-binstall@v1 + id: install-cargo-cache + with: + crate: cargo-cache + version: 0.8.3 + strategies: quick-install + + - name: Clean the cargo cache + uses: oxidize-rb/actions/post-run@v1 + with: + run: cargo-cache --autoclean + cwd: ruby-sdk + always: true + + - name: Setup rb-sys + shell: bash + working-directory: ruby-sdk + run: | + version="${{ steps.configure.outputs.rb_sys_version }}" + echo "Installing rb_sys@$version" + + if [ "$version" = "latest" ]; then + gem install rb_sys + else + gem install rb_sys -v $version + fi + + - name: Build gem + shell: bash + run: | + : Compile gem + echo "Docker Working Directory: $(pwd)" + set -x + + # We can't actually parallelize the Ruby versions because + # they get bundled into the same gem. + # + # However, not parallelizing versions is actually helpful + # because Cargo is able to reuse most of compile work + # between versions. + rb-sys-dock \ + --platform ${{ matrix._.platform }} \ + --mount-toolchains \ + --ruby-versions 3.3,3.2,3.1,3.0 \ + --build \ + -- ${{ matrix._.rb-sys-dock-setup }} + + - name: Set outputs + id: set-outputs + shell: bash + run: | + : Set output + echo "gem-path=ruby-sdk/$(find pkg -name '*-${{ matrix._.platform }}.gem')" >> $GITHUB_OUTPUT + + - name: Upload the cross-compiled gems + uses: actions/upload-artifact@v3 + with: + name: cross-gem + path: ${{ steps.set-outputs.outputs.gem-path }} + publish_ruby: name: Publish to RubyGems runs-on: ubuntu-latest if: ${{ startsWith(github.ref_name, 'ruby-sdk@') }} + needs: cross_gems steps: - uses: actions/checkout@v4 with: submodules: true + - uses: actions/setup-node@v3 with: node-version: '20' + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '3.3' + + - name: Download the cross-compiled gems + uses: actions/download-artifact@v3 + with: + name: cross-gem + - name: Check Cargo.lock # Ensure that Cargo.lock matches Cargo.toml run: cargo update --workspace --locked --verbose working-directory: ruby-sdk + - name: Install dependencies run: bundle install working-directory: ruby-sdk + - name: Build run: bundle exec rake build working-directory: ruby-sdk + + - name: Move the downloaded artifacts + run: | + mv *.gem ruby-sdk/pkg + - name: Publish to RubyGems + working-directory: ruby-sdk/pkg/ + env: + RUBYGEMS_API_KEY: "${{ secrets.RUBYGEMS_API_KEY }}" run: | mkdir -p $HOME/.gem touch $HOME/.gem/credentials chmod 0600 $HOME/.gem/credentials printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials - gem push pkg/eppo-server-sdk-*.gem - env: - RUBYGEMS_API_KEY: "${{ secrets.RUBYGEMS_API_KEY }}" - working-directory: ruby-sdk + ls -l + for i in *.gem; do + if [ -f "$i" ]; then + if ! gem push "$i" >push.out; then + gemerr=$? + sed 's/^/::error:: /' push.out + if ! grep -q "Repushing of gem" push.out; then + exit $gemerr + fi + fi + fi + done diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml new file mode 100644 index 00000000..dacbc569 --- /dev/null +++ b/.github/workflows/ruby.yml @@ -0,0 +1,55 @@ +--- +name: Ruby SDK + +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ruby_test: + runs-on: ${{ matrix.os }}-latest + outputs: + gem-path: ${{ steps.set-outputs.outputs.gem-path }} + strategy: + fail-fast: false + matrix: + os: [ubuntu, macos] + ruby: [3.0, 3.1, 3.2] + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Ruby & Rust + uses: oxidize-rb/actions/setup-ruby-and-rust@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + cargo-cache: true + rubygems: '3.5.11' + + - run: npm ci + + - name: Override eppo_core for testing + run: | + mkdir -p ~/.cargo/ + echo "[patch.crates-io.eppo_core]" >> "${CARGO_HOME:-$HOME/.cargo}/config.toml" + echo "path = '$PWD/eppo_core'" >> "${CARGO_HOME:-$HOME/.cargo}/config.toml" + + - name: Install dependencies + run: bundle install + working-directory: ruby-sdk + + - name: Build + run: bundle exec rake build + working-directory: ruby-sdk + + - name: Run tests + run: npm run with-server test:ruby diff --git a/ruby-sdk/Cargo.lock b/ruby-sdk/Cargo.lock index 61388465..3aae1425 100644 --- a/ruby-sdk/Cargo.lock +++ b/ruby-sdk/Cargo.lock @@ -909,9 +909,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.3.2+3.3.2" +version = "300.3.1+3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" dependencies = [ "cc", ] @@ -1032,18 +1032,18 @@ dependencies = [ [[package]] name = "rb-sys" -version = "0.9.100" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565" +checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1" dependencies = [ "rb-sys-build", ] [[package]] name = "rb-sys-build" -version = "0.9.100" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48" +checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570" dependencies = [ "bindgen", "lazy_static", diff --git a/ruby-sdk/Gemfile b/ruby-sdk/Gemfile index 2e848fef..f7419c75 100644 --- a/ruby-sdk/Gemfile +++ b/ruby-sdk/Gemfile @@ -5,11 +5,13 @@ source "https://rubygems.org" # Specify your gem's dependencies in eppo-server-sdk.gemspec gemspec -gem "rake", "~> 13.0" +group :test do + gem "rspec", "~> 3.0" +end -gem "rake-compiler" -gem "rb_sys", "~> 0.9.63" - -gem "rspec", "~> 3.0" - -gem "rubocop", "~> 1.21" +group :development do + gem "rake", "~> 13.0" + gem "rake-compiler" + gem "rb_sys", "~> 0.9.102" + gem "rubocop", "~> 1.21" +end diff --git a/ruby-sdk/Gemfile.lock b/ruby-sdk/Gemfile.lock index 8f495797..76ec1745 100644 --- a/ruby-sdk/Gemfile.lock +++ b/ruby-sdk/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - eppo-server-sdk (3.2.0) + eppo-server-sdk (3.2.1) GEM remote: https://rubygems.org/ @@ -19,7 +19,7 @@ GEM rake (13.2.1) rake-compiler (1.2.7) rake - rb_sys (0.9.97) + rb_sys (0.9.102) regexp_parser (2.9.2) rexml (3.3.0) strscan @@ -60,9 +60,9 @@ DEPENDENCIES eppo-server-sdk! rake (~> 13.0) rake-compiler - rb_sys (~> 0.9.63) + rb_sys (~> 0.9.102) rspec (~> 3.0) rubocop (~> 1.21) BUNDLED WITH - 2.5.9 + 2.4.4 diff --git a/ruby-sdk/Rakefile b/ruby-sdk/Rakefile index f8b3e7fa..67eaa751 100644 --- a/ruby-sdk/Rakefile +++ b/ruby-sdk/Rakefile @@ -1,18 +1,11 @@ # frozen_string_literal: true require "bundler/gem_tasks" -require "rspec/core/rake_task" require_relative 'lib/eppo_client/version' GEM_NAME = 'eppo-server-sdk' GEM_VERSION = EppoClient::VERSION -RSpec::Core::RakeTask.new(:spec) - -require "rubocop/rake_task" - -RuboCop::RakeTask.new - require "rb_sys/extensiontask" task default: :build @@ -21,6 +14,8 @@ GEMSPEC = Gem::Specification.load("eppo-server-sdk.gemspec") RbSys::ExtensionTask.new("eppo_client", GEMSPEC) do |ext| ext.lib_dir = "lib/eppo_client" + + ext.cross_compile = true end task build: :compile do @@ -43,11 +38,5 @@ task :clean do system 'rm *.gem' end -RSpec::Core::RakeTask.new(:test) do |task| - root_dir = Rake.application.original_dir - task.pattern = "#{root_dir}/spec/*_spec.rb" - task.verbose = false -end - task test: :devinstall task test_refreshed_data: [:devinstall, 'test-data'] diff --git a/ruby-sdk/build/x86-64_linux-setup.sh b/ruby-sdk/build/x86-64_linux-setup.sh new file mode 100755 index 00000000..dd97bb34 --- /dev/null +++ b/ruby-sdk/build/x86-64_linux-setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +## From: +## https://github.com/BoundaryML/baml/blob/canary/engine/language_client_ruby/x86-64_linux-setup.sh +## + +set -euxo pipefail + +# from https://serverfault.com/questions/1161816/mirrorlist-centos-org-no-longer-resolve +sudo sed -i 's/mirror.centos.org/vault.centos.org/g' /etc/yum.repos.d/*.repo +sudo sed -i 's/^#.*baseurl=http/baseurl=http/g' /etc/yum.repos.d/*.repo +sudo sed -i 's/^mirrorlist=http/#mirrorlist=http/g' /etc/yum.repos.d/*.repo + +ls /etc/yum.repos.d/ + +# We need this to build engine/, since it's needed for OpenSSL +sudo yum install -y perl-IPC-Cmd diff --git a/ruby-sdk/ext/eppo_client/Cargo.toml b/ruby-sdk/ext/eppo_client/Cargo.toml index ed961426..1499f9f0 100644 --- a/ruby-sdk/ext/eppo_client/Cargo.toml +++ b/ruby-sdk/ext/eppo_client/Cargo.toml @@ -17,5 +17,5 @@ log = { version = "0.4.21", features = ["kv_serde"] } magnus = { version = "0.6.4" } serde = { version = "1.0.203", features = ["derive"] } serde_magnus = "0.8.1" -rb-sys = "0.9" +rb-sys = "0.9.102" serde_json = "1.0.128"