From 686f78657495ebe8a3593f752e5a1ade340e3c51 Mon Sep 17 00:00:00 2001 From: Jeff Dickey <216188+jdx@users.noreply.github.com> Date: Wed, 6 Dec 2023 02:01:09 -0600 Subject: [PATCH] added building docker images for CI --- .github/workflows/docker.yml | 63 ++++++++++++++++++++++++ .github/workflows/rtx.yml | 25 ++++------ packaging/github-actions/Dockerfile | 75 +++++++++++++++++++++++++++++ scripts/build-tarball.sh | 6 +++ scripts/release-npm.sh | 2 + scripts/release.sh | 9 ++-- scripts/render-homebrew.sh | 2 + scripts/render-install.sh | 2 + src/cli/version.rs | 19 +++----- src/file.rs | 7 ++- src/http.rs | 31 ++++++++---- src/plugins/core/bun.rs | 11 ++--- src/plugins/core/deno.rs | 11 ++--- src/plugins/core/python.rs | 4 +- src/plugins/core/ruby.rs | 15 ++---- 15 files changed, 209 insertions(+), 73 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 packaging/github-actions/Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..7ae2b38fcf --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,63 @@ +name: docker + +on: + push: + tags: ["v*"] + branches: ['docker-release'] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + docker: + name: docker-${{ matrix.flavor }} + strategy: + fail-fast: false + matrix: + flavor: + - alpine + - deb + - github-actions + - rpm + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ghcr.io/jdx/rtx:${{ matrix.flavor }} + labels: ${{ steps.meta.outputs.labels }} + file: packaging/${{ matrix.flavor }}/Dockerfile + test: + runs-on: ubuntu-22.04 + container: ghcr.io/jdx/rtx:github-actions + timeout-minutes: 10 + steps: + - run: node -v + - run: cargo -V + - name: Checkout + uses: actions/checkout@v4 + - name: Run cargo nextest + run: cargo nextest run --all-features + env: + RUST_BACKTRACE: "1" diff --git a/.github/workflows/rtx.yml b/.github/workflows/rtx.yml index 0f22f02a14..175b47b6d4 100644 --- a/.github/workflows/rtx.yml +++ b/.github/workflows/rtx.yml @@ -16,6 +16,7 @@ env: jobs: unit: runs-on: ubuntu-22.04 + container: ghcr.io/jdx/rtx:github-actions timeout-minutes: 10 steps: - name: Checkout @@ -24,11 +25,6 @@ jobs: uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - - uses: taiki-e/install-action@v2 - with: - tool: nextest,just,cargo-deny,cargo-msrv,cargo-machete - - name: Install direnv/shfmt - run: sudo apt-get update; sudo apt-get install direnv shfmt - name: Run cargo nextest run: cargo nextest run --all-features env: @@ -40,6 +36,7 @@ jobs: coverage: name: coverage-${{matrix.tranche}} + container: ghcr.io/jdx/rtx:github-actions runs-on: ubuntu-latest timeout-minutes: 30 strategy: @@ -49,18 +46,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - run: rustup toolchain install nightly --component llvm-tools-preview - name: Rust Cache uses: Swatinem/rust-cache@v2 with: shared-key: coverage save-if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - - uses: taiki-e/install-action@cargo-llvm-cov - - name: Install zsh/fish/direnv - run: sudo apt-get update; sudo apt-get install zsh fish direnv shfmt - - run: npm i -g markdown-magic - - name: Install just - uses: taiki-e/install-action@just - name: Run tests with coverage uses: nick-fields/retry@v2 env: @@ -69,6 +59,7 @@ jobs: RTX_GITHUB_BOT_TOKEN: ${{ secrets.RTX_GITHUB_BOT_TOKEN }} TEST_TRANCHE: ${{matrix.tranche}} TEST_TRANCHE_COUNT: 4 + RTX_DEBUG: "1" with: timeout_minutes: 30 max_attempts: 2 @@ -89,6 +80,8 @@ jobs: target: - aarch64-unknown-linux-gnu - x86_64-unknown-linux-gnu + - arm-unknown-linux-musleabihf + - armv7-unknown-linux-gnueabihf steps: - uses: actions/checkout@v4 - name: Rust Cache @@ -142,6 +135,7 @@ jobs: if-no-files-found: error e2e-linux: runs-on: ubuntu-22.04 + container: ghcr.io/jdx/rtx:github-actions needs: [build-linux] timeout-minutes: 30 if: github.event_name != 'pull_request' @@ -168,7 +162,7 @@ jobs: runs-on: ubuntu-22.04 needs: [build-linux] timeout-minutes: 10 - container: jdxcode/rtx:rpm + container: ghcr.io/jdx/rtx:rpm if: github.event_name != 'pull_request' steps: - uses: actions/checkout@v4 @@ -191,7 +185,7 @@ jobs: if-no-files-found: error deb: runs-on: ubuntu-22.04 - container: jdxcode/rtx:deb + container: ghcr.io/jdx/rtx:deb timeout-minutes: 10 if: github.event_name != 'pull_request' needs: [build-linux] @@ -217,6 +211,7 @@ jobs: release: runs-on: ubuntu-22.04 if: startsWith(github.event.ref, 'refs/tags/v') + container: ghcr.io/jdx/rtx:github-actions timeout-minutes: 10 permissions: contents: write @@ -305,7 +300,7 @@ jobs: formula: rtx bump-alpine: runs-on: ubuntu-22.04 - container: jdxcode/rtx:alpine + container: ghcr.io/jdx/rtx:alpine timeout-minutes: 30 needs: [release] steps: diff --git a/packaging/github-actions/Dockerfile b/packaging/github-actions/Dockerfile new file mode 100644 index 0000000000..acd21c9b68 --- /dev/null +++ b/packaging/github-actions/Dockerfile @@ -0,0 +1,75 @@ +FROM ubuntu +LABEL maintainer="jdx" + +ENV PATH="/root/.cargo/bin:${PATH}" +ENV CARGO_HOME="/root/.cargo" +ENV RUSTUP_HOME="/root/.rustup" +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y \ + autoconf \ + bash \ + build-essential \ + ca-certificates \ + curl \ + direnv \ + fd-find \ + fish \ + git \ + gnupg \ + libbz2-dev \ + libdb-dev \ + libffi-dev \ + libgdbm-dev \ + libgdbm6 \ + libgmp-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libyaml-dev \ + patch \ + pkg-config \ + shellcheck \ + shfmt \ + sudo \ + tk-dev \ + uuid-dev \ + xz-utils \ + zlib1g-dev \ + zsh \ + && ln -s /usr/bin/{fdfind,fd} \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list \ + && apt-get update && apt-get install -y nodejs \ + && node -v \ + && npm i -g markdown-magic \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash \ + && rustup install stable && rustup default stable \ + && rustup toolchain install nightly --component llvm-tools-preview \ + && cargo install \ + cargo-msrv \ + && cargo binstall -y \ + cargo-deny \ + cargo-llvm-cov \ + cargo-machete \ + cargo-nextest \ + cross \ + just \ + zipsign \ + && apt-get clean \ + && rustc -vV \ + && cargo -V \ + && node -v \ + && npm -v \ + && just --version diff --git a/scripts/build-tarball.sh b/scripts/build-tarball.sh index d05b7982a1..ef7c5d0ab9 100755 --- a/scripts/build-tarball.sh +++ b/scripts/build-tarball.sh @@ -45,6 +45,12 @@ get_arch() { aarch64-*) echo "arm64" ;; + arm-*) + echo "armv6" + ;; + armv7-*) + echo "armv7" + ;; x86_64-*) echo "x64" ;; diff --git a/scripts/release-npm.sh b/scripts/release-npm.sh index 849629248f..01152814f8 100755 --- a/scripts/release-npm.sh +++ b/scripts/release-npm.sh @@ -23,6 +23,8 @@ dist_tag="$(dist_tag_from_version "$RTX_VERSION")" platforms=( linux-x64 linux-arm64 + linux-armv6 + linux-armv7 macos-x64 macos-arm64 ) diff --git a/scripts/release.sh b/scripts/release.sh index 2cf0630e19..c9bbdaa33b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -10,12 +10,11 @@ export RTX_VERSION RELEASE_DIR rm -rf "${RELEASE_DIR:?}/$RTX_VERSION" mkdir -p "$RELEASE_DIR/$RTX_VERSION" -#cp artifacts/tarball-x86_64-pc-windows-gnu/*.zip "$RELEASE_DIR/$RTX_VERSION" -#cp artifacts/tarball-x86_64-pc-windows-gnu/*.zip "$RELEASE_DIR/rtx-latest-windows.zip" - targets=( x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu + arm-unknown-linux-gnueabihf + armv7-unknown-linux-gnueabihf x86_64-apple-darwin aarch64-apple-darwin ) @@ -27,6 +26,8 @@ done platforms=( linux-x64 linux-arm64 + linux-armv6 + linux-armv7 macos-x64 macos-arm64 ) @@ -60,7 +61,7 @@ gpg -u 408B88DB29DDE9E0 --output "$RELEASE_DIR"/install.sh.sig --sign "$RELEASE_ NPM_PREFIX=@jdxcode/rtx ./rtx/scripts/release-npm.sh NPM_PREFIX=rtx-cli ./rtx/scripts/release-npm.sh -AWS_S3_BUCKET=rtx.pub ./rtx/scripts/publish-s3.sh +#AWS_S3_BUCKET=rtx.pub ./rtx/scripts/publish-s3.sh ./rtx/scripts/publish-r2.sh ./rtx/scripts/render-homebrew.sh >homebrew-tap/rtx.rb diff --git a/scripts/render-homebrew.sh b/scripts/render-homebrew.sh index 11788fb92c..c906a41f3a 100755 --- a/scripts/render-homebrew.sh +++ b/scripts/render-homebrew.sh @@ -5,6 +5,8 @@ set -euxo pipefail RTX_VERSION=${RTX_VERSION#v*} \ RTX_CHECKSUM_LINUX_X86_64=$(grep "rtx-v$RTX_VERSION-linux-x64.tar.xz" "$RELEASE_DIR/v$RTX_VERSION/SHASUMS256.txt" | cut -d ' ' -f1) \ RTX_CHECKSUM_LINUX_ARM64=$(grep "rtx-v$RTX_VERSION-linux-arm64.tar.xz" "$RELEASE_DIR/v$RTX_VERSION/SHASUMS256.txt" | cut -d ' ' -f1) \ + RTX_CHECKSUM_LINUX_ARMV6=$(grep "rtx-v$RTX_VERSION-linux-armv6.tar.xz" "$RELEASE_DIR/v$RTX_VERSION/SHASUMS256.txt" | cut -d ' ' -f1) \ + RTX_CHECKSUM_LINUX_ARMV7=$(grep "rtx-v$RTX_VERSION-linux-armv7.tar.xz" "$RELEASE_DIR/v$RTX_VERSION/SHASUMS256.txt" | cut -d ' ' -f1) \ RTX_CHECKSUM_MACOS_X86_64=$(grep "rtx-v$RTX_VERSION-macos-x64.tar.xz" "$RELEASE_DIR/v$RTX_VERSION/SHASUMS256.txt" | cut -d ' ' -f1) \ RTX_CHECKSUM_MACOS_ARM64=$(grep "rtx-v$RTX_VERSION-macos-arm64.tar.xz" "$RELEASE_DIR/v$RTX_VERSION/SHASUMS256.txt" | cut -d ' ' -f1) \ envsubst '$RTX_VERSION,$RTX_CHECKSUM_LINUX_X86_64,$RTX_CHECKSUM_LINUX_ARM64,$RTX_CHECKSUM_MACOS_X86_64,$RTX_CHECKSUM_MACOS_ARM64' \ diff --git a/scripts/render-install.sh b/scripts/render-install.sh index f417830637..fa3d375640 100755 --- a/scripts/render-install.sh +++ b/scripts/render-install.sh @@ -5,6 +5,8 @@ set -euxo pipefail RTX_VERSION=$RTX_VERSION \ RTX_CHECKSUM_LINUX_X86_64=$(grep "rtx-v.*linux-x64.tar.gz" "$RELEASE_DIR/$RTX_VERSION/SHASUMS256.txt") \ RTX_CHECKSUM_LINUX_ARM64=$(grep "rtx-v.*linux-arm64.tar.gz" "$RELEASE_DIR/$RTX_VERSION/SHASUMS256.txt") \ + RTX_CHECKSUM_LINUX_ARMV6=$(grep "rtx-v.*linux-armv6.tar.gz" "$RELEASE_DIR/$RTX_VERSION/SHASUMS256.txt") \ + RTX_CHECKSUM_LINUX_ARMV7=$(grep "rtx-v.*linux-armv7.tar.gz" "$RELEASE_DIR/$RTX_VERSION/SHASUMS256.txt") \ RTX_CHECKSUM_MACOS_X86_64=$(grep "rtx-v.*macos-x64.tar.gz" "$RELEASE_DIR/$RTX_VERSION/SHASUMS256.txt") \ RTX_CHECKSUM_MACOS_ARM64=$(grep "rtx-v.*macos-arm64.tar.gz" "$RELEASE_DIR/$RTX_VERSION/SHASUMS256.txt") \ envsubst '$RTX_VERSION,$RTX_CHECKSUM_LINUX_X86_64,$RTX_CHECKSUM_LINUX_ARM64,$RTX_CHECKSUM_MACOS_X86_64,$RTX_CHECKSUM_MACOS_ARM64' \ diff --git a/src/cli/version.rs b/src/cli/version.rs index 25258a5791..d2fcc04141 100644 --- a/src/cli/version.rs +++ b/src/cli/version.rs @@ -113,22 +113,17 @@ fn get_latest_version_call() -> Option { let timeout = Duration::from_secs(3); const URL: &str = "http://rtx.pub/VERSION"; debug!("checking rtx version from {}", URL); - let client = crate::http::Client::new().ok()?; - match client.get(URL).timeout(timeout).send() { - Ok(res) => { - if res.status().is_success() { - return res.text().ok().map(|text| { - debug!("got version {text}"); - text.trim().to_string() - }); - } - debug!("failed to check for version: {:#?}", res); + let client = crate::http::Client::new_with_timeout(timeout).ok()?; + match client.get_text(URL) { + Ok(text) => { + debug!("got version {text}"); + Some(text.trim().to_string()) } Err(err) => { debug!("failed to check for version: {:#?}", err); + None } - }; - None + } } #[cfg(test)] diff --git a/src/file.rs b/src/file.rs index 2a2e4e5651..fa114ebe36 100644 --- a/src/file.rs +++ b/src/file.rs @@ -202,7 +202,8 @@ pub fn is_executable(path: &Path) -> bool { pub fn make_executable(path: &Path) -> Result<()> { let mut perms = path.metadata()?.permissions(); perms.set_mode(perms.mode() | 0o111); - fs::set_permissions(path, perms)?; + fs::set_permissions(path, perms) + .wrap_err_with(|| format!("failed to chmod +x: {}", display_path(path)))?; Ok(()) } @@ -264,7 +265,9 @@ pub fn untar(archive: &Path, dest: &Path) -> Result<()> { } pub fn unzip(archive: &Path, dest: &Path) -> Result<()> { - cmd!("unzip", archive, "-d", dest).run()?; + cmd!("unzip", archive, "-d", dest) + .run() + .wrap_err_with(|| eyre!("unzip {} -d {}", display_path(archive), display_path(dest)))?; Ok(()) } diff --git a/src/http.rs b/src/http.rs index 6f5b522be0..1ed00cbadb 100644 --- a/src/http.rs +++ b/src/http.rs @@ -2,8 +2,10 @@ use std::fs::File; use std::path::Path; use std::time::Duration; +use crate::file::display_path; +use crate::{env, file}; use eyre::{Report, Result}; -use reqwest::blocking::{ClientBuilder, RequestBuilder}; +use reqwest::blocking::{ClientBuilder, Response}; use reqwest::IntoUrl; #[derive(Debug)] @@ -30,16 +32,25 @@ impl Client { .gzip(true) } - pub fn get(&self, url: U) -> RequestBuilder { + pub fn get(&self, url: U) -> Result { let url = url.into_url().unwrap(); debug!("GET {}", url); - self.reqwest.get(url) + dbg!(&url, url.host_str()); + let mut req = self.reqwest.get(url.clone()); + if url.host_str() == Some("api.github.com") { + if let Some(token) = &*env::GITHUB_API_TOKEN { + req = req.header("authorization", format!("token {}", token)); + } + } + let resp = req.send()?; + debug!("GET {url} {}", resp.status()); + resp.error_for_status_ref()?; + Ok(resp) } pub fn get_text(&self, url: U) -> Result { let url = url.into_url().unwrap(); - let resp = self.get(url).send()?; - resp.error_for_status_ref()?; + let resp = self.get(url)?; let text = resp.text()?; Ok(text) } @@ -49,17 +60,17 @@ impl Client { T: serde::de::DeserializeOwned, { let url = url.into_url().unwrap(); - let resp = self.get(url.clone()).send()?; - resp.error_for_status_ref()?; + let resp = self.get(url.clone())?; let json = resp.json()?; Ok(json) } pub fn download_file(&self, url: U, path: &Path) -> Result<()> { let url = url.into_url()?; - debug!("Downloading {} to {}", &url, path.display()); - let mut resp = self.get(url).send()?; - resp.error_for_status_ref()?; + debug!("GET Downloading {} to {}", &url, display_path(path)); + let mut resp = self.get(url.clone())?; + + file::create_dir_all(path.parent().unwrap())?; let mut file = File::create(path)?; resp.copy_to(&mut file)?; Ok(()) diff --git a/src/plugins/core/bun.rs b/src/plugins/core/bun.rs index 312292adfd..08e722999f 100644 --- a/src/plugins/core/bun.rs +++ b/src/plugins/core/bun.rs @@ -13,7 +13,7 @@ use crate::plugins::core::CorePlugin; use crate::plugins::Plugin; use crate::toolset::{ToolVersion, ToolVersionRequest}; use crate::ui::progress_report::ProgressReport; -use crate::{env, file, http}; +use crate::{file, http}; #[derive(Debug)] pub struct BunPlugin { @@ -28,13 +28,8 @@ impl BunPlugin { fn fetch_remote_versions(&self) -> Result> { let http = http::Client::new()?; - let mut req = http.get("https://api.github.com/repos/oven-sh/bun/releases?per_page=100"); - if let Some(token) = &*env::GITHUB_API_TOKEN { - req = req.header("authorization", format!("token {}", token)); - } - let resp = req.send()?; - resp.error_for_status_ref()?; - let releases: Vec = resp.json()?; + let releases: Vec = + http.json("https://api.github.com/repos/oven-sh/bun/releases?per_page=100")?; let versions = releases .into_iter() .map(|r| r.tag_name) diff --git a/src/plugins/core/deno.rs b/src/plugins/core/deno.rs index b7e9d817d0..3abd7679c6 100644 --- a/src/plugins/core/deno.rs +++ b/src/plugins/core/deno.rs @@ -14,7 +14,7 @@ use crate::plugins::core::CorePlugin; use crate::plugins::Plugin; use crate::toolset::{ToolVersion, ToolVersionRequest, Toolset}; use crate::ui::progress_report::ProgressReport; -use crate::{env, file, http}; +use crate::{file, http}; #[derive(Debug)] pub struct DenoPlugin { @@ -29,13 +29,8 @@ impl DenoPlugin { fn fetch_remote_versions(&self) -> Result> { let http = http::Client::new()?; - let mut req = http.get("https://api.github.com/repos/denoland/deno/releases?per_page=100"); - if let Some(token) = &*env::GITHUB_API_TOKEN { - req = req.header("authorization", format!("token {}", token)); - } - let resp = req.send()?; - resp.error_for_status_ref()?; - let releases: Vec = resp.json()?; + let releases: Vec = + http.json("https://api.github.com/repos/denoland/deno/releases?per_page=100")?; let versions = releases .into_iter() .map(|r| r.name) diff --git a/src/plugins/core/python.rs b/src/plugins/core/python.rs index a277439a4d..9092e393de 100644 --- a/src/plugins/core/python.rs +++ b/src/plugins/core/python.rs @@ -187,9 +187,7 @@ impl Plugin for PythonPlugin { ctx.pr .set_message(format!("with patch file from: {patch_url}")); let http = http::Client::new()?; - let resp = http.get(patch_url).send()?; - resp.error_for_status_ref()?; - let patch = resp.text()?; + let patch = http.get_text(patch_url)?; cmd = cmd.arg("--patch").stdin_string(patch) } if let Some(patches_dir) = &*env::RTX_PYTHON_PATCHES_DIRECTORY { diff --git a/src/plugins/core/ruby.rs b/src/plugins/core/ruby.rs index 3ddea4f374..d68d6aa39a 100644 --- a/src/plugins/core/ruby.rs +++ b/src/plugins/core/ruby.rs @@ -7,7 +7,7 @@ use color_eyre::eyre::Result; use crate::cmd::CmdLineRunner; use crate::config::{Config, Settings}; use crate::duration::DAILY; -use crate::env::GITHUB_API_TOKEN; + use crate::git::Git; use crate::github::GithubRelease; use crate::install_context::InstallContext; @@ -220,13 +220,8 @@ impl RubyPlugin { fn latest_ruby_build_version(&self) -> Result { let http = http::Client::new()?; - let mut req = http.get("https://api.github.com/repos/rbenv/ruby-build/releases/latest"); - if let Some(token) = &*GITHUB_API_TOKEN { - req = req.header("authorization", format!("token {}", token)); - } - let resp = req.send()?; - resp.error_for_status_ref()?; - let release: GithubRelease = resp.json()?; + let release: GithubRelease = + http.json("https://api.github.com/repos/rbenv/ruby-build/releases/latest")?; Ok(release.tag_name.trim_start_matches('v').to_string()) } @@ -311,9 +306,7 @@ impl RubyPlugin { for f in &self.fetch_patch_sources() { if regex!(r#"^[Hh][Tt][Tt][Pp][Ss]?://"#).is_match(f) { let http = http::Client::new()?; - let resp = http.get(f).send()?; - resp.error_for_status_ref()?; - patches.push(resp.text()?); + patches.push(http.get_text(f)?); } else { patches.push(file::read_to_string(f)?); }