diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6df348b72140..d78d492127ad8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,10 @@ jobs: - name: install awscli run: src/ci/scripts/install-awscli.sh + - if: contains(matrix.os, 'codebuild-ubuntu') + name: Install sotftware for codebuild + run: chmod +x src/ci/scripts/ubuntu-codebuild.sh && ./src/ci/scripts/ubuntu-codebuild.sh + - name: install sccache run: src/ci/scripts/install-sccache.sh @@ -174,6 +178,8 @@ jobs: - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh + # Don't run on codebuild because systemctl is not available + if: ${{ !contains(matrix.os, 'codebuild-ubuntu') }} # Disable automatic line ending conversion (again). On Windows, when we're # installing dependencies, something switches the git configuration directory or diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index d554186df4cfe..82f17f82261b8 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -282,8 +282,12 @@ else # LOCAL_USER_ID will map to a different subuid range on the host. # The "keep-id" mode maps the current UID directly into the container. args="$args --env NO_CHANGE_USER=1 --userns=keep-id" - else + elif [[ "$id" != 0 ]]; then args="$args --env LOCAL_USER_ID=$id" + else + # If we're running as root, we don't want to run the container as root, + # so we set id `1001` instead of `0`. + args="$args --env LOCAL_USER_ID=1001" fi fi diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 7de6d5fcd5f75..969acd992dcc0 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -176,6 +176,21 @@ def format_run_type(run_type: WorkflowRunType) -> str: raise AssertionError() +# Add new function before main: +def substitute_github_vars(jobs: list) -> list: + """Replace GitHub context variables with environment variables in job configs.""" + for job in jobs: + if "os" in job: + job["os"] = job["os"].replace( + "${{ github.run_id }}", + os.environ["GITHUB_RUN_ID"] + ).replace( + "${{ github.run_attempt }}", + os.environ["GITHUB_RUN_ATTEMPT"] + ) + return jobs + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -194,6 +209,8 @@ def format_run_type(run_type: WorkflowRunType) -> str: if run_type is not None: jobs = calculate_jobs(run_type, data) jobs = skip_jobs(jobs, channel) + jobs = substitute_github_vars(jobs) + if not jobs: raise Exception("Scheduled job list is empty, this is an error") diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 2ea37c168dd3c..10756dd4951cd 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -46,6 +46,10 @@ runners: - &job-aarch64-linux os: ubuntu-22.04-arm64-8core-32gb + - &job-linux-8c-codebuild + os: codebuild-ubuntu-22-8c-${{ github.run_id }}-${{ github.run_attempt }} + <<: *base-job + envs: env-x86_64-apple-tests: &env-x86_64-apple-tests SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact @@ -237,7 +241,7 @@ auto: <<: *job-linux-4c - image: x86_64-gnu - <<: *job-linux-4c + <<: *job-linux-8c-codebuild # This job ensures commits landing on nightly still pass the full # test suite on the stable channel. There are some UI tests that diff --git a/src/ci/run.sh b/src/ci/run.sh index b874f71832d73..99dc2e6e6a009 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -2,12 +2,15 @@ set -e +echo "Running inside src/ci/run.sh script" + if [ -n "$CI_JOB_NAME" ]; then echo "[CI_JOB_NAME=$CI_JOB_NAME]" fi if [ "$NO_CHANGE_USER" = "" ]; then if [ "$LOCAL_USER_ID" != "" ]; then + echo "Starting with UID: $LOCAL_USER_ID" id -u user &>/dev/null || useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user export HOME=/home/user unset LOCAL_USER_ID @@ -21,7 +24,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then # For NO_CHANGE_USER done in the small number of Dockerfiles affected. echo -e '[safe]\n\tdirectory = *' > /home/user/.gitconfig + echo "Switching to user" exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user + echo "whoami: $(whoami)" fi fi diff --git a/src/ci/scripts/enable-docker-ipv6.sh b/src/ci/scripts/enable-docker-ipv6.sh index 03d5a75e24e27..fb09a399d9bec 100755 --- a/src/ci/scripts/enable-docker-ipv6.sh +++ b/src/ci/scripts/enable-docker-ipv6.sh @@ -7,6 +7,12 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" +# Print docker version +docker --version +lsb_release -a +echo "is docker active" +systemctl is-active docker + if isLinux; then sudo mkdir -p /etc/docker echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' \ diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh index 55e75800d91c4..90d6894c7c234 100755 --- a/src/ci/scripts/run-build-from-ci.sh +++ b/src/ci/scripts/run-build-from-ci.sh @@ -17,7 +17,9 @@ echo "::add-matcher::src/ci/github-actions/problem_matchers.json" # the environment rustup self uninstall -y || true if [ -z "${IMAGE+x}" ]; then + echo "Running ci/run.sh" src/ci/run.sh else + echo "Running docker/run.sh with image ${IMAGE}" src/ci/docker/run.sh "${IMAGE}" fi diff --git a/src/ci/scripts/ubuntu-codebuild.sh b/src/ci/scripts/ubuntu-codebuild.sh new file mode 100644 index 0000000000000..da5371965de46 --- /dev/null +++ b/src/ci/scripts/ubuntu-codebuild.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# apt-get update +# apt-get install -y docker-buildx