diff --git a/.github/workflows/docker-build-push-backend-container-on-tag.yml b/.github/workflows/docker-build-push-backend-container-on-tag.yml index ef33750c271..162c0221308 100644 --- a/.github/workflows/docker-build-push-backend-container-on-tag.yml +++ b/.github/workflows/docker-build-push-backend-container-on-tag.yml @@ -6,7 +6,7 @@ on: - "*" env: - REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'danswer/danswer-backend-cloud' || 'danswer/danswer-backend' }} + REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }} LATEST_TAG: ${{ contains(github.ref_name, 'latest') }} jobs: @@ -44,7 +44,7 @@ jobs: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} ${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }} build-args: | - DANSWER_VERSION=${{ github.ref_name }} + ONYX_VERSION=${{ github.ref_name }} # trivy has their own rate limiting issues causing this action to flake # we worked around it by hardcoding to different db repos in env @@ -57,7 +57,7 @@ jobs: TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2" TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1" with: - # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend + # To run locally: trivy image --severity HIGH,CRITICAL onyxdotapp/onyx-backend image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} severity: "CRITICAL,HIGH" trivyignores: ./backend/.trivyignore diff --git a/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml b/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml index 45cd5093a0c..99caf6392a0 100644 --- a/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml +++ b/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml @@ -7,7 +7,7 @@ on: - "*" env: - REGISTRY_IMAGE: danswer/danswer-web-server-cloud + REGISTRY_IMAGE: onyxdotapp/onyx-web-server-cloud LATEST_TAG: ${{ contains(github.ref_name, 'latest') }} jobs: @@ -60,7 +60,7 @@ jobs: platforms: ${{ matrix.platform }} push: true build-args: | - DANSWER_VERSION=${{ github.ref_name }} + ONYX_VERSION=${{ github.ref_name }} NEXT_PUBLIC_CLOUD_ENABLED=true NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }} NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }} diff --git a/.github/workflows/docker-build-push-model-server-container-on-tag.yml b/.github/workflows/docker-build-push-model-server-container-on-tag.yml index 3e0445ab04a..7df47c416ce 100644 --- a/.github/workflows/docker-build-push-model-server-container-on-tag.yml +++ b/.github/workflows/docker-build-push-model-server-container-on-tag.yml @@ -6,20 +6,70 @@ on: - "*" env: - REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'danswer/danswer-model-server-cloud' || 'danswer/danswer-model-server' }} + REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }} LATEST_TAG: ${{ contains(github.ref_name, 'latest') }} + DOCKER_BUILDKIT: 1 + BUILDKIT_PROGRESS: plain jobs: - build-and-push: - # See https://runs-on.com/runners/linux/ - runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}"] + build-amd64: + runs-on: + [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-amd64"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: System Info + run: | + df -h + free -h + docker system prune -af --volumes + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + image=moby/buildkit:latest + network=host + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push AMD64 + uses: docker/build-push-action@v5 + with: + context: ./backend + file: ./backend/Dockerfile.model_server + platforms: linux/amd64 + push: true + tags: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 + build-args: | + DANSWER_VERSION=${{ github.ref_name }} + outputs: type=registry + provenance: false + build-arm64: + runs-on: + [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-arm64"] steps: - name: Checkout code uses: actions/checkout@v4 + - name: System Info + run: | + df -h + free -h + docker system prune -af --volumes + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + image=moby/buildkit:latest + network=host - name: Login to Docker Hub uses: docker/login-action@v3 @@ -27,29 +77,47 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Model Server Image Docker Build and Push + - name: Build and Push ARM64 uses: docker/build-push-action@v5 with: context: ./backend file: ./backend/Dockerfile.model_server - platforms: linux/amd64,linux/arm64 + platforms: linux/arm64 push: true - tags: | - ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} - ${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }} + tags: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64 build-args: | DANSWER_VERSION=${{ github.ref_name }} + outputs: type=registry + provenance: false + + merge-and-scan: + needs: [build-amd64, build-arm64] + runs-on: ubuntu-latest + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Create and Push Multi-arch Manifest + run: | + docker buildx create --use + docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} \ + ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \ + ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64 + if [[ "${{ env.LATEST_TAG }}" == "true" ]]; then + docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:latest \ + ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \ + ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64 + fi - # trivy has their own rate limiting issues causing this action to flake - # we worked around it by hardcoding to different db repos in env - # can re-enable when they figure it out - # https://github.com/aquasecurity/trivy/discussions/7538 - # https://github.com/aquasecurity/trivy-action/issues/389 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master env: TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2" TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1" with: - image-ref: docker.io/danswer/danswer-model-server:${{ github.ref_name }} + image-ref: docker.io/onyxdotapp/onyx-model-server:${{ github.ref_name }} severity: "CRITICAL,HIGH" + timeout: "10m" diff --git a/.github/workflows/docker-build-push-web-container-on-tag.yml b/.github/workflows/docker-build-push-web-container-on-tag.yml index 4f1fd804969..b7f4a5dbc68 100644 --- a/.github/workflows/docker-build-push-web-container-on-tag.yml +++ b/.github/workflows/docker-build-push-web-container-on-tag.yml @@ -3,12 +3,12 @@ name: Build and Push Web Image on Tag on: push: tags: - - '*' + - "*" env: - REGISTRY_IMAGE: danswer/danswer-web-server + REGISTRY_IMAGE: onyxdotapp/onyx-web-server LATEST_TAG: ${{ contains(github.ref_name, 'latest') }} - + jobs: build: runs-on: @@ -27,11 +27,11 @@ jobs: - name: Prepare run: | platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Checkout uses: actions/checkout@v4 - + - name: Docker meta id: meta uses: docker/metadata-action@v5 @@ -40,16 +40,16 @@ jobs: tags: | type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} type=raw,value=${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }} - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - + - name: Build and push by digest id: build uses: docker/build-push-action@v5 @@ -59,18 +59,18 @@ jobs: platforms: ${{ matrix.platform }} push: true build-args: | - DANSWER_VERSION=${{ github.ref_name }} - # needed due to weird interactions with the builds for different platforms + ONYX_VERSION=${{ github.ref_name }} + # needed due to weird interactions with the builds for different platforms no-cache: true labels: ${{ steps.meta.outputs.labels }} outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - + - name: Export digest run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest uses: actions/upload-artifact@v4 with: @@ -90,42 +90,42 @@ jobs: path: /tmp/digests pattern: digests-* merge-multiple: true - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY_IMAGE }} - + - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - + - name: Create manifest list and push working-directory: /tmp/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) - + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - name: Inspect image run: | docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} - # trivy has their own rate limiting issues causing this action to flake - # we worked around it by hardcoding to different db repos in env - # can re-enable when they figure it out - # https://github.com/aquasecurity/trivy/discussions/7538 - # https://github.com/aquasecurity/trivy-action/issues/389 + # trivy has their own rate limiting issues causing this action to flake + # we worked around it by hardcoding to different db repos in env + # can re-enable when they figure it out + # https://github.com/aquasecurity/trivy/discussions/7538 + # https://github.com/aquasecurity/trivy-action/issues/389 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master env: - TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2' - TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1' + TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2" + TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1" with: image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} - severity: 'CRITICAL,HIGH' + severity: "CRITICAL,HIGH" diff --git a/.github/workflows/docker-tag-latest.yml b/.github/workflows/docker-tag-latest.yml index e2c7c30f31e..fd0c07e5ba7 100644 --- a/.github/workflows/docker-tag-latest.yml +++ b/.github/workflows/docker-tag-latest.yml @@ -7,31 +7,31 @@ on: workflow_dispatch: inputs: version: - description: 'The version (ie v0.0.1) to tag as latest' + description: "The version (ie v0.0.1) to tag as latest" required: true jobs: tag: # See https://runs-on.com/runners/linux/ # use a lower powered instance since this just does i/o to docker hub - runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"] steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} - - name: Enable Docker CLI experimental features - run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV + - name: Enable Docker CLI experimental features + run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV - - name: Pull, Tag and Push Web Server Image - run: | - docker buildx imagetools create -t danswer/danswer-web-server:latest danswer/danswer-web-server:${{ github.event.inputs.version }} + - name: Pull, Tag and Push Web Server Image + run: | + docker buildx imagetools create -t onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:${{ github.event.inputs.version }} - - name: Pull, Tag and Push API Server Image - run: | - docker buildx imagetools create -t danswer/danswer-backend:latest danswer/danswer-backend:${{ github.event.inputs.version }} + - name: Pull, Tag and Push API Server Image + run: | + docker buildx imagetools create -t onyxdotapp/onyx-backend:latest onyxdotapp/onyx-backend:${{ github.event.inputs.version }} diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml index 0e921f8d694..6e14fa8269e 100644 --- a/.github/workflows/hotfix-release-branches.yml +++ b/.github/workflows/hotfix-release-branches.yml @@ -8,43 +8,42 @@ on: workflow_dispatch: inputs: hotfix_commit: - description: 'Hotfix commit hash' + description: "Hotfix commit hash" required: true hotfix_suffix: - description: 'Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})' + description: "Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})" required: true release_branch_pattern: - description: 'Release branch pattern (regex)' + description: "Release branch pattern (regex)" required: true - default: 'release/.*' + default: "release/.*" auto_merge: - description: 'Automatically merge the hotfix PRs' + description: "Automatically merge the hotfix PRs" required: true type: choice - default: 'true' + default: "true" options: - true - false - + jobs: hotfix_release_branches: permissions: write-all # See https://runs-on.com/runners/linux/ # use a lower powered instance since this just does i/o to docker hub - runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"] steps: - # needs RKUO_DEPLOY_KEY for write access to merge PR's - name: Checkout Repository uses: actions/checkout@v4 with: ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}" fetch-depth: 0 - + - name: Set up Git user run: | git config user.name "Richard Kuo [bot]" - git config user.email "rkuo[bot]@danswer.ai" + git config user.email "rkuo[bot]@onyx.app" - name: Fetch All Branches run: | @@ -62,10 +61,10 @@ jobs: echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'." exit 1 fi - + echo "Found release branches:" echo "$BRANCHES" - + # Join the branches into a single line separated by commas BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//') @@ -169,4 +168,4 @@ jobs: echo "Failed to merge pull request #$PR_NUMBER." fi fi - done \ No newline at end of file + done diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml index 273f00a5c5a..8b0906916e3 100644 --- a/.github/workflows/pr-backport-autotrigger.yml +++ b/.github/workflows/pr-backport-autotrigger.yml @@ -4,7 +4,7 @@ name: Backport on Merge on: pull_request: - types: [closed] # Later we check for merge so only PRs that go in can get backported + types: [closed] # Later we check for merge so only PRs that go in can get backported permissions: contents: write @@ -26,9 +26,9 @@ jobs: - name: Set up Git user run: | git config user.name "Richard Kuo [bot]" - git config user.email "rkuo[bot]@danswer.ai" + git config user.email "rkuo[bot]@onyx.app" git fetch --prune - + - name: Check for Backport Checkbox id: checkbox-check run: | @@ -51,14 +51,14 @@ jobs: # Fetch latest tags for beta and stable LATEST_BETA_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*-beta.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$" | grep -v -- "-cloud" | sort -Vr | head -n 1) LATEST_STABLE_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -Vr | head -n 1) - + # Handle case where no beta tags exist if [[ -z "$LATEST_BETA_TAG" ]]; then NEW_BETA_TAG="v1.0.0-beta.1" else NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}') fi - + # Increment latest stable tag NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}') echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT @@ -80,10 +80,10 @@ jobs: run: | set -e echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}" - + # Echo the merge commit SHA echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}" - + # Fetch all history for all branches and tags git fetch --prune @@ -98,7 +98,7 @@ jobs: echo "Cherry-pick to beta failed due to conflicts." exit 1 } - + # Create new beta branch/tag git tag ${{ steps.list-branches.outputs.new_beta_tag }} # Push the changes and tag to the beta branch using PAT @@ -110,13 +110,13 @@ jobs: echo "Last 5 commits on stable branch:" git log -n 5 --pretty=format:"%H" echo "" # Newline for formatting - + # Cherry-pick the merge commit from the merged PR git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || { echo "Cherry-pick to stable failed due to conflicts." exit 1 } - + # Create new stable branch/tag git tag ${{ steps.list-branches.outputs.new_stable_tag }} # Push the changes and tag to the stable branch using PAT diff --git a/.github/workflows/pr-chromatic-tests.yml b/.github/workflows/pr-chromatic-tests.yml index 5d8b29ed572..1ebb7598116 100644 --- a/.github/workflows/pr-chromatic-tests.yml +++ b/.github/workflows/pr-chromatic-tests.yml @@ -14,18 +14,24 @@ jobs: name: Playwright Tests # See https://runs-on.com/runners/linux/ - runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"] + runs-on: + [ + runs-on, + runner=32cpu-linux-x64, + disk=large, + "run-id=${{ github.run_id }}", + ] steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - cache: 'pip' + python-version: "3.11" + cache: "pip" cache-dependency-path: | backend/requirements/default.txt backend/requirements/dev.txt @@ -35,7 +41,7 @@ jobs: pip install --retries 5 --timeout 30 -r backend/requirements/default.txt pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt - + - name: Setup node uses: actions/setup-node@v4 with: @@ -48,7 +54,7 @@ jobs: - name: Install playwright browsers working-directory: ./web run: npx playwright install --with-deps - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -60,13 +66,13 @@ jobs: # tag every docker image with "test" so that we can spin up the correct set # of images during testing - + # we use the runs-on cache for docker builds # in conjunction with runs-on runners, it has better speed and unlimited caching # https://runs-on.com/caching/s3-cache-for-github-actions/ # https://runs-on.com/caching/docker/ # https://github.com/moby/buildkit#s3-cache-experimental - + # images are built and run locally for testing purposes. Not pushed. - name: Build Web Docker image @@ -75,7 +81,7 @@ jobs: context: ./web file: ./web/Dockerfile platforms: linux/amd64 - tags: danswer/danswer-web-server:test + tags: onyxdotapp/onyx-web-server:test push: false load: true cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} @@ -87,7 +93,7 @@ jobs: context: ./backend file: ./backend/Dockerfile platforms: linux/amd64 - tags: danswer/danswer-backend:test + tags: onyxdotapp/onyx-backend:test push: false load: true cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} @@ -99,7 +105,7 @@ jobs: context: ./backend file: ./backend/Dockerfile.model_server platforms: linux/amd64 - tags: danswer/danswer-model-server:test + tags: onyxdotapp/onyx-model-server:test push: false load: true cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} @@ -110,6 +116,7 @@ jobs: cd deployment/docker_compose ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \ AUTH_TYPE=basic \ + GEN_AI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ REQUIRE_EMAIL_VERIFICATION=false \ DISABLE_TELEMETRY=true \ IMAGE_TAG=test \ @@ -119,12 +126,12 @@ jobs: - name: Wait for service to be ready run: | echo "Starting wait-for-service script..." - + docker logs -f danswer-stack-api_server-1 & start_time=$(date +%s) timeout=300 # 5 minutes in seconds - + while true; do current_time=$(date +%s) elapsed_time=$((current_time - start_time)) @@ -152,7 +159,7 @@ jobs: - name: Run pytest playwright test init working-directory: ./backend - env: + env: PYTEST_IGNORE_SKIP: true run: pytest -s tests/integration/tests/playwright/test_playwright.py @@ -168,7 +175,7 @@ jobs: name: test-results path: ./web/test-results retention-days: 30 - + # save before stopping the containers so the logs can be captured - name: Save Docker logs if: success() || failure() @@ -176,7 +183,7 @@ jobs: cd deployment/docker_compose docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log mv docker-compose.log ${{ github.workspace }}/docker-compose.log - + - name: Upload logs if: success() || failure() uses: actions/upload-artifact@v4 @@ -191,35 +198,41 @@ jobs: chromatic-tests: name: Chromatic Tests - + needs: playwright-tests - runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"] + runs-on: + [ + runs-on, + runner=32cpu-linux-x64, + disk=large, + "run-id=${{ github.run_id }}", + ] steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Setup node uses: actions/setup-node@v4 with: node-version: 22 - + - name: Install node dependencies working-directory: ./web run: npm ci - + - name: Download Playwright test results uses: actions/download-artifact@v4 with: name: test-results path: ./web/test-results - + - name: Run Chromatic uses: chromaui/action@latest with: playwright: true projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} workingDir: ./web - env: + env: CHROMATIC_ARCHIVE_LOCATION: ./test-results diff --git a/.github/workflows/pr-integration-tests.yml b/.github/workflows/pr-integration-tests.yml index f2dc97e75da..f0004c4e256 100644 --- a/.github/workflows/pr-integration-tests.yml +++ b/.github/workflows/pr-integration-tests.yml @@ -8,7 +8,7 @@ on: pull_request: branches: - main - - 'release/**' + - "release/**" env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -16,11 +16,11 @@ env: CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} - + jobs: integration-tests: # See https://runs-on.com/runners/linux/ - runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"] steps: - name: Checkout code uses: actions/checkout@v4 @@ -36,21 +36,21 @@ jobs: # tag every docker image with "test" so that we can spin up the correct set # of images during testing - + # We don't need to build the Web Docker image since it's not yet used - # in the integration tests. We have a separate action to verify that it builds + # in the integration tests. We have a separate action to verify that it builds # successfully. - name: Pull Web Docker image run: | - docker pull danswer/danswer-web-server:latest - docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:test + docker pull onyxdotapp/onyx-web-server:latest + docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test # we use the runs-on cache for docker builds # in conjunction with runs-on runners, it has better speed and unlimited caching # https://runs-on.com/caching/s3-cache-for-github-actions/ # https://runs-on.com/caching/docker/ # https://github.com/moby/buildkit#s3-cache-experimental - + # images are built and run locally for testing purposes. Not pushed. - name: Build Backend Docker image uses: ./.github/actions/custom-build-and-push @@ -58,7 +58,7 @@ jobs: context: ./backend file: ./backend/Dockerfile platforms: linux/amd64 - tags: danswer/danswer-backend:test + tags: onyxdotapp/onyx-backend:test push: false load: true cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} @@ -70,19 +70,19 @@ jobs: context: ./backend file: ./backend/Dockerfile.model_server platforms: linux/amd64 - tags: danswer/danswer-model-server:test + tags: onyxdotapp/onyx-model-server:test push: false load: true cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max - + - name: Build integration test Docker image uses: ./.github/actions/custom-build-and-push with: context: ./backend file: ./backend/tests/integration/Dockerfile platforms: linux/amd64 - tags: danswer/danswer-integration:test + tags: onyxdotapp/onyx-integration:test push: false load: true cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} @@ -119,7 +119,7 @@ jobs: -e TEST_WEB_HOSTNAME=test-runner \ -e AUTH_TYPE=cloud \ -e MULTI_TENANT=true \ - danswer/danswer-integration:test \ + onyxdotapp/onyx-integration:test \ /app/tests/integration/multitenant_tests continue-on-error: true id: run_multitenant_tests @@ -131,15 +131,14 @@ jobs: exit 1 else echo "All integration tests passed successfully." - fi + fi - name: Stop multi-tenant Docker containers run: | cd deployment/docker_compose docker compose -f docker-compose.dev.yml -p danswer-stack down -v - - - name: Start Docker containers + - name: Start Docker containers run: | cd deployment/docker_compose ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \ @@ -153,12 +152,12 @@ jobs: - name: Wait for service to be ready run: | echo "Starting wait-for-service script..." - + docker logs -f danswer-stack-api_server-1 & start_time=$(date +%s) timeout=300 # 5 minutes in seconds - + while true; do current_time=$(date +%s) elapsed_time=$((current_time - start_time)) @@ -202,7 +201,7 @@ jobs: -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \ -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \ -e TEST_WEB_HOSTNAME=test-runner \ - danswer/danswer-integration:test \ + onyxdotapp/onyx-integration:test \ /app/tests/integration/tests \ /app/tests/integration/connector_job_tests continue-on-error: true @@ -229,7 +228,7 @@ jobs: run: | cd deployment/docker_compose docker compose -f docker-compose.dev.yml -p danswer-stack down -v - + - name: Upload logs if: success() || failure() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tag-nightly.yml b/.github/workflows/tag-nightly.yml index 50bb20808a3..61d1818f4d4 100644 --- a/.github/workflows/tag-nightly.yml +++ b/.github/workflows/tag-nightly.yml @@ -2,53 +2,52 @@ name: Nightly Tag Push on: schedule: - - cron: '0 10 * * *' # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC + - cron: "0 10 * * *" # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC permissions: - contents: write # Allows pushing tags to the repository + contents: write # Allows pushing tags to the repository jobs: create-and-push-tag: - runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"] + runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"] steps: - # actions using GITHUB_TOKEN cannot trigger another workflow, but we do want this to trigger docker pushes - # see https://github.com/orgs/community/discussions/27028#discussioncomment-3254367 for the workaround we - # implement here which needs an actual user's deploy key - - name: Checkout code - uses: actions/checkout@v4 - with: - ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}" - - - name: Set up Git user - run: | - git config user.name "Richard Kuo [bot]" - git config user.email "rkuo[bot]@danswer.ai" - - - name: Check for existing nightly tag - id: check_tag - run: | - if git tag --points-at HEAD --list "nightly-latest*" | grep -q .; then - echo "A tag starting with 'nightly-latest' already exists on HEAD." - echo "tag_exists=true" >> $GITHUB_OUTPUT - else - echo "No tag starting with 'nightly-latest' exists on HEAD." - echo "tag_exists=false" >> $GITHUB_OUTPUT - fi - - # don't tag again if HEAD already has a nightly-latest tag on it - - name: Create Nightly Tag - if: steps.check_tag.outputs.tag_exists == 'false' - env: - DATE: ${{ github.run_id }} - run: | - TAG_NAME="nightly-latest-$(date +'%Y%m%d')" - echo "Creating tag: $TAG_NAME" - git tag $TAG_NAME - - - name: Push Tag - if: steps.check_tag.outputs.tag_exists == 'false' - run: | - TAG_NAME="nightly-latest-$(date +'%Y%m%d')" - git push origin $TAG_NAME - \ No newline at end of file + # actions using GITHUB_TOKEN cannot trigger another workflow, but we do want this to trigger docker pushes + # see https://github.com/orgs/community/discussions/27028#discussioncomment-3254367 for the workaround we + # implement here which needs an actual user's deploy key + - name: Checkout code + uses: actions/checkout@v4 + with: + ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}" + + - name: Set up Git user + run: | + git config user.name "Richard Kuo [bot]" + git config user.email "rkuo[bot]@onyx.app" + + - name: Check for existing nightly tag + id: check_tag + run: | + if git tag --points-at HEAD --list "nightly-latest*" | grep -q .; then + echo "A tag starting with 'nightly-latest' already exists on HEAD." + echo "tag_exists=true" >> $GITHUB_OUTPUT + else + echo "No tag starting with 'nightly-latest' exists on HEAD." + echo "tag_exists=false" >> $GITHUB_OUTPUT + fi + + # don't tag again if HEAD already has a nightly-latest tag on it + - name: Create Nightly Tag + if: steps.check_tag.outputs.tag_exists == 'false' + env: + DATE: ${{ github.run_id }} + run: | + TAG_NAME="nightly-latest-$(date +'%Y%m%d')" + echo "Creating tag: $TAG_NAME" + git tag $TAG_NAME + + - name: Push Tag + if: steps.check_tag.outputs.tag_exists == 'false' + run: | + TAG_NAME="nightly-latest-$(date +'%Y%m%d')" + git push origin $TAG_NAME