diff --git a/.dockerignore b/.dockerignore index 282ed0918..ce728ccad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,10 @@ docs/* -keep-ui/node_modules -keep-ui/.next/* -keep-ui/.env.local +**/node_modules +keep-ui/ +tests/ +**/__pycache__ +*.pyc +.git .venv/ .vercel/ .vscode/ diff --git a/.github/workflows/test-pr-e2e-build-and-run.yml b/.github/workflows/test-pr-e2e-build-and-run.yml new file mode 100644 index 000000000..0a03d9109 --- /dev/null +++ b/.github/workflows/test-pr-e2e-build-and-run.yml @@ -0,0 +1,101 @@ +name: Build and up (E2E) + +on: + workflow_dispatch: + pull_request: + paths: + - 'keep/**' + - 'keep-ui/**' + - 'tests/**' + +env: + PYTHON_VERSION: 3.11 + STORAGE_MANAGER_DIRECTORY: /tmp/storage-manager + # MySQL server environment variables + MYSQL_ROOT_PASSWORD: keep + MYSQL_DATABASE: keep + # Postgres environment variables + POSTGRES_USER: keepuser + POSTGRES_PASSWORD: keeppassword + POSTGRES_DB: keepdb + # To test if imports are working properly + EE_ENABLED: true + +jobs: + tests-e2e: + permissions: + contents: read + id-token: write + runs-on: depot-ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: chartboost/ruff-action@v1 + with: + src: "./keep" + + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Set up Depot CLI + uses: depot/setup-action@v1 + + - name: Bake Docker images + uses: depot/bake-action@v1 + with: + token: ${{ secrets.DEPOT_TOKEN }} + project: ${{ secrets.DEPOT_PROJECT }} + workdir: . + files: | + tests/e2e_tests/docker-compose-e2e-mysql.yml + load: true + + - name: Set up Keep environment + run: | + COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose \ + --project-directory . \ + -f tests/e2e_tests/docker-compose-e2e-mysql.yml up -d + + - name: Wait for services to be ready + run: | + # Add commands to wait for the database to be ready + until docker exec $(docker ps -qf "name=keep-database") mysqladmin ping -h "localhost" --silent; do + echo "Waiting for MySQL to be ready..." + sleep 2 + done + + # wait to keep backend on port 8080 + echo "Waiting for Keep backend to be ready..." + attempt=0 + max_attempts=10 + + until $(curl --output /dev/null --silent --fail http://localhost:8080/healthcheck); do + if [ "$attempt" -ge "$max_attempts" ]; then + echo "Max attempts reached, exiting... Sometimes Keep can't start because of double-headed migrations, use: 'alembic -c keep/alembic.ini history' to investigate, or check artifacts." + exit 1 + fi + echo "Waiting for Keep backend to be ready... (Attempt: $((attempt+1)))" + attempt=$((attempt+1)) + sleep 2 + done + + echo "Keep backend is ready!" + # wait to the backend + echo "Waiting for Keep frontend to be ready..." + attempt=0 + max_attempts=10 + + until $(curl --output /dev/null --silent --fail http://localhost:3000/); do + if [ "$attempt" -ge "$max_attempts" ]; then + echo "Max attempts reached, exiting..." + exit 1 + fi + echo "Waiting for Keep frontend to be ready... (Attempt: $((attempt+1)))" + attempt=$((attempt+1)) + sleep 2 + done + + # create the state directory + # mkdir -p ./state && chown -R root:root ./state && chmod -R 777 ./state + diff --git a/.github/workflows/test-pr-e2e.yml b/.github/workflows/test-pr-e2e.yml deleted file mode 100644 index ba7cb43e5..000000000 --- a/.github/workflows/test-pr-e2e.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: Tests (E2E) - -on: - workflow_dispatch: - pull_request: - paths: - - 'keep/**' - - 'keep-ui/**' - - 'tests/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref }} - cancel-in-progress: true - -env: - PYTHON_VERSION: 3.11 - STORAGE_MANAGER_DIRECTORY: /tmp/storage-manager - # MySQL server environment variables - MYSQL_ROOT_PASSWORD: keep - MYSQL_DATABASE: keep - # Postgres environment variables - POSTGRES_USER: keepuser - POSTGRES_PASSWORD: keeppassword - POSTGRES_DB: keepdb - # To test if imports are working properly - EE_ENABLED: true - -jobs: - tests-e2e: - runs-on: ubuntu-latest - strategy: - matrix: - db_type: [mysql, postgres] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - uses: chartboost/ruff-action@v1 - with: - src: "./keep" - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Cache dependencies - id: cache-deps - uses: actions/cache@v2 - with: - path: .venv - key: pydeps-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies using poetry - run: poetry install --no-interaction --no-root --with dev - - - name: Install Playwright dependencies - run: npx playwright install --with-deps - - - name: Install playwright - run: poetry run playwright install - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Set up Keep environment - run: | - DOCKER_BUILDKIT=1 docker compose \ - --project-directory . \ - -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml up -d - - - name: Wait for database to be ready - run: | - # Add commands to wait for the database to be ready - if [ "${{ matrix.db_type }}" == "mysql" ]; then - until docker exec $(docker ps -qf "name=keep-database") mysqladmin ping -h "localhost" --silent; do - echo "Waiting for MySQL to be ready..." - sleep 2 - done - elif [ "${{ matrix.db_type }}" == "postgres" ]; then - until docker exec $(docker ps -qf "name=keep-database") pg_isready -h localhost -U keepuser; do - echo "Waiting for Postgres to be ready..." - sleep 2 - done - fi - - # wait to keep backend on port 8080 - echo "Waiting for Keep backend to be ready..." - attempt=0 - max_attempts=10 - - until $(curl --output /dev/null --silent --fail http://localhost:8080/healthcheck); do - if [ "$attempt" -ge "$max_attempts" ]; then - echo "Max attempts reached, exiting... Sometimes Keep can't start because of double-headed migrations, use: 'alembic -c keep/alembic.ini history' to investigate, or check artifacts." - exit 1 - fi - echo "Waiting for Keep backend to be ready... (Attempt: $((attempt+1)))" - attempt=$((attempt+1)) - sleep 2 - done - - echo "Keep backend is ready!" - # wait to the backend - echo "Waiting for Keep frontend to be ready..." - attempt=0 - max_attempts=10 - - until $(curl --output /dev/null --silent --fail http://localhost:3000/); do - if [ "$attempt" -ge "$max_attempts" ]; then - echo "Max attempts reached, exiting..." - exit 1 - fi - echo "Waiting for Keep frontend to be ready... (Attempt: $((attempt+1)))" - attempt=$((attempt+1)) - sleep 2 - done - - # create the state directory - # mkdir -p ./state && chown -R root:root ./state && chmod -R 777 ./state - - - name: Run e2e tests and report coverage - run: | - poetry run coverage run --branch -m pytest -s tests/e2e_tests/ - - - name: Convert coverage results to JSON (for CodeCov support) - run: poetry run coverage json --omit="keep/providers/*" - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - fail_ci_if_error: false # don't fail if we didn't manage to upload the coverage report - files: coverage.json - verbose: true - - - name: Dump backend logs - if: always() - run: | - docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-backend > backend_logs-${{ matrix.db_type }}.txt - docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-frontend > frontend_logs-${{ matrix.db_type }}.txt - continue-on-error: true - - - name: Upload test artifacts on failure - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-artifacts - path: | - playwright_dump_*.html - playwright_dump_*.png - backend_logs-${{ matrix.db_type }}.txt - frontend_logs-${{ matrix.db_type }}.txt - continue-on-error: true - - - name: Tear down environment - run: | - docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml down diff --git a/docker/Dockerfile.ui b/docker/Dockerfile.ui index 6b0d4210a..7a156a3bb 100644 --- a/docker/Dockerfile.ui +++ b/docker/Dockerfile.ui @@ -1,6 +1,4 @@ - - -FROM node:18-alpine AS base +FROM node:20-alpine AS base # Install dependencies only when needed FROM base AS deps @@ -26,7 +24,7 @@ ENV NEXT_TELEMETRY_DISABLED 1 # If using npm comment out above and use below instead ENV API_URL http://localhost:8080 -RUN npm run build +RUN NODE_OPTIONS="--max-old-space-size=8192" npm run build # Production image, copy all the files and run next diff --git a/keep-ui/.dockerignore b/keep-ui/.dockerignore new file mode 100644 index 000000000..d38b75ee9 --- /dev/null +++ b/keep-ui/.dockerignore @@ -0,0 +1,8 @@ +node_modules +.next +.vercel +.env.* +.venv/ +.vscode/ +.github/ +.pytest_cache diff --git a/keep-ui/app/(keep)/page.tsx b/keep-ui/app/(keep)/page.tsx index 8c4ad84ad..f56dbe805 100644 --- a/keep-ui/app/(keep)/page.tsx +++ b/keep-ui/app/(keep)/page.tsx @@ -4,5 +4,4 @@ export const metadata = { title: "Keep", description: "The open-source AIOps and alert management platform.", }; - export default ProvidersPage; diff --git a/keep/api/api.py b/keep/api/api.py index e0f9cf0fd..9d5ca0d76 100644 --- a/keep/api/api.py +++ b/keep/api/api.py @@ -285,3 +285,4 @@ def run(app: FastAPI): port=PORT, log_config=logging_config, ) + diff --git a/tests/e2e_tests/docker-compose-e2e-postgres.yml b/tests/e2e_tests/docker-compose-e2e-postgres.yml index 182ca6be2..fc2b0d0c2 100644 --- a/tests/e2e_tests/docker-compose-e2e-postgres.yml +++ b/tests/e2e_tests/docker-compose-e2e-postgres.yml @@ -20,6 +20,11 @@ services: build: context: ./keep-ui/ dockerfile: ../docker/Dockerfile.ui + x-bake: + cache-from: + - type=gha,scope=frontend + cache-to: + - type=gha,mode=max,scope=frontend environment: - AUTH_TYPE=NO_AUTH - API_URL=http://keep-backend:8080 @@ -33,6 +38,11 @@ services: build: context: . dockerfile: docker/Dockerfile.api + x-bake: + cache-from: + - type=gha,scope=backend + cache-to: + - type=gha,mode=max,scope=backend environment: - AUTH_TYPE=NO_AUTH - DATABASE_CONNECTION_STRING=postgresql://keepuser:keeppassword@keep-database:5432/keepdb