[DOP-22425] Do not run tests as admin DB users

MobileTeleSystems · Feb 20, 2025 · 91e72c4 · 91e72c4
1 parent 10e221c
commit 91e72c4
Show file tree

Hide file tree

Showing 33 changed files with 439 additions and 313 deletions.
diff --git a/.env.docker b/.env.docker
@@ -1,10 +1,11 @@
 TZ=UTC
+SPARK_EXTERNAL_IP=onetl
 
 # Greenplum
 ONETL_GP_HOST=greenplum
 ONETL_GP_PORT=5432
-ONETL_GP_DATABASE=postgres
-ONETL_GP_USER=gpadmin
+ONETL_GP_DATABASE=onetl
+ONETL_GP_USER=onetl
 ONETL_GP_PASSWORD=123UsedForTestOnly@!
 
 # ClickHouse
@@ -25,9 +26,12 @@ ONETL_KAFKA_PASSWORD=123UsedForTestOnly@!
 # Mongo
 ONETL_MONGO_HOST=mongodb
 ONETL_MONGO_PORT=27017
-ONETL_MONGO_DB=admin
+ONETL_MONGO_DB=onetl
 ONETL_MONGO_USER=onetl
 ONETL_MONGO_PASSWORD=123UsedForTestOnly@!
+ONETL_MONGO_ROOT_DB=admin
+ONETL_MONGO_ROOT_USER=admin
+ONETL_MONGO_ROOT_PASSWORD=123AdminUserForTests@!
 
 # MSSQL
 ONETL_MSSQL_HOST=mssql

diff --git a/.env.local b/.env.local
@@ -1,10 +1,11 @@
 export TZ=UTC
+export SPARK_EXTERNAL_IP=$(docker network inspect onetl_onetl --format '{{ (index .IPAM.Config 0).Gateway }}')
 
 # Greenplum
 export ONETL_GP_HOST=localhost
 export ONETL_GP_PORT=5433
-export ONETL_GP_DATABASE=postgres
-export ONETL_GP_USER=gpadmin
+export ONETL_GP_DATABASE=onetl
+export ONETL_GP_USER=onetl
 export ONETL_GP_PASSWORD=123UsedForTestOnly@!
 
 # ClickHouse
@@ -25,9 +26,12 @@ export ONETL_KAFKA_PASSWORD=123UsedForTestOnly@!
 # Mongo
 export ONETL_MONGO_HOST=localhost
 export ONETL_MONGO_PORT=27017
-export ONETL_MONGO_DB=admin
+export ONETL_MONGO_DB=onetl
 export ONETL_MONGO_USER=onetl
 export ONETL_MONGO_PASSWORD=123UsedForTestOnly@!
+export ONETL_MONGO_ROOT_DB=admin
+export ONETL_MONGO_ROOT_USER=admin
+export ONETL_MONGO_ROOT_PASSWORD=123AdminUserForTests@!
 
 # MSSQL
 export ONETL_MSSQL_HOST=localhost

diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml
@@ -32,25 +32,19 @@ jobs:
   test-clickhouse:
     name: Run Clickhouse tests (server=${{ inputs.clickhouse-version }}, spark=${{ inputs.spark-version }}, pydantic=${{ inputs.pydantic-version }}, java=${{ inputs.java-version }}, python=${{ inputs.python-version }}, os=${{ inputs.os }})
     runs-on: ${{ inputs.os }}
-    services:
-      clickhouse:
-        image: ${{ inputs.clickhouse-image }}:${{ inputs.clickhouse-version }}
-        env:
-          TZ: UTC
-          # yandex/clickhouse-server doesn't read these options at all, so we using default values
-          CLICKHOUSE_DB: default
-          CLICKHOUSE_USER: default
-          CLICKHOUSE_PASSWORD: 123UsedForTestOnly@!
-          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
-        ports:
-          - 8123:8123
-          - 9001:9000
-        options: --sysctl net.ipv6.conf.all.disable_ipv6=1
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start Clickhouse
+        run: |
+          docker compose --profile all down -v --remove-orphans
+          docker compose --profile clickhouse pull
+          docker compose --profile clickhouse up -d --wait --wait-timeout 200
+        env:
+          CLICKHOUSE_IMAGE: ${{ inputs.clickhouse-image }}:${{ inputs.clickhouse-version }}
+
       - name: Set up Java ${{ inputs.java-version }}
         uses: actions/setup-java@v4
         with:
@@ -92,8 +86,7 @@ jobs:
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           ./pytest_runner.sh -m clickhouse
 
       - name: Dump Clickhouse logs on failure
@@ -110,6 +103,11 @@ jobs:
           name: container-logs-clickhouse-${{ inputs.clickhouse-version }}-spark-${{ inputs.spark-version }}
           path: logs/*
 
+      - name: Shutdown Clickhouse
+        if: always()
+        run: |
+          docker compose --profile all down -v --remove-orphans
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-ftp.yml b/.github/workflows/test-ftp.yml
@@ -28,6 +28,14 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start FTP
+        run: |
+          docker compose --profile all down -v --remove-orphans
+          docker compose --profile ftp pull
+          docker compose --profile ftp up -d --wait --wait-timeout 200
+        env:
+          FTP_IMAGE: chonjay21/ftps:${{ inputs.ftp-version }}
+
       - name: Set up Python ${{ inputs.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -50,21 +58,10 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/ftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-      # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftp/pull/3
-      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-      # See https://github.com/orgs/community/discussions/25792
-      - name: Start FTP
-        run: |
-          docker compose down -v --remove-orphans
-          docker compose up -d ftp --wait --wait --wait-timeout 200
-        env:
-          FTP_IMAGE: chonjay21/ftps:${{ inputs.ftp-version }}
-
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           ./pytest_runner.sh -m ftp
 
       - name: Dump FTP logs on failure
@@ -86,7 +83,7 @@ jobs:
       - name: Shutdown FTP
         if: always()
         run: |
-          docker compose down -v --remove-orphans
+          docker compose --profile all down -v --remove-orphans
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-ftps.yml b/.github/workflows/test-ftps.yml
@@ -28,6 +28,12 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start FTPS
+        run: |
+          docker compose --profile all down -v --remove-orphans
+          docker compose --profile ftps pull
+          docker compose --profile ftps up -d --wait --wait-timeout 200
+
       - name: Set up Python ${{ inputs.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -49,22 +55,13 @@ jobs:
       - name: Install dependencies
         run: |
           pip install -I -r requirements/core.txt -r requirements/ftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
-
-      # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftps/pull/3
-      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-      # See https://github.com/orgs/community/discussions/25792
-      - name: Start FTPS
-        run: |
-          docker compose down -v --remove-orphans
-          docker compose up -d ftps --wait --wait --wait-timeout 200
         env:
           FTPS_IMAGE: chonjay21/ftps:${{ inputs.ftps-version }}
 
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           ./pytest_runner.sh -m ftps
 
       - name: Dump FTPS logs on failure
@@ -84,7 +81,7 @@ jobs:
       - name: Shutdown FTPS
         if: always()
         run: |
-          docker compose down -v --remove-orphans
+          docker compose --profile all down -v --remove-orphans
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-greenplum.yml b/.github/workflows/test-greenplum.yml
@@ -38,33 +38,19 @@ jobs:
     if: github.repository == 'MobileTeleSystems/onetl'  # prevent running on forks
     name: Run Greenplum tests (server=${{ inputs.greenplum-version }}, spark=${{ inputs.spark-version }}, pydantic=${{ inputs.pydantic-version }}, java=${{ inputs.java-version }}, python=${{ inputs.python-version }}, os=${{ inputs.os }})
     runs-on: ${{ inputs.os }}
-    services:
-      greenplum:
-        image: andruche/greenplum:${{ inputs.greenplum-version }}
-        env:
-          TZ: UTC
-        ports:
-          - 5433:5432
-        # TODO: remove after https://github.com/andruche/docker-greenplum/pull/2
-        options: --sysctl net.ipv6.conf.all.disable_ipv6=1
-      pgbouncer-transaction-gp:
-        image: bitnami/pgbouncer:latest
-        env:
-          TZ: UTC
-          PGBOUNCER_DATABASE: postgres
-          PGBOUNCER_POOL_MODE: transaction
-          POSTGRESQL_HOST: greenplum
-          POSTGRESQL_PORT: 5432
-          POSTGRESQL_DATABASE: postgres
-          POSTGRESQL_USERNAME: gpadmin
-          POSTGRESQL_PASSWORD: 123UsedForTestOnly@!
-        ports:
-          - 6433:6432
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start Grenplum
+        run: |
+          docker compose --profile all down -v --remove-orphans
+          docker compose --profile greenplum pull
+          docker compose --profile greenplum up -d --wait --wait-timeout 200
+        env:
+          GREENPLUM_IMAGE: andruche/greenplum:${{ inputs.greenplum-version }}
+
       - name: Set up Java ${{ inputs.java-version }}
         uses: actions/setup-java@v4
         with:
@@ -111,8 +97,7 @@ jobs:
 
       - name: Wait for Greenplum to be ready
         run: |
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
 
           # Greenplum init script is running very late
           sleep 30
@@ -123,8 +108,7 @@ jobs:
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           export ONETL_GP_PACKAGE_VERSION=${{ inputs.package-version }}
           ./pytest_runner.sh -m greenplum
         env:
@@ -134,8 +118,7 @@ jobs:
       - name: Run tests with PGBouncer
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           export ONETL_GP_PACKAGE_VERSION=${{ inputs.package-version }}
           export ONETL_GP_PORT=6433
           # Run only some basic tests
@@ -158,6 +141,11 @@ jobs:
           name: container-logs-greenplum-${{ inputs.greenplum-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
           path: logs/*
 
+      - name: Shutdown Greenplum
+        if: always()
+        run: |
+          docker compose --profile all down -v --remove-orphans
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-hdfs.yml b/.github/workflows/test-hdfs.yml
@@ -34,6 +34,12 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start HDFS
+        run: |
+          docker compose --profile all down -v --remove-orphans
+          docker compose --profile hdfs pull
+          docker compose --profile hdfs up -d --wait --wait-timeout 200
+
       - name: Set up Java ${{ inputs.java-version }}
         uses: actions/setup-java@v4
         with:
@@ -69,24 +75,13 @@ jobs:
       - name: Install dependencies
         run: |
           pip install -I -r requirements/core.txt -r requirements/kerberos.txt -r requirements/hdfs.txt -r requirements/tests/base.txt -r requirements/tests/spark-${{ inputs.spark-version }}.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
-
-      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-      # See https://github.com/orgs/community/discussions/25792
-      - name: Start HDFS
-        run: |
-          docker compose down -v --remove-orphans
-          docker compose up -d hdfs --wait --wait-timeout 200 &
-          wait_pid=$!
-          docker compose logs -f hdfs &
-          wait $wait_pid
         env:
           HDFS_IMAGE: mtsrus/hadoop:${{ inputs.hadoop-version }}
 
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           echo "127.0.0.1 hdfs" | sudo tee -a /etc/hosts
           ./pytest_runner.sh -m hdfs
 
@@ -107,7 +102,7 @@ jobs:
       - name: Shutdown HDFS
         if: always()
         run: |
-          docker compose down -v --remove-orphans
+          docker compose --profile all down -v --remove-orphans
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-hive.yml b/.github/workflows/test-hive.yml
@@ -72,8 +72,7 @@ jobs:
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"
-          sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
-          source ./env
+          source .env.local
           ./pytest_runner.sh -m hive
 
       - name: Upload coverage results