Merge branch 'main' into mon_wmde12-1-38-7

wmde · Oct 31, 2023 · 57b5dac · 57b5dac
2 parents 23c1415 + 86c994a
commit 57b5dac
Show file tree

Hide file tree

Showing 351 changed files with 18,324 additions and 11,996 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -2,7 +2,9 @@
 cache/*
 git_cache/*
 Docker/**/*.tar.gz
+test/suites/**/results
 test/log
-diagrams/node_modules/
+test/mwlog
+docs/diagrams/node_modules/
 artifacts/*
-docs/*
+docs/*
diff --git a/.envrc b/.envrc
@@ -0,0 +1 @@
+use_flake
diff --git a/.githooks/pre-commit b/.githooks/pre-commit
@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+./test/scripts/lint.sh
diff --git a/test/reporter/.gitignore → .github/reporter/.gitignore b/test/reporter/.gitignore → .github/reporter/.gitignore
diff --git a/test/reporter/package-lock.json → .github/reporter/package-lock.json b/test/reporter/package-lock.json → .github/reporter/package-lock.json
diff --git a/test/reporter/package.json → .github/reporter/package.json b/test/reporter/package.json → .github/reporter/package.json
diff --git a/test/reporter/report.js → .github/reporter/report.js b/test/reporter/report.js → .github/reporter/report.js
@@ -7,20 +7,20 @@ if( !process.env.SUITE ) {
     return;
 }
 
-const filePath = '../log/selenium/result-'+ process.env.SUITE + '.json';
+const filePath = `../../test/suites/${process.env.SUITE}/results/result.json`;
 
 var resultObject = {};
 
 if (fs.existsSync(filePath)) {
-    resultObject = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+    resultObject = JSON.parse(fs.readFileSync(filePath, 'utf8'))[process.env.SUITE];
 
     if (resultObject.fail.length != 0) {
 
         resultObject.fail.forEach(test => {
             const error = extractModuleLineAndColumn(test.error.stack);
             let filePath = '';
             if( error.file ) {
-                filePath = error.file.replace('/usr/src/app/', 'Docker/test/selenium/');
+                filePath = error.file.replace('/usr/src/app/', 'test/');
             }
             const message = test.fullTitle + ": " + test.error.message;
 

diff --git a/.github/workflows/built_and_test.yml → .github/workflows/build_and_test.yml b/.github/workflows/built_and_test.yml → .github/workflows/build_and_test.yml
@@ -17,7 +17,8 @@ on:
   pull_request:
     paths-ignore:
       - 'docs/**'
-      - 'README.md'
+      - '**/*.md'
+      - '**/*.txt'
 
 env:
   env_file: ${{ github.event.inputs.env_file || 'versions/wmde12.env' }}
@@ -28,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -78,7 +79,7 @@ jobs:
           path: artifacts/Wikibase.tar.gz
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -112,7 +113,7 @@ jobs:
     needs:
       - build_wikibase
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -154,7 +155,7 @@ jobs:
         run: bash build.sh wikibase_bundle ${{ env.env_file }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -193,7 +194,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -228,7 +229,7 @@ jobs:
           path: artifacts/quickstatements.docker.tar.gz
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -253,7 +254,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -282,7 +283,7 @@ jobs:
         run: bash build.sh wdqs ${{ env.env_file }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -314,7 +315,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -334,7 +335,7 @@ jobs:
         run: bash build.sh wdqs-proxy ${{ env.env_file }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -366,7 +367,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -415,7 +416,7 @@ jobs:
           path: ./artifacts/wdqs-frontend.docker.tar.gz
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -440,7 +441,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Add extra DEBUG env vars
         if: "${{ github.event.inputs.debug_extra_env != '' }}"
@@ -467,7 +468,7 @@ jobs:
           path: ./artifacts/elasticsearch.docker.tar.gz
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
@@ -492,7 +493,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        databaseImageName: [ 'mariadb:10.9' ] # 'mysql:5.6' disabled https://phabricator.wikimedia.org/T296066
         suite: [ 
           repo, 
           fedprops,
@@ -507,6 +507,7 @@ jobs:
           base__pingback,
           base__fedprops,
         ]
+        databaseImageName: [ 'mariadb:10.9' ] # 'mysql:5.6' disabled https://phabricator.wikimedia.org/T296066
     needs:
       - build_wikibase_bundle
       - build_wdqs
@@ -517,7 +518,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Get Wikibase docker image artifact
         uses: actions/download-artifact@v3
         with:
@@ -532,27 +533,27 @@ jobs:
       - name: Report Selenium
         if: always()
         run: |
-          cd test/reporter
+          cd .github/reporter
           npm install
           SUITE=${{ matrix.suite }} node report.js
 
       - name: docker logs
         if: always()
-        run: test/docker_logs.sh
+        run: test/scripts/docker_logs.sh
 
       - name: mediawiki logs
         if: always()
         continue-on-error: true
         run: |
-          ls -lahr test/mwlog test/mwlog/*
-          tail -n +1 test/mwlog/*/*.log
+          ls -lahr test/suites/${{ matrix.suite }}/results test/suites/${{ matrix.suite }}/results/*
+          tail -n +1 test/suites/${{ matrix.suite }}/results/*.log
 
       - name: Archive docker test artifacts
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: TestArtifacts
-          path: test/log
+          path: test/suites/**/results
 
   test_upgrade_base:
     timeout-minutes: 20
@@ -569,7 +570,7 @@ jobs:
       - build_wikibase
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Get Wikibase docker image artifact
         uses: actions/download-artifact@v3
         with:
@@ -584,14 +585,15 @@ jobs:
 
       - name: docker logs
         if: always()
-        run: test/docker_logs.sh
+        run: test/scripts/docker_logs.sh
 
       - name: Archive docker test artifacts
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: TestUpgradeArtifacts
-          path: test/log
+          path: test/suites/**/results
+
 
   test_upgrade_bundle:
     timeout-minutes: 20
@@ -610,7 +612,7 @@ jobs:
     env:
       TARGET_WIKIBASE_UPGRADE_IMAGE_NAME: wikibase-bundle
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Get Wikibase docker image artifact
         uses: actions/download-artifact@v3
         with:
@@ -625,22 +627,22 @@ jobs:
 
       - name: docker logs
         if: always()
-        run: test/docker_logs.sh
+        run: test/scripts/docker_logs.sh
 
       - name: Archive docker test artifacts
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: TestUpgradeArtifactsBundle
-          path: test/log
+          path: test/suites/**/results
 
   versions:
     needs:
       - test_wikibase
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: ${{ env.env_file }}

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -5,26 +5,30 @@ on:
     branches:
       - "main"
   pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '*.txt'
 
 jobs:
   lint_dockerfiles:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: hadolint/[email protected]
         with:
           recursive: true
 
   lint_shell_scripts:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: bewuethr/shellcheck-action@v2
 
   lint_selenium_tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
@@ -36,13 +40,13 @@ jobs:
           key: npm-lint-cache
       - name: lint the selenium javascript tests
         run: |
-          cd Docker/test/selenium 
+          cd test
           npm install
           npm run lint
   lint_diagrams:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
@@ -54,11 +58,6 @@ jobs:
           key: npm-lint-cache
       - name: lint the selenium javascript tests
         run: |
-          cd diagrams
+          cd docs/diagrams
           npm install
           npm run lint
-      - name: test building diagram
-        run: |
-          cd diagrams
-          node make_overview.js
-          
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -21,12 +21,12 @@ jobs:
   publish_docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
       - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: ${{ env.env_file }}
       - name: run the publish script
-        run: bash publish/dockerhub.sh
+        run: bash publish/dockerhub.sh