From ee0c7895dab9bb206eeff6b97ca6fee965f18450 Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Sat, 2 Sep 2023 14:01:55 +0300 Subject: [PATCH] Separate unit and integration test runs (#98) --- .dx/.yamllint.yml | 7 ++++++ .dx/README.adoc | 1 - .dx/binaries.yml | 14 +++++------ .dx/group_vars/all.yml | 1 + .dx/images.yml | 28 ++++++++++++++++++--- .dx/pipeline.yml | 4 +-- .dx/sources.yml | 9 +++++++ .dx/stack.yml | 12 ++++----- .dx/tasks/pipeline/package.yml | 2 +- .dx/tasks/stack/package.yml | 2 +- .env/amber/.env | 2 +- .env/amber/requirements.txt | 1 + .env/green/.env | 2 +- .env/green/requirements.txt | 1 + .github/actions/prepare/action.yml | 6 +++++ .github/dependabot.yml | 7 +++++- .github/workflows/pipeline-beyond-doubt.yml | 6 ++++- .github/workflows/pipeline-minimal.yml | 6 ++++- .github/workflows/solution-beyond-doubt.yml | 13 ++++++---- .github/workflows/solution-minimal.yml | 3 ++- README.adoc | 7 +++--- apps/pom.xml | 6 +---- libs/pom.xml | 9 ------- tests/pom.xml | 1 - tools/pom.xml | 1 - 25 files changed, 99 insertions(+), 52 deletions(-) create mode 100644 .dx/.yamllint.yml delete mode 100644 .dx/README.adoc diff --git a/.dx/.yamllint.yml b/.dx/.yamllint.yml new file mode 100644 index 00000000..e1353bee --- /dev/null +++ b/.dx/.yamllint.yml @@ -0,0 +1,7 @@ +extends: default + +rules: + line-length: + max: 120 + level: warning + truthy: disable diff --git a/.dx/README.adoc b/.dx/README.adoc deleted file mode 100644 index 4519021b..00000000 --- a/.dx/README.adoc +++ /dev/null @@ -1 +0,0 @@ -Developer Experience \ No newline at end of file diff --git a/.dx/binaries.yml b/.dx/binaries.yml index 25737c30..0a49125e 100644 --- a/.dx/binaries.yml +++ b/.dx/binaries.yml @@ -24,7 +24,7 @@ ansible.builtin.debug: msg: "{{ binary_status.cmd|join(' ') }}" when: binary_status is changed - - name: Create binary + - name: Test and create binaries ansible.builtin.command: cmd: > mvn @@ -55,7 +55,7 @@ register: image_tag changed_when: false tags: [always] - - name: Capture status + - name: Capture statuses ansible.builtin.command: cmd: docker {{ docker_entity }} inspect {{ item.value }}:{{ image_tag.stdout[:7] }} register: image_status @@ -70,7 +70,7 @@ msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" when: image_status is changed tags: [always] - - name: Create binary + - name: Test units and create binaries ansible.builtin.command: cmd: > mvn @@ -78,13 +78,13 @@ --fail-fast --batch-mode --threads 2C - {% if not hostvars.lib.binary_status is changed %} + {% if not hostvars.lib.binary_status|default({}) is changed %} --projects {{ image_status.results|select('changed')|map(attribute='item.key')|join(',') }} {% endif %} --define revision={{ app_version }} --define lib.version={{ lib_version }} clean - verify + package chdir: "{{ playbook_dir }}/../apps" strip_empty_ends: false changed_when: true @@ -108,5 +108,5 @@ strip_empty_ends: false changed_when: true when: > - hostvars.lib.binary_status is changed or - hostvars.app.image_status is changed + hostvars.lib.binary_status|default({}) is changed or + hostvars.app.image_status|default({}) is changed diff --git a/.dx/group_vars/all.yml b/.dx/group_vars/all.yml index 322a60ab..6d88c64f 100644 --- a/.dx/group_vars/all.yml +++ b/.dx/group_vars/all.yml @@ -19,6 +19,7 @@ app_version: 0.1.0-{{ app_cid }} docker_registry: localhost docker_repo: smecalculus +# TODO: introduce enum like variable is_local_building: "{{ docker_registry == 'localhost' }}" is_remote_building: "{{ docker_registry != 'localhost' }}" diff --git a/.dx/images.yml b/.dx/images.yml index 2a6620c0..f05f4460 100644 --- a/.dx/images.yml +++ b/.dx/images.yml @@ -1,11 +1,11 @@ --- -- import_playbook: binaries.yml # noqa: name[play] +- import_playbook: binaries.yml # noqa: name[play] tags: [binaries, deps] - name: Images hosts: schema tasks: - - name: Capture status + - name: Capture statuses ansible.builtin.command: cmd: docker {{ docker_entity }} inspect {{ item.value }}:{{ schema_cid }} register: image_status @@ -18,7 +18,8 @@ ansible.builtin.debug: msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" when: image_status is changed - - ansible.builtin.include_role: # noqa: name[missing] + - name: Create packages + ansible.builtin.include_role: name: image vars: image_tag: "{{ schema_cid }}" @@ -29,7 +30,26 @@ - name: Images hosts: app tasks: - - ansible.builtin.include_role: # noqa: name[missing] + - name: Test integrations + ansible.builtin.command: + cmd: > + mvn + --no-snapshot-updates + --fail-fast + --batch-mode + --threads 2C + {% if not hostvars.lib.binary_status|default({}) is changed %} + --projects {{ image_status.results|select('changed')|map(attribute='item.key')|join(',') }} + {% endif %} + --define lib.version={{ lib_version }} + failsafe:integration-test + failsafe:verify + chdir: "{{ playbook_dir }}/../apps" + strip_empty_ends: false + changed_when: true + when: image_status is changed + - name: Create packages + ansible.builtin.include_role: name: image vars: image_tag: "{{ hostvars.app.image_tag.stdout[:7] }}" diff --git a/.dx/pipeline.yml b/.dx/pipeline.yml index 93c76b25..1595f923 100644 --- a/.dx/pipeline.yml +++ b/.dx/pipeline.yml @@ -3,8 +3,8 @@ hosts: pipeline run_once: true tasks: - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/pipeline/status.yml - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/pipeline/package.yml when: pipeline_status is changed diff --git a/.dx/sources.yml b/.dx/sources.yml index 01085328..40f663ff 100644 --- a/.dx/sources.yml +++ b/.dx/sources.yml @@ -67,3 +67,12 @@ pipeline.yml strip_empty_ends: false changed_when: false + +- name: Sources + hosts: pipeline + tasks: + - name: Analyze + ansible.builtin.command: + cmd: yamllint ../.github + strip_empty_ends: false + changed_when: false diff --git a/.dx/stack.yml b/.dx/stack.yml index e0114758..d18d8b3f 100644 --- a/.dx/stack.yml +++ b/.dx/stack.yml @@ -1,31 +1,31 @@ --- -- import_playbook: images.yml # noqa: name[play] +- import_playbook: images.yml # noqa: name[play] tags: [images, deps] - name: Stack hosts: stack run_once: true tasks: - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/pipeline/status.yml tags: [always] - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/stack/status.yml tags: [always] - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/stack/compile.yml tags: [compile] # remotely compile stack even if it hasn't changed (it's fast enough) when: > is_remote_building or stack_status is changed - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/stack/test.yml tags: [test] when: > stack_status is changed or pipeline_status is changed - - ansible.builtin.import_tasks: # noqa: name[missing] + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/stack/package.yml tags: [package] when: stack_status is changed diff --git a/.dx/tasks/pipeline/package.yml b/.dx/tasks/pipeline/package.yml index 3b78fc47..47a806d0 100644 --- a/.dx/tasks/pipeline/package.yml +++ b/.dx/tasks/pipeline/package.yml @@ -1,5 +1,5 @@ --- -- ansible.builtin.import_role: # noqa: name[missing] +- ansible.builtin.import_role: # noqa: name[missing] name: image vars: image_tag: "{{ pipeline_tag.stdout[:7] }}" diff --git a/.dx/tasks/stack/package.yml b/.dx/tasks/stack/package.yml index f3634fda..998406f9 100644 --- a/.dx/tasks/stack/package.yml +++ b/.dx/tasks/stack/package.yml @@ -1,5 +1,5 @@ --- -- ansible.builtin.import_role: # noqa: name[missing] +- ansible.builtin.import_role: # noqa: name[missing] name: image vars: image_tag: "{{ stack_tag.stdout[:7] }}" diff --git a/.env/amber/.env b/.env/amber/.env index 42c294d6..2d00f69c 100644 --- a/.env/amber/.env +++ b/.env/amber/.env @@ -1,2 +1,2 @@ -PYTHON_VERSION=3.8 JAVA_VERSION=17.0 +PYTHON_VERSION=3.8 diff --git a/.env/amber/requirements.txt b/.env/amber/requirements.txt index 0963342b..4ca8dd87 100644 --- a/.env/amber/requirements.txt +++ b/.env/amber/requirements.txt @@ -1,2 +1,3 @@ ansible-core==2.13.10 ansible-lint==6.11.0 +yamllint==1.32.0 diff --git a/.env/green/.env b/.env/green/.env index f1a6ecde..ef37bb1c 100644 --- a/.env/green/.env +++ b/.env/green/.env @@ -1,2 +1,2 @@ -PYTHON_VERSION=3.9 JAVA_VERSION=19.0 +PYTHON_VERSION=3.9 diff --git a/.env/green/requirements.txt b/.env/green/requirements.txt index 70c99061..c37c5a1b 100644 --- a/.env/green/requirements.txt +++ b/.env/green/requirements.txt @@ -1,2 +1,3 @@ ansible-core==2.15.3 ansible-lint==6.18.0 +yamllint==1.32.0 diff --git a/.github/actions/prepare/action.yml b/.github/actions/prepare/action.yml index fc12ba63..9726caa6 100644 --- a/.github/actions/prepare/action.yml +++ b/.github/actions/prepare/action.yml @@ -1,10 +1,16 @@ +--- +name: Prepare tools +description: Tools preparation action for specific environment inputs: name: + description: Environment name required: true default: green tools: + description: Tools to prepare required: true token: + description: Github token required: false runs: using: "composite" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 030f5d25..7c31e846 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,3 +1,4 @@ +--- version: 2 updates: - package-ecosystem: "github-actions" @@ -34,6 +35,10 @@ updates: schedule: interval: "weekly" - package-ecosystem: "pip" - directory: "/" + directory: "/.env/green" + schedule: + interval: "weekly" + - package-ecosystem: "pip" + directory: "/.env/amber" schedule: interval: "weekly" diff --git a/.github/workflows/pipeline-beyond-doubt.yml b/.github/workflows/pipeline-beyond-doubt.yml index 42f9d241..3b960fc2 100644 --- a/.github/workflows/pipeline-beyond-doubt.yml +++ b/.github/workflows/pipeline-beyond-doubt.yml @@ -1,9 +1,10 @@ +--- name: Pipeline beyond doubt on: pull_request: branches: [main] - paths: # waiting for https://github.com/actions/runner/issues/2324 + paths: # waiting for https://github.com/actions/runner/issues/2324 - .dx/** - .env/** - .github/** @@ -41,6 +42,9 @@ jobs: ansible-playbook sources.yml -l dx:test -e docker_registry=${{ env.REGISTRY }} working-directory: .dx + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > ansible-playbook stack.yml --skip-tags package diff --git a/.github/workflows/pipeline-minimal.yml b/.github/workflows/pipeline-minimal.yml index 852eefc9..ae10ae61 100644 --- a/.github/workflows/pipeline-minimal.yml +++ b/.github/workflows/pipeline-minimal.yml @@ -1,9 +1,10 @@ +--- name: Pipeline minimal on: pull_request: branches-ignore: [main] - paths: # waiting for https://github.com/actions/runner/issues/2324 + paths: # waiting for https://github.com/actions/runner/issues/2324 - .dx/** - .env/** - .github/** @@ -35,3 +36,6 @@ jobs: ansible-playbook sources.yml -l dx:test -e docker_registry=${{ env.REGISTRY }} working-directory: .dx + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/solution-beyond-doubt.yml b/.github/workflows/solution-beyond-doubt.yml index 02b8fd5a..21c008df 100644 --- a/.github/workflows/solution-beyond-doubt.yml +++ b/.github/workflows/solution-beyond-doubt.yml @@ -1,3 +1,4 @@ +--- name: Solution beyond doubt on: @@ -5,7 +6,7 @@ on: branches: [main] pull_request: branches: [main] - paths: # waiting for https://github.com/actions/runner/issues/2324 + paths: # waiting for https://github.com/actions/runner/issues/2324 - libs/** - apps/** - schemas/** @@ -46,10 +47,6 @@ jobs: checks: write packages: write pull-requests: write - env: - JACOCO_UNITS: tools/target/coverage/units/jacoco.xml - JACOCO_INTEGRATIONS: tools/target/coverage/integrations/jacoco.xml - JACOCO_OVERALL: tools/target/coverage/overall/jacoco.xml steps: - uses: actions/checkout@v3.6.0 - uses: ./.github/actions/prepare @@ -96,6 +93,8 @@ jobs: paths: ${{ env.JACOCO_UNITS }} token: ${{ secrets.GITHUB_TOKEN }} update-comment: true + env: + JACOCO_UNITS: tools/target/coverage/units/jacoco.xml - uses: madrapps/jacoco-report@v1.6 if: ${{ github.event_name == 'pull_request' && hashFiles(env.JACOCO_INTEGRATIONS) != '' }} with: @@ -103,6 +102,8 @@ jobs: paths: ${{ env.JACOCO_INTEGRATIONS }} token: ${{ secrets.GITHUB_TOKEN }} update-comment: true + env: + JACOCO_INTEGRATIONS: tools/target/coverage/integrations/jacoco.xml - uses: madrapps/jacoco-report@v1.6 if: ${{ github.event_name == 'pull_request' && hashFiles(env.JACOCO_OVERALL) != '' }} with: @@ -110,6 +111,8 @@ jobs: paths: ${{ env.JACOCO_OVERALL }} token: ${{ secrets.GITHUB_TOKEN }} update-comment: true + env: + JACOCO_OVERALL: tools/target/coverage/overall/jacoco.xml stack-compiling: strategy: diff --git a/.github/workflows/solution-minimal.yml b/.github/workflows/solution-minimal.yml index 00709bc6..51f6945e 100644 --- a/.github/workflows/solution-minimal.yml +++ b/.github/workflows/solution-minimal.yml @@ -1,9 +1,10 @@ +--- name: Solution minimal on: pull_request: branches-ignore: [main] - paths: # waiting for https://github.com/actions/runner/issues/2324 + paths: # waiting for https://github.com/actions/runner/issues/2324 - libs/** - apps/** - schemas/** diff --git a/README.adoc b/README.adoc index 3c19501c..c7a83632 100644 --- a/README.adoc +++ b/README.adoc @@ -15,7 +15,7 @@ ==== Минимальная (minimal) -Запускаем линтеры, статический анализ и компиляцию для исходного кода решения. +Запускаем линтинг, статический анализ и компиляцию (по желанию) исходного кода решения. ==== В пределах разумного (reasonable) @@ -27,7 +27,7 @@ ==== За пределами сомнений (beyond doubt) -Все перечисленное ранее. Плюс запускаем сквозные тесты, собираем и публикуем стеки. +Все перечисленное ранее. Плюс запускаем сквозные тесты, собираем и публикуем конфигурации решения (стеки). === Проверка конвейера @@ -35,7 +35,7 @@ ==== Минимальная (minimal) -Запускаем линтеры для исходного кода конвейера. +Запускаем линтинг исходного кода конвейера. ==== За пределами сомнений (beyond doubt) @@ -48,6 +48,7 @@ == Todo . Релизная сборка +. Параллелизация модульных тестов . Версионирование библиотек и приложений (взглянуть критически) . Агрегация отчетов покрытия (взглянуть критически) . Локальные проверки в удаленном репозитории diff --git a/apps/pom.xml b/apps/pom.xml index 8014c012..cc02736b 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -54,15 +54,12 @@ - org.apache.maven.plugins maven-dependency-plugin - org.apache.maven.plugins maven-surefire-plugin - org.apache.maven.plugins maven-failsafe-plugin @@ -106,7 +103,6 @@ - org.apache.maven.plugins maven-failsafe-plugin 3.1.2 @@ -183,7 +179,7 @@ - 2.30.0 + 2.35.0 diff --git a/libs/pom.xml b/libs/pom.xml index 343fdae7..7969060b 100644 --- a/libs/pom.xml +++ b/libs/pom.xml @@ -307,7 +307,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -315,7 +314,6 @@ flatten-maven-plugin - org.apache.maven.plugins maven-dependency-plugin @@ -323,7 +321,6 @@ spotless-maven-plugin - org.apache.maven.plugins maven-enforcer-plugin @@ -349,7 +346,6 @@ - org.apache.maven.plugins maven-compiler-plugin 3.11.0 @@ -377,12 +373,10 @@ ${kotlin.version} - org.apache.maven.plugins maven-resources-plugin 3.3.1 - org.apache.maven.plugins maven-surefire-plugin 3.1.2 @@ -390,7 +384,6 @@ - org.apache.maven.plugins maven-failsafe-plugin 3.1.2 @@ -430,7 +423,6 @@ - org.apache.maven.plugins maven-dependency-plugin 3.6.0 @@ -505,7 +497,6 @@ - org.apache.maven.plugins maven-enforcer-plugin 3.4.0 diff --git a/tests/pom.xml b/tests/pom.xml index 2f2d4786..841c468d 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -81,7 +81,6 @@ - org.apache.maven.plugins maven-surefire-plugin 3.1.2 diff --git a/tools/pom.xml b/tools/pom.xml index 1c843afe..32965e87 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -12,7 +12,6 @@ - org.apache.maven.plugins maven-antrun-plugin 3.1.0