Separate unit and integration test runs (#98)

.dx/.yamllint.yml

@@ -0,0 +1,7 @@
+extends: default
+
+rules:
+  line-length:
+    max: 120
+    level: warning
+  truthy: disable

.dx/binaries.yml

@@ -24,7 +24,7 @@
       ansible.builtin.debug:
         msg: "{{ binary_status.cmd|join(' ') }}"
       when: binary_status is changed
-    - name: Create binary
+    - name: Test and create binaries
       ansible.builtin.command:
         cmd: >
           mvn
@@ -55,7 +55,7 @@
       register: image_tag
       changed_when: false
       tags: [always]
-    - name: Capture status
+    - name: Capture statuses
       ansible.builtin.command:
         cmd: docker {{ docker_entity }} inspect {{ item.value }}:{{ image_tag.stdout[:7] }}
       register: image_status
@@ -70,21 +70,21 @@
         msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}"
       when: image_status is changed
       tags: [always]
-    - name: Create binary
+    - name: Test units and create binaries
       ansible.builtin.command:
         cmd: >
           mvn
           --no-snapshot-updates
           --fail-fast
           --batch-mode
           --threads 2C
-          {% if not hostvars.lib.binary_status is changed %}
+          {% if not hostvars.lib.binary_status|default({}) is changed %}
           --projects {{ image_status.results|select('changed')|map(attribute='item.key')|join(',') }}
           {% endif %}
           --define revision={{ app_version }}
           --define lib.version={{ lib_version }}
           clean
-          verify
+          package
         chdir: "{{ playbook_dir }}/../apps"
         strip_empty_ends: false
       changed_when: true
@@ -108,5 +108,5 @@
         strip_empty_ends: false
       changed_when: true
       when: >
-        hostvars.lib.binary_status is changed or
-        hostvars.app.image_status is changed
+        hostvars.lib.binary_status|default({}) is changed or
+        hostvars.app.image_status|default({}) is changed

.dx/group_vars/all.yml

@@ -19,6 +19,7 @@ app_version: 0.1.0-{{ app_cid }}
 docker_registry: localhost
 docker_repo: smecalculus
 
+# TODO: introduce enum like variable
 is_local_building: "{{ docker_registry == 'localhost' }}"
 is_remote_building: "{{ docker_registry != 'localhost' }}"
 

.dx/images.yml

@@ -1,11 +1,11 @@
 ---
-- import_playbook: binaries.yml # noqa: name[play]
+- import_playbook: binaries.yml   # noqa: name[play]
   tags: [binaries, deps]
 
 - name: Images
   hosts: schema
   tasks:
-    - name: Capture status
+    - name: Capture statuses
       ansible.builtin.command:
         cmd: docker {{ docker_entity }} inspect {{ item.value }}:{{ schema_cid }}
       register: image_status
@@ -18,7 +18,8 @@
       ansible.builtin.debug:
         msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}"
       when: image_status is changed
-    - ansible.builtin.include_role: # noqa: name[missing]
+    - name: Create packages
+      ansible.builtin.include_role:
         name: image
       vars:
         image_tag: "{{ schema_cid }}"
@@ -29,7 +30,26 @@
 - name: Images
   hosts: app
   tasks:
-    - ansible.builtin.include_role: # noqa: name[missing]
+    - name: Test integrations
+      ansible.builtin.command:
+        cmd: >
+          mvn
+          --no-snapshot-updates
+          --fail-fast
+          --batch-mode
+          --threads 2C
+          {% if not hostvars.lib.binary_status|default({}) is changed %}
+          --projects {{ image_status.results|select('changed')|map(attribute='item.key')|join(',') }}
+          {% endif %}
+          --define lib.version={{ lib_version }}
+          failsafe:integration-test
+          failsafe:verify
+        chdir: "{{ playbook_dir }}/../apps"
+        strip_empty_ends: false
+      changed_when: true
+      when: image_status is changed
+    - name: Create packages
+      ansible.builtin.include_role:
         name: image
       vars:
         image_tag: "{{ hostvars.app.image_tag.stdout[:7] }}"

.dx/pipeline.yml

@@ -3,8 +3,8 @@
   hosts: pipeline
   run_once: true
   tasks:
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/pipeline/status.yml
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/pipeline/package.yml
       when: pipeline_status is changed

.dx/sources.yml

@@ -67,3 +67,12 @@
           pipeline.yml
         strip_empty_ends: false
       changed_when: false
+
+- name: Sources
+  hosts: pipeline
+  tasks:
+    - name: Analyze
+      ansible.builtin.command:
+        cmd: yamllint ../.github
+        strip_empty_ends: false
+      changed_when: false

.dx/stack.yml

@@ -1,31 +1,31 @@
 ---
-- import_playbook: images.yml # noqa: name[play]
+- import_playbook: images.yml  # noqa: name[play]
   tags: [images, deps]
 
 - name: Stack
   hosts: stack
   run_once: true
   tasks:
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/pipeline/status.yml
       tags: [always]
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/stack/status.yml
       tags: [always]
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/stack/compile.yml
       tags: [compile]
       # remotely compile stack even if it hasn't changed (it's fast enough)
       when: >
         is_remote_building or
         stack_status is changed
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/stack/test.yml
       tags: [test]
       when: >
         stack_status is changed or
         pipeline_status is changed
-    - ansible.builtin.import_tasks: # noqa: name[missing]
+    - ansible.builtin.import_tasks:  # noqa: name[missing]
         file: tasks/stack/package.yml
       tags: [package]
       when: stack_status is changed

.dx/tasks/pipeline/package.yml

@@ -1,5 +1,5 @@
 ---
-- ansible.builtin.import_role: # noqa: name[missing]
+- ansible.builtin.import_role:  # noqa: name[missing]
     name: image
   vars:
     image_tag: "{{ pipeline_tag.stdout[:7] }}"

.dx/tasks/stack/package.yml

@@ -1,5 +1,5 @@
 ---
-- ansible.builtin.import_role: # noqa: name[missing]
+- ansible.builtin.import_role:  # noqa: name[missing]
     name: image
   vars:
     image_tag: "{{ stack_tag.stdout[:7] }}"

.env/amber/.env

@@ -1,2 +1,2 @@
-PYTHON_VERSION=3.8
 JAVA_VERSION=17.0
+PYTHON_VERSION=3.8

.env/amber/requirements.txt

@@ -1,2 +1,3 @@
 ansible-core==2.13.10
 ansible-lint==6.11.0
+yamllint==1.32.0

.env/green/.env

@@ -1,2 +1,2 @@
-PYTHON_VERSION=3.9
 JAVA_VERSION=19.0
+PYTHON_VERSION=3.9

.env/green/requirements.txt

@@ -1,2 +1,3 @@
 ansible-core==2.15.3
 ansible-lint==6.18.0
+yamllint==1.32.0

.github/actions/prepare/action.yml

@@ -1,10 +1,16 @@
+---
+name: Prepare tools
+description: Tools preparation action for specific environment
 inputs:
   name:
+    description: Environment name
     required: true
     default: green
   tools:
+    description: Tools to prepare
     required: true
   token:
+    description: Github token
     required: false
 runs:
   using: "composite"

.github/dependabot.yml

@@ -1,3 +1,4 @@
+---
 version: 2
 updates:
   - package-ecosystem: "github-actions"
@@ -34,6 +35,10 @@ updates:
     schedule:
       interval: "weekly"
   - package-ecosystem: "pip"
-    directory: "/"
+    directory: "/.env/green"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/.env/amber"
     schedule:
       interval: "weekly"

.github/workflows/pipeline-beyond-doubt.yml

@@ -1,9 +1,10 @@
+---
 name: Pipeline beyond doubt
 
 on:
   pull_request:
     branches: [main]
-    paths: # waiting for https://github.com/actions/runner/issues/2324
+    paths:  # waiting for https://github.com/actions/runner/issues/2324
       - .dx/**
       - .env/**
       - .github/**
@@ -41,6 +42,9 @@ jobs:
           ansible-playbook sources.yml -l dx:test
           -e docker_registry=${{ env.REGISTRY }}
         working-directory: .dx
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: >
           ansible-playbook stack.yml
           --skip-tags package

.github/workflows/pipeline-minimal.yml

@@ -1,9 +1,10 @@
+---
 name: Pipeline minimal
 
 on:
   pull_request:
     branches-ignore: [main]
-    paths: # waiting for https://github.com/actions/runner/issues/2324
+    paths:  # waiting for https://github.com/actions/runner/issues/2324
       - .dx/**
       - .env/**
       - .github/**
@@ -35,3 +36,6 @@ jobs:
           ansible-playbook sources.yml -l dx:test
           -e docker_registry=${{ env.REGISTRY }}
         working-directory: .dx
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/solution-beyond-doubt.yml

@@ -1,11 +1,12 @@
+---
 name: Solution beyond doubt
 
 on:
   push:
     branches: [main]
   pull_request:
     branches: [main]
-    paths: # waiting for https://github.com/actions/runner/issues/2324
+    paths:  # waiting for https://github.com/actions/runner/issues/2324
       - libs/**
       - apps/**
       - schemas/**
@@ -46,10 +47,6 @@ jobs:
       checks: write
       packages: write
       pull-requests: write
-    env:
-      JACOCO_UNITS: tools/target/coverage/units/jacoco.xml
-      JACOCO_INTEGRATIONS: tools/target/coverage/integrations/jacoco.xml
-      JACOCO_OVERALL: tools/target/coverage/overall/jacoco.xml
     steps:
       - uses: actions/[email protected]
       - uses: ./.github/actions/prepare
@@ -96,20 +93,26 @@ jobs:
           paths: ${{ env.JACOCO_UNITS }}
           token: ${{ secrets.GITHUB_TOKEN }}
           update-comment: true
+        env:
+          JACOCO_UNITS: tools/target/coverage/units/jacoco.xml
       - uses: madrapps/[email protected]
         if: ${{ github.event_name == 'pull_request' && hashFiles(env.JACOCO_INTEGRATIONS) != '' }}
         with:
           title: Integration Coverage Report
           paths: ${{ env.JACOCO_INTEGRATIONS }}
           token: ${{ secrets.GITHUB_TOKEN }}
           update-comment: true
+        env:
+          JACOCO_INTEGRATIONS: tools/target/coverage/integrations/jacoco.xml
       - uses: madrapps/[email protected]
         if: ${{ github.event_name == 'pull_request' && hashFiles(env.JACOCO_OVERALL) != '' }}
         with:
           title: Overall Coverage Report
           paths: ${{ env.JACOCO_OVERALL }}
           token: ${{ secrets.GITHUB_TOKEN }}
           update-comment: true
+        env:
+          JACOCO_OVERALL: tools/target/coverage/overall/jacoco.xml
 
   stack-compiling:
     strategy:

.github/workflows/solution-minimal.yml

@@ -1,9 +1,10 @@
+---
 name: Solution minimal
 
 on:
   pull_request:
     branches-ignore: [main]
-    paths: # waiting for https://github.com/actions/runner/issues/2324
+    paths:  # waiting for https://github.com/actions/runner/issues/2324
       - libs/**
       - apps/**
       - schemas/**

README.adoc

@@ -15,7 +15,7 @@
 
 ==== Минимальная (minimal)
 
-Запускаем линтеры, статический анализ и компиляцию для исходного кода решения.
+Запускаем линтинг, статический анализ и компиляцию (по желанию) исходного кода решения.
 
 ==== В пределах разумного (reasonable)
 
@@ -27,15 +27,15 @@
 
 ==== За пределами сомнений (beyond doubt)
 
-Все перечисленное ранее. Плюс запускаем сквозные тесты, собираем и публикуем стеки.
+Все перечисленное ранее. Плюс запускаем сквозные тесты, собираем и публикуем конфигурации решения (стеки).
 
 === Проверка конвейера
 
 Конвейер проверяем во всех окружениях, в которых его могут эксплуатировать разработчики.
 
 ==== Минимальная (minimal)
 
-Запускаем линтеры для исходного кода конвейера.
+Запускаем линтинг исходного кода конвейера.
 
 ==== За пределами сомнений (beyond doubt)
 
@@ -48,6 +48,7 @@
 == Todo
 
 . Релизная сборка
+. Параллелизация модульных тестов
 . Версионирование библиотек и приложений (взглянуть критически)
 . Агрегация отчетов покрытия (взглянуть критически)
 . Локальные проверки в удаленном репозитории