From 61ee6847857a357172dbe4af9ce19619bb9306e8 Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Thu, 22 Feb 2024 19:29:55 +0300 Subject: [PATCH 1/8] Separate building & publishing --- app/pom.xml | 40 ++++++- app/sepuling-java/pom.xml | 4 + app/sepuling-kotlin/pom.xml | 4 + baker/ansible/binaries.yaml | 70 +----------- baker/ansible/images.yaml | 103 ++++++++---------- baker/ansible/inventory/aliases.yaml | 3 +- .../group_vars/all/{vars.yaml => all.yaml} | 0 baker/ansible/roles/image/tasks/clean.yaml | 22 ++++ baker/ansible/roles/image/tasks/main.yaml | 33 +----- baker/ansible/stacks.yaml | 6 +- baker/ansible/tasks/app/build-and-check.yaml | 39 +++++++ baker/ansible/tasks/app/publish.yaml | 22 ++++ baker/ansible/tasks/app/status.yaml | 41 +++++++ baker/ansible/tasks/db/build.yaml | 13 +++ baker/ansible/tasks/db/publish.yaml | 19 ++++ baker/ansible/tasks/db/status.yaml | 16 +++ pom.xml | 15 ++- 17 files changed, 282 insertions(+), 168 deletions(-) rename baker/ansible/inventory/group_vars/all/{vars.yaml => all.yaml} (100%) create mode 100644 baker/ansible/roles/image/tasks/clean.yaml create mode 100644 baker/ansible/tasks/app/build-and-check.yaml create mode 100644 baker/ansible/tasks/app/publish.yaml create mode 100644 baker/ansible/tasks/app/status.yaml create mode 100644 baker/ansible/tasks/db/build.yaml create mode 100644 baker/ansible/tasks/db/publish.yaml create mode 100644 baker/ansible/tasks/db/status.yaml diff --git a/app/pom.xml b/app/pom.xml index 3528c002..3f80500a 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -66,8 +66,8 @@ maven-dependency-plugin - dep-copy - package + deps-context + pre-integration-test copy-dependencies @@ -78,8 +78,8 @@ - lib-copy - package + lib-context + pre-integration-test copy-dependencies @@ -90,8 +90,8 @@ - app-copy - package + app-context + pre-integration-test copy @@ -108,6 +108,34 @@ + + org.codehaus.mojo + exec-maven-plugin + + + app-image + pre-integration-test + + exec + + + + + docker + + build + ${project.build.directory}/image-context + --file + ${project.basedir}/Dockerfile + --tag + app/${project.artifactId}:godel + --label + image.key=app/${project.artifactId}/godel + --build-arg + JAVA_RELEASE=${maven.compiler.release} + + + diff --git a/app/sepuling-java/pom.xml b/app/sepuling-java/pom.xml index fbeeee92..03e60aa6 100644 --- a/app/sepuling-java/pom.xml +++ b/app/sepuling-java/pom.xml @@ -56,6 +56,10 @@ org.apache.maven.plugins maven-dependency-plugin + + org.codehaus.mojo + exec-maven-plugin + diff --git a/app/sepuling-kotlin/pom.xml b/app/sepuling-kotlin/pom.xml index 6d9e6721..35e88389 100644 --- a/app/sepuling-kotlin/pom.xml +++ b/app/sepuling-kotlin/pom.xml @@ -77,6 +77,10 @@ org.apache.maven.plugins maven-dependency-plugin + + org.codehaus.mojo + exec-maven-plugin + diff --git a/baker/ansible/binaries.yaml b/baker/ansible/binaries.yaml index ce645e1e..fffb2092 100644 --- a/baker/ansible/binaries.yaml +++ b/baker/ansible/binaries.yaml @@ -3,69 +3,7 @@ hosts: app run_once: true tasks: - - name: Capture cids - ansible.builtin.command: - cmd: git hash-object --stdin - stdin: | - {{ lookup('ansible.builtin.pipe', 'git write-tree --prefix=lib') }} - {{ lookup('ansible.builtin.pipe', 'git write-tree --prefix=app/' ~ app) }} - register: image_cid - changed_when: false - loop: "{{ product.apps.values()|flatten|map(attribute='binary') }}" - loop_control: - loop_var: app - tags: [always] - - name: Declare tags - ansible.builtin.set_fact: - image_tags: >- - {{ dict(image_cid.results - |map(attribute='stdout') - |map('truncate', 7, True, '') - |map('regex_replace', '^', devenv ~ '-') - |zip(image_cid.results|map(attribute='app')) - |map('reverse')) - }} - tags: [always] - - name: Capture statuses - ansible.builtin.command: - cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ app.image }}:{{ image_tags[app.binary] }} - register: image_status - changed_when: - - image_status.rc != 0 - failed_when: false - loop: "{{ product.apps.values()|flatten }}" - loop_control: - loop_var: app - label: "{{ app.binary }}" - tags: [always] - - name: Status commands - ansible.builtin.debug: - msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" - when: image_status is changed - tags: [always] - - name: Create binaries - ansible.builtin.command: - cmd: >- - mvn - --no-snapshot-updates - --fail-fast - --batch-mode - --threads 2 - --projects {{ - image_status.results - | select('changed') - | map(attribute='app.binary') - | map('regex_replace', '^', ':') - | join(',') - }} - --also-make - clean - {{ maven_phase | default('package') }} - --define maven.compiler.release={{ dev[devenv].jdk.release }} - {% if dev[devenv].kotlin is defined %} - --define kotlin.version={{ dev[devenv].kotlin.version }} - {% endif %} - chdir: "{{ project.dir }}" - strip_empty_ends: false - changed_when: true - when: image_status is changed + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/app/build-and-check.yaml + vars: + maven_phase: package diff --git a/baker/ansible/images.yaml b/baker/ansible/images.yaml index fd1e97b1..d4a8c262 100644 --- a/baker/ansible/images.yaml +++ b/baker/ansible/images.yaml @@ -1,72 +1,57 @@ --- -- import_playbook: binaries.yaml # noqa: name[play] - vars: - maven_phase: "{{ 'install' if binary_repo == 'local' else 'deploy' }}" - tags: [binaries, deps] - -- name: Images +- name: Building hosts: db run_once: true tasks: - - name: Capture statuses - ansible.builtin.command: - cmd: >- - docker {{ docker_entity }} inspect - {{ image_ns }}/{{ product.databases[storage].image }}:{{ product.databases[storage].cid }} - register: image_status - changed_when: image_status.rc != 0 - failed_when: false - loop: "{{ product.schemas.keys() }}" - loop_control: - loop_var: storage - - name: Status commands - ansible.builtin.debug: - msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" - when: image_status is changed - - name: Create images - ansible.builtin.include_role: + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/db/status.yaml + tags: [always] + - ansible.builtin.import_role: # noqa: name[missing] name: image + tasks_from: clean vars: - image_tag: "{{ product.databases[storage].cid }}" - image_name: "{{ image_ns }}/{{ product.databases[storage].image }}" - image_home: "{{ project.dir }}/db/{{ storage }}" - image_key: "{{ product.databases[storage].image }}" - image_push: "{{ image_repo != 'local' }}" - loop: "{{ image_status.results|select('changed')|map(attribute='storage') }}" - loop_control: - loop_var: storage + image_keys: >- + {{ image_status.results + | select('changed') + | map(attribute='db.image') + | map('regex_replace', '$', '/' ~ devenv) + }} + when: image_status is changed + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/db/build.yaml + when: image_status is changed -- name: Images +- name: Building & Checking hosts: app run_once: true tasks: - - name: Analyze coverage - ansible.builtin.command: - cmd: >- - mvn - --no-snapshot-updates - --batch-mode - --projects tool - clean - antrun:run@coverage - chdir: "{{ project.dir }}" - strip_empty_ends: false - changed_when: true - when: image_status is changed - - name: Create images - ansible.builtin.include_role: + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/app/status.yaml + tags: [always] + - ansible.builtin.import_role: # noqa: name[missing] name: image + tasks_from: clean vars: - image_tag: "{{ image_tags[app.binary] }}" - image_name: "{{ image_ns }}/{{ app.image }}" - image_home: "{{ project.dir }}/app/{{ app.binary }}" - image_context: target/image-context - image_key: "{{ app.image }}/{{ devenv }}" - image_push: "{{ image_repo != 'local' }}" - image_args: - JAVA_RELEASE: "{{ dev[devenv].jdk.release }}" - loop: "{{ image_status.results|select('changed')|map(attribute='app') }}" - loop_control: - loop_var: app - label: "{{ app.binary }}" + image_keys: >- + {{ image_status.results + | select('changed') + | map(attribute='app.image') + | map('regex_replace', '$', '/' ~ devenv) + }} when: image_status is changed + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/app/build-and-check.yaml + vars: + maven_phase: "{{ 'install' if binary_repo == 'local' else 'deploy' }}" + when: image_status is changed + +- name: Publishing + hosts: image + run_once: true + tasks: + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/db/publish.yaml + when: hostvars.db.image_status is changed + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/app/publish.yaml + when: hostvars.app.image_status is changed diff --git a/baker/ansible/inventory/aliases.yaml b/baker/ansible/inventory/aliases.yaml index 5230392f..a23bd895 100644 --- a/baker/ansible/inventory/aliases.yaml +++ b/baker/ansible/inventory/aliases.yaml @@ -8,9 +8,10 @@ stack: hosts: gear: product: -all: +image: hosts: db: app: +all: vars: ansible_connection: local diff --git a/baker/ansible/inventory/group_vars/all/vars.yaml b/baker/ansible/inventory/group_vars/all/all.yaml similarity index 100% rename from baker/ansible/inventory/group_vars/all/vars.yaml rename to baker/ansible/inventory/group_vars/all/all.yaml diff --git a/baker/ansible/roles/image/tasks/clean.yaml b/baker/ansible/roles/image/tasks/clean.yaml new file mode 100644 index 00000000..bd4e2511 --- /dev/null +++ b/baker/ansible/roles/image/tasks/clean.yaml @@ -0,0 +1,22 @@ +--- +- name: Enforce variables + ansible.builtin.assert: + quiet: true + that: + - image_keys is defined + +- name: Kill containers + ansible.builtin.shell: + cmd: docker container rm -f $(docker ps -aq --filter label=image.key={{ item }}) + strip_empty_ends: false + changed_when: false + failed_when: false + loop: "{{ image_keys }}" + +- name: Remove images + ansible.builtin.shell: + cmd: docker image rm -f $(docker images -q --filter label=image.key={{ item }}) + strip_empty_ends: false + changed_when: false + failed_when: false + loop: "{{ image_keys }}" diff --git a/baker/ansible/roles/image/tasks/main.yaml b/baker/ansible/roles/image/tasks/main.yaml index 60d21797..c605b065 100644 --- a/baker/ansible/roles/image/tasks/main.yaml +++ b/baker/ansible/roles/image/tasks/main.yaml @@ -1,5 +1,5 @@ --- -- name: Check vars +- name: Enforce variables ansible.builtin.assert: quiet: true that: @@ -7,38 +7,15 @@ - image_home is defined - image_key is defined -- name: Kill containers - ansible.builtin.shell: - cmd: docker container rm -f $(docker ps -aq --filter label=image.key={{ image_key }}) - strip_empty_ends: false - changed_when: false - failed_when: false - no_log: true - -- name: Remove outdated - ansible.builtin.shell: - cmd: docker image rm -f $(docker images -q --filter label=image.key={{ image_key }}) - strip_empty_ends: false - changed_when: false - failed_when: false - no_log: true - -- name: Build fresh +- name: Build image ansible.builtin.command: cmd: >- - docker build {{ image_context }} --file Dockerfile - --tag {{ image_name }}:{{ image_tag }} + docker build {{ image_context }} + --file {{ image_home }}/Dockerfile --label image.key={{ image_key }} + --tag {{ image_name }}:{{ image_tag }} {% for key, value in image_args.items() %} --build-arg {{ key }}={{ value }} {% endfor %} - chdir: "{{ image_home }}" - strip_empty_ends: false - changed_when: true - -- name: Push built - ansible.builtin.command: - cmd: docker image push {{ image_name }}:{{ image_tag }} strip_empty_ends: false changed_when: true - when: image_push|bool diff --git a/baker/ansible/stacks.yaml b/baker/ansible/stacks.yaml index 90ce5a84..8f4a7669 100644 --- a/baker/ansible/stacks.yaml +++ b/baker/ansible/stacks.yaml @@ -18,7 +18,7 @@ file: tasks/gear/status.yaml tags: [always] -- name: Testing +- name: Building & Checking hosts: stack run_once: true vars: @@ -37,7 +37,7 @@ hostvars.product.stack_status is changed or hostvars.gear.stack_status is changed -- name: Packaging +- name: Publishing hosts: product run_once: true tasks: @@ -46,7 +46,7 @@ tags: [package] when: stack_status is changed -- name: Packaging +- name: Publishing hosts: gear run_once: true tasks: diff --git a/baker/ansible/tasks/app/build-and-check.yaml b/baker/ansible/tasks/app/build-and-check.yaml new file mode 100644 index 00000000..ba646387 --- /dev/null +++ b/baker/ansible/tasks/app/build-and-check.yaml @@ -0,0 +1,39 @@ +--- +- name: Build binaries + ansible.builtin.command: + cmd: >- + mvn + --no-snapshot-updates + --batch-mode + --fail-fast + --threads 2 + --projects {{ + image_status.results + | select('changed') + | map(attribute='app.binary') + | map('regex_replace', '^', ':') + | join(',') + }} + --also-make + clean + {{ maven_phase|mandatory }} + --define maven.compiler.release={{ dev[devenv].jdk.release }} + {% if dev[devenv].kotlin is defined %} + --define kotlin.version={{ dev[devenv].kotlin.version }} + {% endif %} + chdir: "{{ project.dir }}" + strip_empty_ends: false + changed_when: true + +- name: Analyze coverage + ansible.builtin.command: + cmd: >- + mvn + --no-snapshot-updates + --batch-mode + --projects tool + clean + antrun:run@coverage + chdir: "{{ project.dir }}" + strip_empty_ends: false + changed_when: true diff --git a/baker/ansible/tasks/app/publish.yaml b/baker/ansible/tasks/app/publish.yaml new file mode 100644 index 00000000..3f5392b4 --- /dev/null +++ b/baker/ansible/tasks/app/publish.yaml @@ -0,0 +1,22 @@ +--- +- name: Tag images + ansible.builtin.command: + cmd: >- + docker tag + {{ app.image }}:{{ devenv }} + {{ image_ns }}/{{ app.image }}:{{ image_tags[app.binary] }} + strip_empty_ends: false + loop: "{{ image_status.results|select('changed')|map(attribute='app') }}" + loop_control: + loop_var: app + label: "{{ app.image }}" + +- name: Push images + ansible.builtin.command: + cmd: docker push {{ image_ns }}/{{ app.image }}:{{ image_tags[app.binary] }} + strip_empty_ends: false + loop: "{{ image_status.results|select('changed')|map(attribute='app') }}" + loop_control: + loop_var: app + label: "{{ app.image }}" + tags: [push, never] diff --git a/baker/ansible/tasks/app/status.yaml b/baker/ansible/tasks/app/status.yaml new file mode 100644 index 00000000..0029385c --- /dev/null +++ b/baker/ansible/tasks/app/status.yaml @@ -0,0 +1,41 @@ +--- +- name: Capture cids + ansible.builtin.command: + cmd: git hash-object --stdin + stdin: | + {{ lookup('ansible.builtin.pipe', 'git write-tree --prefix=lib') }} + {{ lookup('ansible.builtin.pipe', 'git write-tree --prefix=app/' ~ app.binary) }} + register: image_cid + changed_when: false + loop: "{{ product.apps.values()|flatten }}" + loop_control: + loop_var: app + label: "{{ app.image }}" + +- name: Declare tags + ansible.builtin.set_fact: + image_tags: >- + {{ dict(image_cid.results + |map(attribute='stdout') + |map('truncate', 7, True, '') + |map('regex_replace', '^', devenv ~ '-') + |zip(image_cid.results|map(attribute='app.binary')) + |map('reverse')) + }} + +- name: Capture statuses + ansible.builtin.command: + cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ app.image }}:{{ image_tags[app.binary] }} + register: image_status + changed_when: + - image_status.rc != 0 + failed_when: false + loop: "{{ product.apps.values()|flatten }}" + loop_control: + loop_var: app + label: "{{ app.image }}" + +- name: Status commands + ansible.builtin.debug: + msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" + when: image_status is changed diff --git a/baker/ansible/tasks/db/build.yaml b/baker/ansible/tasks/db/build.yaml new file mode 100644 index 00000000..2b2081e2 --- /dev/null +++ b/baker/ansible/tasks/db/build.yaml @@ -0,0 +1,13 @@ +--- +- name: Build images + ansible.builtin.include_role: + name: image + vars: + image_home: "{{ project.dir }}/{{ db.image }}" + image_name: "{{ db.image }}" + image_key: "{{ db.image }}/{{ devenv }}" + image_tag: "{{ devenv }}" + loop: "{{ image_status.results|select('changed')|map(attribute='db') }}" + loop_control: + loop_var: db + label: "{{ db.image }}" diff --git a/baker/ansible/tasks/db/publish.yaml b/baker/ansible/tasks/db/publish.yaml new file mode 100644 index 00000000..f246e1b3 --- /dev/null +++ b/baker/ansible/tasks/db/publish.yaml @@ -0,0 +1,19 @@ +--- +- name: Tag images + ansible.builtin.command: + cmd: docker tag {{ db.image }}:{{ devenv }} {{ image_ns }}/{{ db.image }}:{{ db.cid }} + strip_empty_ends: false + loop: "{{ image_status.results|select('changed')|map(attribute='db') }}" + loop_control: + loop_var: db + label: "{{ db.image }}" + +- name: Push images + ansible.builtin.command: + cmd: docker push {{ image_ns }}/{{ db.image }}:{{ db.cid }} + strip_empty_ends: false + loop: "{{ image_status.results|select('changed')|map(attribute='db') }}" + loop_control: + loop_var: db + label: "{{ db.image }}" + tags: [push, never] diff --git a/baker/ansible/tasks/db/status.yaml b/baker/ansible/tasks/db/status.yaml new file mode 100644 index 00000000..a24d114a --- /dev/null +++ b/baker/ansible/tasks/db/status.yaml @@ -0,0 +1,16 @@ +--- +- name: Capture statuses + ansible.builtin.command: + cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ db.image }}:{{ db.cid }} + register: image_status + changed_when: image_status.rc != 0 + failed_when: false + loop: "{{ product.databases.values() }}" + loop_control: + loop_var: db + label: "{{ db.image }}" + +- name: Status commands + ansible.builtin.debug: + msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" + when: image_status is changed diff --git a/pom.xml b/pom.xml index 64f25b06..6c8ca452 100644 --- a/pom.xml +++ b/pom.xml @@ -233,6 +233,16 @@ ${skipUnits} + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.1 + + + org.codehaus.mojo + exec-maven-plugin + 3.1.1 + org.apache.maven.plugins maven-failsafe-plugin @@ -284,11 +294,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - 3.6.1 - org.apache.maven.plugins maven-jar-plugin From 8ce43017eb3d9675594a1af2e370f92cc930e63c Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Thu, 22 Feb 2024 22:27:45 +0300 Subject: [PATCH 2/8] Stacks building & publishing --- .github/workflows/beyond-doubt.yaml | 17 ++++------- .github/workflows/convincing.yaml | 3 +- baker/ansible/binaries.yaml | 2 +- baker/ansible/images.yaml | 6 ++-- baker/ansible/inventory/aliases.yaml | 8 +++--- .../ansible/inventory/group_vars/all/all.yaml | 7 +++-- baker/ansible/roles/image/defaults/main.yaml | 2 -- .../image/tasks/{main.yaml => build.yaml} | 9 +++--- baker/ansible/roles/image/tasks/clean.yaml | 2 +- baker/ansible/roles/image/tasks/publish.yaml | 20 +++++++++++++ baker/ansible/sources.yaml | 10 +++---- baker/ansible/stacks.yaml | 28 +++++++++++-------- .../tasks/app/{status.yaml => capture.yaml} | 19 +++++-------- baker/ansible/tasks/app/publish.yaml | 26 ++++++----------- baker/ansible/tasks/db/build.yaml | 9 +++--- .../tasks/db/{status.yaml => capture.yaml} | 7 +---- baker/ansible/tasks/db/publish.yaml | 23 ++++++--------- baker/ansible/tasks/gear/build.yaml | 10 +++---- .../tasks/gear/{status.yaml => capture.yaml} | 13 +++++++-- baker/ansible/tasks/gear/package.yaml | 9 ------ baker/ansible/tasks/gear/publish.yaml | 8 ++++++ baker/ansible/tasks/product/build.yaml | 9 ++++++ .../product/{status.yaml => capture.yaml} | 6 ++-- .../tasks/product/{test.yaml => check.yaml} | 2 +- baker/ansible/tasks/product/package.yaml | 10 ------- baker/ansible/tasks/product/publish.yaml | 8 ++++++ stack/gear/Dockerfile | 3 ++ 27 files changed, 140 insertions(+), 136 deletions(-) rename baker/ansible/roles/image/tasks/{main.yaml => build.yaml} (74%) create mode 100644 baker/ansible/roles/image/tasks/publish.yaml rename baker/ansible/tasks/app/{status.yaml => capture.yaml} (66%) rename baker/ansible/tasks/db/{status.yaml => capture.yaml} (61%) rename baker/ansible/tasks/gear/{status.yaml => capture.yaml} (76%) delete mode 100644 baker/ansible/tasks/gear/package.yaml create mode 100644 baker/ansible/tasks/gear/publish.yaml rename baker/ansible/tasks/product/{status.yaml => capture.yaml} (86%) rename baker/ansible/tasks/product/{test.yaml => check.yaml} (97%) delete mode 100644 baker/ansible/tasks/product/package.yaml create mode 100644 baker/ansible/tasks/product/publish.yaml create mode 100644 stack/gear/Dockerfile diff --git a/.github/workflows/beyond-doubt.yaml b/.github/workflows/beyond-doubt.yaml index fe846adc..a521166a 100644 --- a/.github/workflows/beyond-doubt.yaml +++ b/.github/workflows/beyond-doubt.yaml @@ -9,8 +9,6 @@ on: env: IMAGE_NAME: stack/gear - BINARY_REPO: maven.pkg.github.com - IMAGE_REPO: ghcr.io jobs: up-to-images: @@ -57,8 +55,7 @@ jobs: ${{ env.IMAGE_NAME }}:${{ matrix.env.dev }} ansible-playbook images.yaml -v -e devenv=${{ matrix.env.dev }} - -e binary_repo=${{ env.BINARY_REPO }} - -e image_repo=${{ env.IMAGE_REPO }} + -e repo_mode=rw - uses: mikepenz/action-junit-report@v4.1.0 if: ${{ !cancelled() && (steps.images.outcome == 'success' || steps.images.outcome == 'failure') }} with: @@ -108,7 +105,6 @@ jobs: --workdir $(pwd)/baker/ansible ${{ env.IMAGE_NAME }}:${{ matrix.env.dev }} ansible-playbook stacks.yaml -t build - -e image_repo=${{ env.IMAGE_REPO }} -e devenv=${{ matrix.env.dev }} -e purpose=${{ matrix.purpose }} -e opsenv=${{ matrix.env.ops }} @@ -149,7 +145,7 @@ jobs: with: name: product-${{ matrix.purpose }}-${{ matrix.env.ops }} path: stack/product/target/image-context - - name: Run ansible-playbook stacks.yaml -t test + - name: Run ansible-playbook stacks.yaml -t check run: >- docker run --rm --network gear @@ -158,8 +154,7 @@ jobs: --volume /var/run/docker.sock:/var/run/docker.sock --workdir $(pwd)/baker/ansible ${{ env.IMAGE_NAME }}:${{ matrix.env.dev }} - ansible-playbook stacks.yaml -t test - -e image_repo=${{ env.IMAGE_REPO }} + ansible-playbook stacks.yaml -t check -e devenv=${{ matrix.env.dev }} -e purpose=${{ matrix.purpose }} -e opsenv=${{ matrix.env.ops }} @@ -204,7 +199,7 @@ jobs: with: name: product-${{ matrix.purpose }}-${{ matrix.env.ops }} path: stack/product/target/image-context - - name: Run ansible-playbook stacks.yaml -t package + - name: Run ansible-playbook stacks.yaml -t publish run: >- docker run --rm --volume $(pwd):$(pwd) @@ -212,11 +207,11 @@ jobs: --volume ~/.docker/config.json:/root/.docker/config.json:ro --workdir $(pwd)/baker/ansible ${{ env.IMAGE_NAME }}:${{ matrix.env.dev }} - ansible-playbook stacks.yaml -t package -v - -e image_repo=${{ env.IMAGE_REPO }} + ansible-playbook stacks.yaml -t publish -v -e devenv=${{ matrix.env.dev }} -e purpose=${{ matrix.purpose }} -e opsenv=${{ matrix.env.ops }} + -e repo_mode=rw stack-reporting: name: Stack reporting diff --git a/.github/workflows/convincing.yaml b/.github/workflows/convincing.yaml index 7f3bd1ec..31da92e8 100644 --- a/.github/workflows/convincing.yaml +++ b/.github/workflows/convincing.yaml @@ -53,8 +53,7 @@ jobs: ${{ env.IMAGE_NAME }}:${{ matrix.env.dev }} ansible-playbook images.yaml -v -e devenv=${{ matrix.env.dev }} - -e binary_repo=maven.pkg.github.com - -e image_repo=ghcr.io + -e repo_mode=rw - uses: mikepenz/action-junit-report@v4.1.0 if: ${{ !cancelled() && (steps.images.outcome == 'success' || steps.images.outcome == 'failure') }} with: diff --git a/baker/ansible/binaries.yaml b/baker/ansible/binaries.yaml index fffb2092..d351e0a8 100644 --- a/baker/ansible/binaries.yaml +++ b/baker/ansible/binaries.yaml @@ -1,5 +1,5 @@ --- -- name: Binaries +- name: Building & Checking hosts: app run_once: true tasks: diff --git a/baker/ansible/images.yaml b/baker/ansible/images.yaml index d4a8c262..f636db7a 100644 --- a/baker/ansible/images.yaml +++ b/baker/ansible/images.yaml @@ -4,7 +4,7 @@ run_once: true tasks: - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/db/status.yaml + file: tasks/db/capture.yaml tags: [always] - ansible.builtin.import_role: # noqa: name[missing] name: image @@ -26,7 +26,7 @@ run_once: true tasks: - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/app/status.yaml + file: tasks/app/capture.yaml tags: [always] - ansible.builtin.import_role: # noqa: name[missing] name: image @@ -42,7 +42,7 @@ - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/app/build-and-check.yaml vars: - maven_phase: "{{ 'install' if binary_repo == 'local' else 'deploy' }}" + maven_phase: "{{ 'deploy' if repo_mode == 'rw' else 'install' }}" when: image_status is changed - name: Publishing diff --git a/baker/ansible/inventory/aliases.yaml b/baker/ansible/inventory/aliases.yaml index a23bd895..de0868c7 100644 --- a/baker/ansible/inventory/aliases.yaml +++ b/baker/ansible/inventory/aliases.yaml @@ -4,14 +4,14 @@ sources: java: ansible: github: -stack: - hosts: - gear: - product: image: hosts: db: app: +stack: + hosts: + gear: + product: all: vars: ansible_connection: local diff --git a/baker/ansible/inventory/group_vars/all/all.yaml b/baker/ansible/inventory/group_vars/all/all.yaml index a6a18b77..dae5b31f 100644 --- a/baker/ansible/inventory/group_vars/all/all.yaml +++ b/baker/ansible/inventory/group_vars/all/all.yaml @@ -4,9 +4,10 @@ project: org: smecalculus dir: "{{ playbook_dir }}/../.." -binary_repo: local -image_repo: local +binary_repo: maven.pkg.github.com +image_repo: ghcr.io +repo_mode: n/a # values: rw, ro, n/a image_ns: "{{ image_repo }}/{{ project.org }}/{{ project.name }}" -docker_entity: "{{ 'image' if image_repo == 'local' else 'manifest' }}" +docker_entity: "{{ 'image' if repo_mode == 'n/a' else 'manifest' }}" diff --git a/baker/ansible/roles/image/defaults/main.yaml b/baker/ansible/roles/image/defaults/main.yaml index 48f0fa1a..028f44bb 100644 --- a/baker/ansible/roles/image/defaults/main.yaml +++ b/baker/ansible/roles/image/defaults/main.yaml @@ -1,5 +1,3 @@ --- -image_tag: latest image_context: . -image_push: false image_args: {} diff --git a/baker/ansible/roles/image/tasks/main.yaml b/baker/ansible/roles/image/tasks/build.yaml similarity index 74% rename from baker/ansible/roles/image/tasks/main.yaml rename to baker/ansible/roles/image/tasks/build.yaml index c605b065..39a30b84 100644 --- a/baker/ansible/roles/image/tasks/main.yaml +++ b/baker/ansible/roles/image/tasks/build.yaml @@ -1,21 +1,22 @@ --- -- name: Enforce variables +- name: Check inputs ansible.builtin.assert: quiet: true that: - - image_name is defined - image_home is defined - image_key is defined + - image_tag is defined - name: Build image ansible.builtin.command: cmd: >- docker build {{ image_context }} - --file {{ image_home }}/Dockerfile + --file Dockerfile --label image.key={{ image_key }} - --tag {{ image_name }}:{{ image_tag }} + --tag {{ image_tag }} {% for key, value in image_args.items() %} --build-arg {{ key }}={{ value }} {% endfor %} strip_empty_ends: false + chdir: "{{ image_home }}" changed_when: true diff --git a/baker/ansible/roles/image/tasks/clean.yaml b/baker/ansible/roles/image/tasks/clean.yaml index bd4e2511..ce462875 100644 --- a/baker/ansible/roles/image/tasks/clean.yaml +++ b/baker/ansible/roles/image/tasks/clean.yaml @@ -1,5 +1,5 @@ --- -- name: Enforce variables +- name: Check inputs ansible.builtin.assert: quiet: true that: diff --git a/baker/ansible/roles/image/tasks/publish.yaml b/baker/ansible/roles/image/tasks/publish.yaml new file mode 100644 index 00000000..b5f04d4c --- /dev/null +++ b/baker/ansible/roles/image/tasks/publish.yaml @@ -0,0 +1,20 @@ +--- +- name: Check inputs + ansible.builtin.assert: + quiet: true + that: + - source_tag is defined + - target_tags is defined + +- name: Tag images + ansible.builtin.command: + cmd: docker tag {{ source_tag }} {{ item }} + strip_empty_ends: false + loop: "{{ target_tags }}" + +- name: Push images + ansible.builtin.command: + cmd: docker push {{ item }} + strip_empty_ends: false + loop: "{{ target_tags }}" + tags: [push, never] diff --git a/baker/ansible/sources.yaml b/baker/ansible/sources.yaml index b33d9ea5..2d213bda 100644 --- a/baker/ansible/sources.yaml +++ b/baker/ansible/sources.yaml @@ -1,5 +1,5 @@ --- -- name: Sources +- name: Checking hosts: java run_once: true tasks: @@ -19,11 +19,11 @@ strip_empty_ends: false changed_when: false -- name: Sources +- name: Checking hosts: ansible run_once: true tasks: - - name: Analyze + - name: Linting ansible.builtin.command: # list all top level playbooks cmd: >- @@ -33,11 +33,11 @@ strip_empty_ends: false changed_when: false -- name: Sources +- name: Checking hosts: github run_once: true tasks: - - name: Analyze + - name: Linting ansible.builtin.command: cmd: yamllint {{ project.dir }}/.github strip_empty_ends: false diff --git a/baker/ansible/stacks.yaml b/baker/ansible/stacks.yaml index 8f4a7669..2b9a44af 100644 --- a/baker/ansible/stacks.yaml +++ b/baker/ansible/stacks.yaml @@ -7,19 +7,23 @@ run_once: true tasks: - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/product/status.yaml + file: tasks/product/capture.yaml tags: [always] -- name: Capturing +- name: Capturing & Building hosts: gear run_once: true tasks: - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/gear/status.yaml + file: tasks/gear/capture.yaml tags: [always] + - ansible.builtin.import_tasks: # noqa: name[missing] + file: tasks/gear/build.yaml + tags: [build] + when: hostvars.gear.stack_status is changed - name: Building & Checking - hosts: stack + hosts: product run_once: true vars: stack_dir: "{{ project.dir }}/stack/product" @@ -31,8 +35,8 @@ hostvars.product.stack_status is changed or hostvars.gear.stack_status is changed - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/product/test.yaml - tags: [test] + file: tasks/product/check.yaml + tags: [check] when: >- hostvars.product.stack_status is changed or hostvars.gear.stack_status is changed @@ -42,15 +46,15 @@ run_once: true tasks: - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/product/package.yaml - tags: [package] - when: stack_status is changed + file: tasks/product/publish.yaml + tags: [publish] + when: hostvars.product.stack_status is changed - name: Publishing hosts: gear run_once: true tasks: - ansible.builtin.import_tasks: # noqa: name[missing] - file: tasks/gear/package.yaml - tags: [package] - when: stack_status is changed + file: tasks/gear/publish.yaml + tags: [publish] + when: hostvars.gear.stack_status is changed diff --git a/baker/ansible/tasks/app/status.yaml b/baker/ansible/tasks/app/capture.yaml similarity index 66% rename from baker/ansible/tasks/app/status.yaml rename to baker/ansible/tasks/app/capture.yaml index 0029385c..54e40080 100644 --- a/baker/ansible/tasks/app/status.yaml +++ b/baker/ansible/tasks/app/capture.yaml @@ -1,5 +1,5 @@ --- -- name: Capture cids +- name: Capture CID's ansible.builtin.command: cmd: git hash-object --stdin stdin: | @@ -16,11 +16,11 @@ ansible.builtin.set_fact: image_tags: >- {{ dict(image_cid.results - |map(attribute='stdout') - |map('truncate', 7, True, '') - |map('regex_replace', '^', devenv ~ '-') - |zip(image_cid.results|map(attribute='app.binary')) - |map('reverse')) + | map(attribute='stdout') + | map('truncate', 7, True, '') + | map('regex_replace', '^', devenv ~ '-') + | zip(image_cid.results|map(attribute='app.binary')) + | map('reverse')) }} - name: Capture statuses @@ -33,9 +33,4 @@ loop: "{{ product.apps.values()|flatten }}" loop_control: loop_var: app - label: "{{ app.image }}" - -- name: Status commands - ansible.builtin.debug: - msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" - when: image_status is changed + label: "{{ app.image }}:{{ image_tags[app.binary] }}" diff --git a/baker/ansible/tasks/app/publish.yaml b/baker/ansible/tasks/app/publish.yaml index 3f5392b4..bdeaac3f 100644 --- a/baker/ansible/tasks/app/publish.yaml +++ b/baker/ansible/tasks/app/publish.yaml @@ -1,22 +1,12 @@ --- -- name: Tag images - ansible.builtin.command: - cmd: >- - docker tag - {{ app.image }}:{{ devenv }} - {{ image_ns }}/{{ app.image }}:{{ image_tags[app.binary] }} - strip_empty_ends: false - loop: "{{ image_status.results|select('changed')|map(attribute='app') }}" +- ansible.builtin.include_role: # noqa: name[missing] + name: image + tasks_from: publish + vars: + source_tag: "{{ app.image }}:{{ devenv }}" + target_tags: + - "{{ image_ns }}/{{ app.image }}:{{ hostvars.app.image_tags[app.binary] }}" + loop: "{{ hostvars.app.image_status.results|select('changed')|map(attribute='app') }}" loop_control: loop_var: app label: "{{ app.image }}" - -- name: Push images - ansible.builtin.command: - cmd: docker push {{ image_ns }}/{{ app.image }}:{{ image_tags[app.binary] }} - strip_empty_ends: false - loop: "{{ image_status.results|select('changed')|map(attribute='app') }}" - loop_control: - loop_var: app - label: "{{ app.image }}" - tags: [push, never] diff --git a/baker/ansible/tasks/db/build.yaml b/baker/ansible/tasks/db/build.yaml index 2b2081e2..ce645497 100644 --- a/baker/ansible/tasks/db/build.yaml +++ b/baker/ansible/tasks/db/build.yaml @@ -1,13 +1,12 @@ --- -- name: Build images - ansible.builtin.include_role: +- ansible.builtin.include_role: # noqa: name[missing] name: image + tasks_from: build vars: image_home: "{{ project.dir }}/{{ db.image }}" - image_name: "{{ db.image }}" image_key: "{{ db.image }}/{{ devenv }}" - image_tag: "{{ devenv }}" - loop: "{{ image_status.results|select('changed')|map(attribute='db') }}" + image_tag: "{{ db.image }}:{{ devenv }}" + loop: "{{ hostvars.db.image_status.results|select('changed')|map(attribute='db') }}" loop_control: loop_var: db label: "{{ db.image }}" diff --git a/baker/ansible/tasks/db/status.yaml b/baker/ansible/tasks/db/capture.yaml similarity index 61% rename from baker/ansible/tasks/db/status.yaml rename to baker/ansible/tasks/db/capture.yaml index a24d114a..2c61778d 100644 --- a/baker/ansible/tasks/db/status.yaml +++ b/baker/ansible/tasks/db/capture.yaml @@ -8,9 +8,4 @@ loop: "{{ product.databases.values() }}" loop_control: loop_var: db - label: "{{ db.image }}" - -- name: Status commands - ansible.builtin.debug: - msg: "{{ image_status.results|map(attribute='cmd')|map('join', ' ') }}" - when: image_status is changed + label: "{{ db.image }}:{{ db.cid }}" diff --git a/baker/ansible/tasks/db/publish.yaml b/baker/ansible/tasks/db/publish.yaml index f246e1b3..2a127563 100644 --- a/baker/ansible/tasks/db/publish.yaml +++ b/baker/ansible/tasks/db/publish.yaml @@ -1,19 +1,12 @@ --- -- name: Tag images - ansible.builtin.command: - cmd: docker tag {{ db.image }}:{{ devenv }} {{ image_ns }}/{{ db.image }}:{{ db.cid }} - strip_empty_ends: false - loop: "{{ image_status.results|select('changed')|map(attribute='db') }}" +- ansible.builtin.include_role: # noqa: name[missing] + name: image + tasks_from: publish + vars: + source_tag: "{{ db.image }}:{{ devenv }}" + target_tags: + - "{{ image_ns }}/{{ db.image }}:{{ db.cid }}" + loop: "{{ hostvars.db.image_status.results|select('changed')|map(attribute='db') }}" loop_control: loop_var: db label: "{{ db.image }}" - -- name: Push images - ansible.builtin.command: - cmd: docker push {{ image_ns }}/{{ db.image }}:{{ db.cid }} - strip_empty_ends: false - loop: "{{ image_status.results|select('changed')|map(attribute='db') }}" - loop_control: - loop_var: db - label: "{{ db.image }}" - tags: [push, never] diff --git a/baker/ansible/tasks/gear/build.yaml b/baker/ansible/tasks/gear/build.yaml index 35232e43..73f160ab 100644 --- a/baker/ansible/tasks/gear/build.yaml +++ b/baker/ansible/tasks/gear/build.yaml @@ -1,11 +1,9 @@ --- - ansible.builtin.import_role: # noqa: name[missing] name: image + tasks_from: build vars: - image_tag: "{{ gear_tag }}" - image_name: "{{ image_ns }}/{{ gear.image }}" - image_home: "{{ project.dir }}/stack/gear/{{ devenv }}" + image_home: "{{ project.dir }}/stack/gear" image_key: "{{ gear.image }}/{{ devenv }}" - image_push: false - image_args: - DOCKER_VERSION: "{{ lookup('ansible.builtin.pipe', 'docker version --format {% raw %}{{.Client.Version}}{% endraw %}') }}" # noqa: yaml[line-length] + image_tag: "{{ gear.image }}:{{ devenv }}" + when: repo_mode != 'rw' diff --git a/baker/ansible/tasks/gear/status.yaml b/baker/ansible/tasks/gear/capture.yaml similarity index 76% rename from baker/ansible/tasks/gear/status.yaml rename to baker/ansible/tasks/gear/capture.yaml index da51e6c3..36f09e56 100644 --- a/baker/ansible/tasks/gear/status.yaml +++ b/baker/ansible/tasks/gear/capture.yaml @@ -1,5 +1,5 @@ --- -- name: Capture cid +- name: Capture CID ansible.builtin.command: cmd: git hash-object --stdin stdin: | @@ -12,11 +12,11 @@ - name: Declare tag ansible.builtin.set_fact: - gear_tag: "{{ devenv }}-{{ stack_cid.stdout[:7] }}" + stack_tag: "{{ devenv }}-{{ stack_cid.stdout[:7] }}" - name: Capture status ansible.builtin.command: - cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ gear.image }}:{{ gear_tag }} + cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ gear.image }}:{{ stack_tag }} register: stack_status changed_when: - stack_status.rc != 0 @@ -26,3 +26,10 @@ ansible.builtin.debug: msg: "{{ stack_status.cmd|join(' ') }}" when: stack_status is changed + +- name: Capture env + ansible.builtin.command: + cmd: test -f /.dockerenv + register: is_docker_gear + changed_when: false + failed_when: false diff --git a/baker/ansible/tasks/gear/package.yaml b/baker/ansible/tasks/gear/package.yaml deleted file mode 100644 index d3128bc7..00000000 --- a/baker/ansible/tasks/gear/package.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- ansible.builtin.import_role: # noqa: name[missing] - name: image - vars: - image_tag: "{{ gear_tag }}" - image_name: "{{ image_ns }}/{{ gear.image }}" - image_home: "{{ project.dir }}/stack/gear/{{ devenv }}" - image_key: "{{ gear.image }}/{{ devenv }}" - image_push: "{{ image_repo != 'local' }}" diff --git a/baker/ansible/tasks/gear/publish.yaml b/baker/ansible/tasks/gear/publish.yaml new file mode 100644 index 00000000..90f20e32 --- /dev/null +++ b/baker/ansible/tasks/gear/publish.yaml @@ -0,0 +1,8 @@ +--- +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: publish + vars: + source_tag: "{{ gear.image }}:{{ devenv }}" + target_tags: + - "{{ image_ns }}/{{ gear.image }}:{{ hostvars.gear.stack_tag }}" diff --git a/baker/ansible/tasks/product/build.yaml b/baker/ansible/tasks/product/build.yaml index 55302072..0db6aad7 100644 --- a/baker/ansible/tasks/product/build.yaml +++ b/baker/ansible/tasks/product/build.yaml @@ -25,3 +25,12 @@ lstrip_blocks: yes trim_blocks: yes mode: "644" + +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: build + vars: + image_home: "{{ project.dir }}/stack/product" + image_context: target/image-context + image_key: "{{ product.image }}/{{ opsenv }}" + image_tag: "{{ product.image }}:{{ opsenv }}" diff --git a/baker/ansible/tasks/product/status.yaml b/baker/ansible/tasks/product/capture.yaml similarity index 86% rename from baker/ansible/tasks/product/status.yaml rename to baker/ansible/tasks/product/capture.yaml index 40450cd5..31959060 100644 --- a/baker/ansible/tasks/product/status.yaml +++ b/baker/ansible/tasks/product/capture.yaml @@ -1,5 +1,5 @@ --- -- name: Capture cid +- name: Capture CID ansible.builtin.command: cmd: git hash-object --stdin stdin: | @@ -12,11 +12,11 @@ - name: Declare tag ansible.builtin.set_fact: - product_tag: "{{ purpose }}-{{ opsenv }}-{{ stack_cid.stdout[:7] }}" + stack_tag: "{{ opsenv }}-{{ stack_cid.stdout[:7] }}" - name: Capture status ansible.builtin.command: - cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ product.image }}:{{ product_tag }} + cmd: docker {{ docker_entity }} inspect {{ image_ns }}/{{ product.image }}:{{ stack_tag }} register: stack_status changed_when: - stack_status.rc != 0 diff --git a/baker/ansible/tasks/product/test.yaml b/baker/ansible/tasks/product/check.yaml similarity index 97% rename from baker/ansible/tasks/product/test.yaml rename to baker/ansible/tasks/product/check.yaml index 0538730d..bb4110ca 100644 --- a/baker/ansible/tasks/product/test.yaml +++ b/baker/ansible/tasks/product/check.yaml @@ -34,7 +34,7 @@ --batch-mode --threads 2 --projects test/e2e - {% if image_repo != 'local' %} + {% if is_docker_gear.rc == 0 %} --activate-profiles docker {% endif %} --also-make diff --git a/baker/ansible/tasks/product/package.yaml b/baker/ansible/tasks/product/package.yaml deleted file mode 100644 index 8c589bc0..00000000 --- a/baker/ansible/tasks/product/package.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- ansible.builtin.import_role: # noqa: name[missing] - name: image - vars: - image_tag: "{{ product_tag }}" - image_name: "{{ image_ns }}/{{ product.image }}" - image_home: "{{ project.dir }}/stack/product" - image_context: target/image-context - image_key: "{{ product.image }}/{{ purpose }}-{{ opsenv }}" - image_push: "{{ image_repo != 'local' }}" diff --git a/baker/ansible/tasks/product/publish.yaml b/baker/ansible/tasks/product/publish.yaml new file mode 100644 index 00000000..8e979015 --- /dev/null +++ b/baker/ansible/tasks/product/publish.yaml @@ -0,0 +1,8 @@ +--- +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: publish + vars: + source_tag: "{{ product.image }}:{{ opsenv }}" + target_tags: + - "{{ image_ns }}/{{ product.image }}:{{ hostvars.product.stack_tag }}" diff --git a/stack/gear/Dockerfile b/stack/gear/Dockerfile new file mode 100644 index 00000000..65f8bd1e --- /dev/null +++ b/stack/gear/Dockerfile @@ -0,0 +1,3 @@ +# native environment lightweight image +FROM scratch +COPY Dockerfile . From 55943c96bc84049c8c1a1d194e49789f49e7e310 Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 23 Feb 2024 10:32:27 +0300 Subject: [PATCH 3/8] Define devenv in pom file --- app/pom.xml | 10 ++-- .../inventory/group_vars/all/envs.yaml | 12 ----- baker/ansible/sources.yaml | 2 +- baker/ansible/stacks.yaml | 4 +- baker/ansible/tasks/app/build-and-check.yaml | 7 +-- baker/ansible/tasks/gear/build.yaml | 8 ++- baker/ansible/tasks/product/build.yaml | 7 +++ baker/ansible/tasks/product/check.yaml | 6 +-- baker/ansible/tasks/product/clean.yaml | 12 +++++ pom.xml | 49 +++++++++++++++++-- 10 files changed, 85 insertions(+), 32 deletions(-) create mode 100644 baker/ansible/tasks/product/clean.yaml diff --git a/app/pom.xml b/app/pom.xml index 3f80500a..dc2e8dec 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -66,7 +66,7 @@ maven-dependency-plugin - deps-context + deps-jars pre-integration-test copy-dependencies @@ -78,7 +78,7 @@ - lib-context + libs-jars pre-integration-test copy-dependencies @@ -90,7 +90,7 @@ - app-context + app-jar pre-integration-test copy @@ -128,9 +128,9 @@ --file ${project.basedir}/Dockerfile --tag - app/${project.artifactId}:godel + app/${artifactId}:${devenv} --label - image.key=app/${project.artifactId}/godel + image.key=app/${artifactId}/${devenv} --build-arg JAVA_RELEASE=${maven.compiler.release} diff --git a/baker/ansible/inventory/group_vars/all/envs.yaml b/baker/ansible/inventory/group_vars/all/envs.yaml index 21137db5..a292d83b 100644 --- a/baker/ansible/inventory/group_vars/all/envs.yaml +++ b/baker/ansible/inventory/group_vars/all/envs.yaml @@ -1,17 +1,5 @@ --- devenv: turing -dev: - church: - jdk: - release: 21 - kotlin: - version: 2.0.0-Beta4 - turing: - jdk: - release: 21 - godel: - jdk: - release: 17 opsenv: durkheim ops: diff --git a/baker/ansible/sources.yaml b/baker/ansible/sources.yaml index 2d213bda..97a74dcd 100644 --- a/baker/ansible/sources.yaml +++ b/baker/ansible/sources.yaml @@ -14,7 +14,7 @@ pmd:check compiler:compile spotbugs:check - --define maven.compiler.release={{ dev[devenv].jdk.release }} + --define devenv={{ devenv }} chdir: "{{ project.dir }}" strip_empty_ends: false changed_when: false diff --git a/baker/ansible/stacks.yaml b/baker/ansible/stacks.yaml index 2b9a44af..e938da24 100644 --- a/baker/ansible/stacks.yaml +++ b/baker/ansible/stacks.yaml @@ -20,7 +20,9 @@ - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/gear/build.yaml tags: [build] - when: hostvars.gear.stack_status is changed + when: >- + hostvars.gear.stack_status is changed and + repo_mode != 'rw' - name: Building & Checking hosts: product diff --git a/baker/ansible/tasks/app/build-and-check.yaml b/baker/ansible/tasks/app/build-and-check.yaml index ba646387..56e9bc16 100644 --- a/baker/ansible/tasks/app/build-and-check.yaml +++ b/baker/ansible/tasks/app/build-and-check.yaml @@ -16,11 +16,8 @@ }} --also-make clean - {{ maven_phase|mandatory }} - --define maven.compiler.release={{ dev[devenv].jdk.release }} - {% if dev[devenv].kotlin is defined %} - --define kotlin.version={{ dev[devenv].kotlin.version }} - {% endif %} + {{ maven_phase | mandatory }} + --define devenv={{ devenv }} chdir: "{{ project.dir }}" strip_empty_ends: false changed_when: true diff --git a/baker/ansible/tasks/gear/build.yaml b/baker/ansible/tasks/gear/build.yaml index 73f160ab..60be7f7b 100644 --- a/baker/ansible/tasks/gear/build.yaml +++ b/baker/ansible/tasks/gear/build.yaml @@ -1,4 +1,11 @@ --- +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: clean + vars: + image_keys: + - "{{ gear.image }}/{{ devenv }}" + - ansible.builtin.import_role: # noqa: name[missing] name: image tasks_from: build @@ -6,4 +13,3 @@ image_home: "{{ project.dir }}/stack/gear" image_key: "{{ gear.image }}/{{ devenv }}" image_tag: "{{ gear.image }}:{{ devenv }}" - when: repo_mode != 'rw' diff --git a/baker/ansible/tasks/product/build.yaml b/baker/ansible/tasks/product/build.yaml index 0db6aad7..2b51e7bf 100644 --- a/baker/ansible/tasks/product/build.yaml +++ b/baker/ansible/tasks/product/build.yaml @@ -1,4 +1,11 @@ --- +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: clean + vars: + image_keys: + - "{{ product.image }}/{{ opsenv }}" + - name: Remove artifacts ansible.builtin.file: path: "{{ stack_dir }}/target" diff --git a/baker/ansible/tasks/product/check.yaml b/baker/ansible/tasks/product/check.yaml index bb4110ca..bd7e5754 100644 --- a/baker/ansible/tasks/product/check.yaml +++ b/baker/ansible/tasks/product/check.yaml @@ -34,12 +34,14 @@ --batch-mode --threads 2 --projects test/e2e - {% if is_docker_gear.rc == 0 %} + {% if hostvars.gear.is_docker_gear.rc == 0 %} --activate-profiles docker {% endif %} --also-make clean test + --define devenv={{ devenv }} + --define skipUnits=true --define props={{ opsenv }} {% if reminder is defined %} --define testing.sharding.reminder={{ reminder }} @@ -47,8 +49,6 @@ {% if modulus is defined %} --define testing.sharding.modulus={{ modulus }} {% endif %} - --define maven.compiler.release={{ dev[devenv].jdk.release }} - --define skipUnits=true chdir: "{{ project.dir }}" strip_empty_ends: false changed_when: true diff --git a/baker/ansible/tasks/product/clean.yaml b/baker/ansible/tasks/product/clean.yaml new file mode 100644 index 00000000..bdfb10bc --- /dev/null +++ b/baker/ansible/tasks/product/clean.yaml @@ -0,0 +1,12 @@ +--- +- name: Remove artifacts + ansible.builtin.file: + path: "{{ stack_dir }}/target" + state: absent + +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: clean + vars: + image_keys: + - "{{ product.image }}/{{ opsenv }}" diff --git a/pom.xml b/pom.xml index 6c8ca452..86fa80e3 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,55 @@ 6.1.3 1.9.22 0.8.11 - - false - - ${skipTests} + godel + false + ${skipTests} 17 UTF-8 + + + experimental + + + devenv + church + + + + 22 + 2.0.0-Beta4 + + + + stable + + + devenv + turing + + + + 21 + 1.9.22 + + + + deprecated + + + devenv + godel + + + + 17 + 1.8.22 + + + + From ddb9fa02ace0d2d31197ad783e4c8bdabdfc54dc Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 23 Feb 2024 10:44:26 +0300 Subject: [PATCH 4/8] Fix linter issues --- baker/ansible/roles/image/tasks/publish.yaml | 14 ++++++++------ baker/ansible/tasks/app/publish.yaml | 4 ++-- baker/ansible/tasks/db/publish.yaml | 4 ++-- baker/ansible/tasks/gear/publish.yaml | 4 ++-- baker/ansible/tasks/product/publish.yaml | 4 ++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/baker/ansible/roles/image/tasks/publish.yaml b/baker/ansible/roles/image/tasks/publish.yaml index b5f04d4c..984d1ef4 100644 --- a/baker/ansible/roles/image/tasks/publish.yaml +++ b/baker/ansible/roles/image/tasks/publish.yaml @@ -3,18 +3,20 @@ ansible.builtin.assert: quiet: true that: - - source_tag is defined - - target_tags is defined + - image_source_tag is defined + - image_target_tags is defined - name: Tag images ansible.builtin.command: - cmd: docker tag {{ source_tag }} {{ item }} + cmd: docker tag {{ image_source_tag }} {{ item }} strip_empty_ends: false - loop: "{{ target_tags }}" + changed_when: true + loop: "{{ image_target_tags }}" - name: Push images ansible.builtin.command: cmd: docker push {{ item }} strip_empty_ends: false - loop: "{{ target_tags }}" - tags: [push, never] + changed_when: true + loop: "{{ image_target_tags }}" + tags: [never] diff --git a/baker/ansible/tasks/app/publish.yaml b/baker/ansible/tasks/app/publish.yaml index bdeaac3f..868d701a 100644 --- a/baker/ansible/tasks/app/publish.yaml +++ b/baker/ansible/tasks/app/publish.yaml @@ -3,8 +3,8 @@ name: image tasks_from: publish vars: - source_tag: "{{ app.image }}:{{ devenv }}" - target_tags: + image_source_tag: "{{ app.image }}:{{ devenv }}" + image_target_tags: - "{{ image_ns }}/{{ app.image }}:{{ hostvars.app.image_tags[app.binary] }}" loop: "{{ hostvars.app.image_status.results|select('changed')|map(attribute='app') }}" loop_control: diff --git a/baker/ansible/tasks/db/publish.yaml b/baker/ansible/tasks/db/publish.yaml index 2a127563..c4eb4c87 100644 --- a/baker/ansible/tasks/db/publish.yaml +++ b/baker/ansible/tasks/db/publish.yaml @@ -3,8 +3,8 @@ name: image tasks_from: publish vars: - source_tag: "{{ db.image }}:{{ devenv }}" - target_tags: + image_source_tag: "{{ db.image }}:{{ devenv }}" + image_target_tags: - "{{ image_ns }}/{{ db.image }}:{{ db.cid }}" loop: "{{ hostvars.db.image_status.results|select('changed')|map(attribute='db') }}" loop_control: diff --git a/baker/ansible/tasks/gear/publish.yaml b/baker/ansible/tasks/gear/publish.yaml index 90f20e32..794f8781 100644 --- a/baker/ansible/tasks/gear/publish.yaml +++ b/baker/ansible/tasks/gear/publish.yaml @@ -3,6 +3,6 @@ name: image tasks_from: publish vars: - source_tag: "{{ gear.image }}:{{ devenv }}" - target_tags: + image_source_tag: "{{ gear.image }}:{{ devenv }}" + image_target_tags: - "{{ image_ns }}/{{ gear.image }}:{{ hostvars.gear.stack_tag }}" diff --git a/baker/ansible/tasks/product/publish.yaml b/baker/ansible/tasks/product/publish.yaml index 8e979015..7f750d78 100644 --- a/baker/ansible/tasks/product/publish.yaml +++ b/baker/ansible/tasks/product/publish.yaml @@ -3,6 +3,6 @@ name: image tasks_from: publish vars: - source_tag: "{{ product.image }}:{{ opsenv }}" - target_tags: + image_source_tag: "{{ product.image }}:{{ opsenv }}" + image_target_tags: - "{{ image_ns }}/{{ product.image }}:{{ hostvars.product.stack_tag }}" From c81274ef592fb46a828b722568007834072db1a7 Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 23 Feb 2024 10:50:09 +0300 Subject: [PATCH 5/8] Downgrade experimental jdk release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 86fa80e3..b92da444 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ - 22 + 21 2.0.0-Beta4 From 24bf2def6e04b2dd53808b4441ce9579ea59b927 Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 23 Feb 2024 11:01:36 +0300 Subject: [PATCH 6/8] Publish in rw repo mode --- .github/workflows/beyond-doubt.yaml | 9 +++++---- app/pom.xml | 4 ++-- baker/ansible/roles/image/tasks/publish.yaml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/beyond-doubt.yaml b/.github/workflows/beyond-doubt.yaml index a521166a..7b3b110b 100644 --- a/.github/workflows/beyond-doubt.yaml +++ b/.github/workflows/beyond-doubt.yaml @@ -102,6 +102,7 @@ jobs: run: >- docker run --rm --volume $(pwd):$(pwd) + --volume /var/run/docker.sock:/var/run/docker.sock --workdir $(pwd)/baker/ansible ${{ env.IMAGE_NAME }}:${{ matrix.env.dev }} ansible-playbook stacks.yaml -t build @@ -114,8 +115,8 @@ jobs: path: stack/product/target/image-context if-no-files-found: ignore - stack-testing: - name: Stack testing (${{ matrix.shard }}, ${{ matrix.purpose }}, ${{ matrix.env.ops }}) + stack-checking: + name: Stack checking (${{ matrix.shard }}, ${{ matrix.purpose }}, ${{ matrix.env.ops }}) timeout-minutes: 15 needs: [stack-building] strategy: @@ -170,7 +171,7 @@ jobs: stack-publishing: name: Stack publishing (${{ matrix.purpose }}, ${{ matrix.env.ops }}) timeout-minutes: 10 - needs: [stack-testing] + needs: [stack-checking] strategy: fail-fast: false matrix: @@ -216,7 +217,7 @@ jobs: stack-reporting: name: Stack reporting timeout-minutes: 5 - needs: [stack-testing] + needs: [stack-checking] if: ${{ !cancelled() && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')) }} runs-on: ubuntu-22.04 permissions: diff --git a/app/pom.xml b/app/pom.xml index dc2e8dec..4bd6d59d 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -128,9 +128,9 @@ --file ${project.basedir}/Dockerfile --tag - app/${artifactId}:${devenv} + app/${project.artifactId}:${devenv} --label - image.key=app/${artifactId}/${devenv} + image.key=app/${project.artifactId}/${devenv} --build-arg JAVA_RELEASE=${maven.compiler.release} diff --git a/baker/ansible/roles/image/tasks/publish.yaml b/baker/ansible/roles/image/tasks/publish.yaml index 984d1ef4..7d65c0f5 100644 --- a/baker/ansible/roles/image/tasks/publish.yaml +++ b/baker/ansible/roles/image/tasks/publish.yaml @@ -19,4 +19,4 @@ strip_empty_ends: false changed_when: true loop: "{{ image_target_tags }}" - tags: [never] + when: repo_mode == 'rw' From daab895f98da6284d4aa760834980c7f8ce783dd Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 23 Feb 2024 11:25:41 +0300 Subject: [PATCH 7/8] Build product stack right before publish --- baker/ansible/tasks/product/build.yaml | 16 ---------------- baker/ansible/tasks/product/publish.yaml | 16 ++++++++++++++++ pom.xml | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/baker/ansible/tasks/product/build.yaml b/baker/ansible/tasks/product/build.yaml index 2b51e7bf..55302072 100644 --- a/baker/ansible/tasks/product/build.yaml +++ b/baker/ansible/tasks/product/build.yaml @@ -1,11 +1,4 @@ --- -- ansible.builtin.import_role: # noqa: name[missing] - name: image - tasks_from: clean - vars: - image_keys: - - "{{ product.image }}/{{ opsenv }}" - - name: Remove artifacts ansible.builtin.file: path: "{{ stack_dir }}/target" @@ -32,12 +25,3 @@ lstrip_blocks: yes trim_blocks: yes mode: "644" - -- ansible.builtin.import_role: # noqa: name[missing] - name: image - tasks_from: build - vars: - image_home: "{{ project.dir }}/stack/product" - image_context: target/image-context - image_key: "{{ product.image }}/{{ opsenv }}" - image_tag: "{{ product.image }}:{{ opsenv }}" diff --git a/baker/ansible/tasks/product/publish.yaml b/baker/ansible/tasks/product/publish.yaml index 7f750d78..37f3bece 100644 --- a/baker/ansible/tasks/product/publish.yaml +++ b/baker/ansible/tasks/product/publish.yaml @@ -1,4 +1,20 @@ --- +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: clean + vars: + image_keys: + - "{{ product.image }}/{{ opsenv }}" + +- ansible.builtin.import_role: # noqa: name[missing] + name: image + tasks_from: build + vars: + image_home: "{{ project.dir }}/stack/product" + image_context: target/image-context + image_key: "{{ product.image }}/{{ opsenv }}" + image_tag: "{{ product.image }}:{{ opsenv }}" + - ansible.builtin.import_role: # noqa: name[missing] name: image tasks_from: publish diff --git a/pom.xml b/pom.xml index b92da444..4c253e46 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ 1.18.30 1.5.5.Final 6.1.3 - 1.9.22 + 1.8.22 0.8.11 godel false From 12bd1fd03752200186822c1dcf96bda82121adbd Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 23 Feb 2024 11:39:45 +0300 Subject: [PATCH 8/8] Fix review notes --- baker/ansible/tasks/gear/capture.yaml | 2 +- baker/ansible/tasks/product/check.yaml | 2 +- baker/ansible/tasks/product/clean.yaml | 12 ------------ 3 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 baker/ansible/tasks/product/clean.yaml diff --git a/baker/ansible/tasks/gear/capture.yaml b/baker/ansible/tasks/gear/capture.yaml index 36f09e56..6b279ce7 100644 --- a/baker/ansible/tasks/gear/capture.yaml +++ b/baker/ansible/tasks/gear/capture.yaml @@ -30,6 +30,6 @@ - name: Capture env ansible.builtin.command: cmd: test -f /.dockerenv - register: is_docker_gear + register: is_virtual changed_when: false failed_when: false diff --git a/baker/ansible/tasks/product/check.yaml b/baker/ansible/tasks/product/check.yaml index bd7e5754..34724be1 100644 --- a/baker/ansible/tasks/product/check.yaml +++ b/baker/ansible/tasks/product/check.yaml @@ -34,7 +34,7 @@ --batch-mode --threads 2 --projects test/e2e - {% if hostvars.gear.is_docker_gear.rc == 0 %} + {% if hostvars.gear.is_virtual.rc == 0 %} --activate-profiles docker {% endif %} --also-make diff --git a/baker/ansible/tasks/product/clean.yaml b/baker/ansible/tasks/product/clean.yaml deleted file mode 100644 index bdfb10bc..00000000 --- a/baker/ansible/tasks/product/clean.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Remove artifacts - ansible.builtin.file: - path: "{{ stack_dir }}/target" - state: absent - -- ansible.builtin.import_role: # noqa: name[missing] - name: image - tasks_from: clean - vars: - image_keys: - - "{{ product.image }}/{{ opsenv }}"