From 8f01f2f7d08c9baad36e2607cd629894fa999caf Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Wed, 6 Sep 2023 21:45:33 +0300 Subject: [PATCH] Introduce focus concept --- .dx/binaries.yml | 16 ++++--- .dx/group_vars/all.yml | 27 ++++++------ .dx/images.yml | 10 +++-- .dx/inventory/aliases.yml | 6 ++- .dx/roles/image/defaults/main.yml | 1 + .dx/roles/image/tasks/main.yml | 2 +- .dx/solution.yml | 48 +++++++++++++-------- .dx/sources.yml | 4 ++ .dx/tasks/pipeline/package.yml | 3 +- .dx/tasks/pipeline/status.yml | 7 ++- .dx/tasks/solution/compile.yml | 2 +- .dx/tasks/solution/package.yml | 3 +- .dx/tasks/solution/status.yml | 7 ++- .github/workflows/pipeline-beyond-doubt.yml | 5 ++- .github/workflows/solution-beyond-doubt.yml | 17 +++++--- .github/workflows/solution-convincing.yml | 4 +- 16 files changed, 97 insertions(+), 65 deletions(-) diff --git a/.dx/binaries.yml b/.dx/binaries.yml index 0618512c..4aeb0c4f 100644 --- a/.dx/binaries.yml +++ b/.dx/binaries.yml @@ -1,16 +1,17 @@ --- - name: Binaries hosts: lib + run_once: true tasks: - name: Capture status ansible.builtin.command: # todo: try to implement more lightweight check cmd: > mvn - {% if repository_mode == 'offline' %} + {% if maven_host == 'local' %} --offline {% else %} - --define remoteRepositories=github::::https://maven.pkg.github.com/smecalculus/bezmen + --define remoteRepositories=github::::https://{{ maven_host }}/smecalculus/bezmen {% endif %} dependency:get --define transitive=false @@ -33,7 +34,7 @@ --threads 2C --define revision={{ lib_version }} clean - {% if repository_mode == 'read_write' %} + {% if focus == 'solution' %} deploy {% else %} install @@ -44,19 +45,20 @@ - name: Binaries hosts: app + run_once: true tasks: - - name: Capture tag + - name: Capture cid ansible.builtin.command: cmd: git hash-object --stdin stdin: | {{ lib_cid }} {{ app_cid }} - register: image_tag + register: image_cid changed_when: false tags: [always] - name: Capture statuses ansible.builtin.command: - cmd: docker {{ docker_entity }} inspect {{ item.value }}:{{ image_tag.stdout[:7] }} + cmd: docker {{ docker_entity }} inspect {{ item.value }}:{{ image_cid.stdout[:7] }} register: image_status changed_when: image_status.rc != 0 failed_when: false @@ -90,7 +92,7 @@ when: image_status is changed - name: Coverage - hosts: lib:app + hosts: tool run_once: true tasks: - name: Generate reports diff --git a/.dx/group_vars/all.yml b/.dx/group_vars/all.yml index 77ff75bd..b7d19b09 100644 --- a/.dx/group_vars/all.yml +++ b/.dx/group_vars/all.yml @@ -17,23 +17,20 @@ app_version: 0.1.0-{{ app_cid }} usage: toy prefs: turing -# How to work with private remote repositories? -# offline - no pull, nor push packages -# read_only - can pull packages -# read_write - can pull and push packages -repository_mode: offline +# solution - focus on solution changes +# pipeline - focus on pipeline changes +# none - without any particular focus +focus: none -docker_registry: ghcr.io -docker_repo: smecalculus +maven_host: local +docker_host: local -docker_entity: "{{ 'image' if repository_mode == 'offline' else 'manifest' }}" +docker_repo: smecalculus +docker_entity: "{{ 'image' if docker_host == 'local' else 'manifest' }}" schema_images: - postgres: "{{ docker_registry }}/{{ docker_repo }}/schema-postgres" + postgres: "{{ docker_host }}/{{ docker_repo }}/schema-postgres" app_images: - foo: "{{ docker_registry }}/{{ docker_repo }}/app-foo" -solution_image: "{{ docker_registry }}/{{ docker_repo }}/solution-{{ usage }}-{{ prefs }}" -pipeline_image: "{{ docker_registry }}/{{ docker_repo }}/pipeline-{{ environ }}" - -capture_solution: true -capture_pipeline: true + foo: "{{ docker_host }}/{{ docker_repo }}/app-foo" +solution_image: "{{ docker_host }}/{{ docker_repo }}/solution-{{ usage }}-{{ prefs }}" +pipeline_image: "{{ docker_host }}/{{ docker_repo }}/pipeline-{{ environ }}" diff --git a/.dx/images.yml b/.dx/images.yml index f5fd0d29..6a2166d3 100644 --- a/.dx/images.yml +++ b/.dx/images.yml @@ -4,6 +4,7 @@ - name: Images hosts: schema + run_once: true tasks: - name: Capture statuses ansible.builtin.command: @@ -25,10 +26,12 @@ image_tag: "{{ schema_cid }}" image_name: "{{ item.value }}" image_home: "{{ playbook_dir }}/../schemas/{{ item.key }}" + image_push: "{{ focus == 'solution' }}" loop: "{{ image_status.results|select('changed')|map(attribute='item') }}" - name: Images hosts: app + run_once: true tasks: - name: Test integrations ansible.builtin.command: @@ -52,9 +55,10 @@ ansible.builtin.include_role: name: image vars: - image_tag: "{{ hostvars.app.image_tag.stdout[:7] }}" + image_tag: "{{ image_cid.stdout[:7] }}" image_name: "{{ item.value }}" image_home: "{{ playbook_dir }}/../apps/{{ item.key }}" image_context: target/docker-context - loop: "{{ hostvars.app.image_status.results|select('changed')|map(attribute='item') }}" - when: hostvars.app.image_status is changed + image_push: "{{ focus == 'solution' }}" + loop: "{{ image_status.results|select('changed')|map(attribute='item') }}" + when: image_status is changed diff --git a/.dx/inventory/aliases.yml b/.dx/inventory/aliases.yml index 133aa5e2..b140fa09 100644 --- a/.dx/inventory/aliases.yml +++ b/.dx/inventory/aliases.yml @@ -1,12 +1,14 @@ --- all: hosts: + system: + solution: lib: app: schema: - solution: + pipeline: dx: test: - pipeline: + tool: vars: ansible_connection: local diff --git a/.dx/roles/image/defaults/main.yml b/.dx/roles/image/defaults/main.yml index 62a7ca52..04a17774 100644 --- a/.dx/roles/image/defaults/main.yml +++ b/.dx/roles/image/defaults/main.yml @@ -1,3 +1,4 @@ --- image_tag: latest image_context: . +image_push: false diff --git a/.dx/roles/image/tasks/main.yml b/.dx/roles/image/tasks/main.yml index c04675a1..12becc32 100644 --- a/.dx/roles/image/tasks/main.yml +++ b/.dx/roles/image/tasks/main.yml @@ -37,4 +37,4 @@ cmd: docker image push {{ image_name }}:{{ image_tag }} strip_empty_ends: false changed_when: true - when: repository_mode == 'read_write' + when: image_push|bool diff --git a/.dx/solution.yml b/.dx/solution.yml index 49470cd6..4ed10630 100644 --- a/.dx/solution.yml +++ b/.dx/solution.yml @@ -2,39 +2,53 @@ - import_playbook: images.yml # noqa: name[play] tags: [images, deps] -- name: Solution & Pipeline +- name: Capturing hosts: solution run_once: true tasks: - - name: Capture solution - ansible.builtin.import_tasks: + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/solution/status.yml tags: [always] - - name: Capture pipeline - ansible.builtin.import_tasks: + +- name: Capturing + hosts: pipeline + run_once: true + tasks: + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/pipeline/status.yml tags: [always] - - name: Compile solution - ansible.builtin.import_tasks: + +- name: Testing + hosts: system + run_once: true + tasks: + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/solution/compile.yml tags: [compile] when: > - solution_status is changed or - pipeline_status is changed - - name: Test solution - ansible.builtin.import_tasks: + hostvars.solution.solution_status is changed or + hostvars.pipeline.pipeline_status is changed + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/solution/test.yml tags: [test] when: > - solution_status is changed or - pipeline_status is changed - - name: Package solution - ansible.builtin.import_tasks: + hostvars.solution.solution_status is changed or + hostvars.pipeline.pipeline_status is changed + +- name: Packaging + hosts: solution + run_once: true + tasks: + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/solution/package.yml tags: [package] when: solution_status is changed - - name: Package pipeline - ansible.builtin.import_tasks: + +- name: Packaging + hosts: pipeline + run_once: true + tasks: + - ansible.builtin.import_tasks: # noqa: name[missing] file: tasks/pipeline/package.yml tags: [package] when: pipeline_status is changed diff --git a/.dx/sources.yml b/.dx/sources.yml index 5a6e1fe6..1d7ebeeb 100644 --- a/.dx/sources.yml +++ b/.dx/sources.yml @@ -1,6 +1,7 @@ --- - name: Sources hosts: lib + run_once: true tasks: - name: Analyze ansible.builtin.command: @@ -16,6 +17,7 @@ - name: Sources hosts: app + run_once: true tasks: - name: Analyze ansible.builtin.command: @@ -30,6 +32,7 @@ - name: Sources hosts: test + run_once: true tasks: - name: Analyze ansible.builtin.command: @@ -44,6 +47,7 @@ - name: Sources hosts: pipeline + run_once: true tasks: - name: Analyze ansible.builtin.command: diff --git a/.dx/tasks/pipeline/package.yml b/.dx/tasks/pipeline/package.yml index 31bffefb..f076c74d 100644 --- a/.dx/tasks/pipeline/package.yml +++ b/.dx/tasks/pipeline/package.yml @@ -2,7 +2,8 @@ - ansible.builtin.import_role: # noqa: name[missing] name: image vars: - image_tag: "{{ pipeline_tag.stdout[:7] }}" + image_tag: "{{ pipeline_cid.stdout[:7] }}" image_name: "{{ pipeline_image }}" image_home: "{{ playbook_dir }}/../.github" image_context: . + image_push: "{{ focus == 'pipeline' }}" diff --git a/.dx/tasks/pipeline/status.yml b/.dx/tasks/pipeline/status.yml index ba2a7557..af1af71d 100644 --- a/.dx/tasks/pipeline/status.yml +++ b/.dx/tasks/pipeline/status.yml @@ -1,5 +1,5 @@ --- -- name: Capture pipeline tag +- name: Capture cid ansible.builtin.command: cmd: git hash-object --stdin stdin: | @@ -7,15 +7,14 @@ {{ env_cid }} {{ tool_cid }} {{ test_cid }} - register: pipeline_tag + register: pipeline_cid changed_when: false - name: Capture pipeline status ansible.builtin.command: - cmd: docker {{ docker_entity }} inspect {{ pipeline_image }}:{{ pipeline_tag.stdout[:7] }} + cmd: docker {{ docker_entity }} inspect {{ pipeline_image }}:{{ pipeline_cid.stdout[:7] }} register: pipeline_status changed_when: - - capture_pipeline|bool - pipeline_status.rc != 0 failed_when: false diff --git a/.dx/tasks/solution/compile.yml b/.dx/tasks/solution/compile.yml index 1723e59a..2944b213 100644 --- a/.dx/tasks/solution/compile.yml +++ b/.dx/tasks/solution/compile.yml @@ -25,7 +25,7 @@ environment: SCHEMA_TAG: "{{ schema_cid }}" SCHEMA_IMAGE: "{{ schema_images.postgres }}" - FOO_TAG: "{{ hostvars.app.image_tag.stdout[:7] }}" + FOO_TAG: "{{ hostvars.app.image_cid.stdout[:7] }}" FOO_IMAGE: "{{ app_images.foo }}" changed_when: true diff --git a/.dx/tasks/solution/package.yml b/.dx/tasks/solution/package.yml index d7836ff2..a8a61525 100644 --- a/.dx/tasks/solution/package.yml +++ b/.dx/tasks/solution/package.yml @@ -2,7 +2,8 @@ - ansible.builtin.import_role: # noqa: name[missing] name: image vars: - image_tag: "{{ solution_tag.stdout[:7] }}" + image_tag: "{{ solution_cid.stdout[:7] }}" image_name: "{{ solution_image }}" image_home: "{{ playbook_dir }}/../solutions" image_context: target/context + image_push: "{{ focus == 'solution' }}" diff --git a/.dx/tasks/solution/status.yml b/.dx/tasks/solution/status.yml index 031b540b..e76989e7 100644 --- a/.dx/tasks/solution/status.yml +++ b/.dx/tasks/solution/status.yml @@ -1,5 +1,5 @@ --- -- name: Capture solution tag +- name: Capture cid ansible.builtin.command: cmd: git hash-object --stdin stdin: | @@ -8,15 +8,14 @@ {{ schema_cid }} {{ solution_cid }} {{ test_cid }} - register: solution_tag + register: solution_cid changed_when: false - name: Capture solution status ansible.builtin.command: - cmd: docker {{ docker_entity }} inspect {{ solution_image }}:{{ solution_tag.stdout[:7] }} + cmd: docker {{ docker_entity }} inspect {{ solution_image }}:{{ solution_cid.stdout[:7] }} register: solution_status changed_when: - - capture_solution|bool - solution_status.rc != 0 failed_when: false diff --git a/.github/workflows/pipeline-beyond-doubt.yml b/.github/workflows/pipeline-beyond-doubt.yml index a45055d3..dbc1d09e 100644 --- a/.github/workflows/pipeline-beyond-doubt.yml +++ b/.github/workflows/pipeline-beyond-doubt.yml @@ -42,9 +42,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > ansible-playbook solution.yml + -e maven_host=maven.pkg.github.com + -e docker_host=ghcr.io -e environ=${{ matrix.environ.name }} - -e repository_mode=read_write - -e capture_solution=false + -e focus=pipeline -e usage=toy -e prefs=turing working-directory: .dx diff --git a/.github/workflows/solution-beyond-doubt.yml b/.github/workflows/solution-beyond-doubt.yml index 2c16fa67..acbe19be 100644 --- a/.github/workflows/solution-beyond-doubt.yml +++ b/.github/workflows/solution-beyond-doubt.yml @@ -6,6 +6,10 @@ on: branches: [main] workflow_dispatch: +env: + MAVEN_HOST: maven.pkg.github.com + DOCKER_HOST: ghcr.io + jobs: sources: name: Sources @@ -38,7 +42,9 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - run: > ansible-playbook images.yml - -e repository_mode=read_write + -e maven_host=${{ env.MAVEN_HOST }} + -e docker_host=${{ env.DOCKER_HOST }} + -e focus=solution working-directory: .dx env: GITHUB_ACTOR: ${{ github.actor }} @@ -88,7 +94,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - run: > ansible-playbook solution.yml -t compile - -e repository_mode=read_only + -e docker_host=${{ env.DOCKER_HOST }} -e usage=${{ matrix.usage }} -e prefs=${{ matrix.prefs }} working-directory: .dx @@ -122,8 +128,7 @@ jobs: path: solutions/target/context - run: > ansible-playbook solution.yml -t test - -e repository_mode=read_only - -e capture_pipeline=false + -e docker_host=${{ env.DOCKER_HOST }} -e test_failure_ignore=true -e usage=${{ matrix.usage }} -e prefs=${{ matrix.prefs }} @@ -165,8 +170,8 @@ jobs: path: solutions/target/context - run: > ansible-playbook solution.yml -t package - -e repository_mode=read_write - -e capture_pipeline=false + -e docker_host=${{ env.DOCKER_HOST }} + -e focus=solution -e usage=${{ matrix.usage }} -e prefs=${{ matrix.prefs }} working-directory: .dx diff --git a/.github/workflows/solution-convincing.yml b/.github/workflows/solution-convincing.yml index 8441df8d..34561737 100644 --- a/.github/workflows/solution-convincing.yml +++ b/.github/workflows/solution-convincing.yml @@ -28,7 +28,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > ansible-playbook images.yml - -e repository_mode=read_write + -e maven_host=maven.pkg.github.com + -e docker_host=ghcr.io + -e focus=solution working-directory: .dx env: GITHUB_ACTOR: ${{ github.actor }}