From 6db3955fdfd39eb4ae6e341db529f6639c0242ab Mon Sep 17 00:00:00 2001 From: Pavel Vetokhin Date: Fri, 8 Mar 2024 19:26:20 +0300 Subject: [PATCH] Resilience test passed --- baker/ansible/roles/image/tasks/clean.yaml | 20 +++++-- baker/ansible/tasks/product/check.yaml | 2 +- .../focus/resilience/compose.yaml.jinja | 55 ++++++++++++++++--- stack/product/focus/resilience/pg_hba.conf | 1 + .../focus/resilience/postgresql.conf.jinja | 2 +- .../resilience/AbstractResilienceTest.java | 31 +++++++---- ...kaTest.java => StorageResilienceTest.java} | 10 +--- tool/pom.xml | 5 ++ tool/testing/pom.xml | 6 ++ .../smecalculus/bezmen/testing/Demiurge.java | 16 +++++- 10 files changed, 112 insertions(+), 36 deletions(-) rename test/e2e/src/test/java/smecalculus/bezmen/resilience/{SepulkaTest.java => StorageResilienceTest.java} (82%) diff --git a/baker/ansible/roles/image/tasks/clean.yaml b/baker/ansible/roles/image/tasks/clean.yaml index 4b84279c..162e332f 100644 --- a/baker/ansible/roles/image/tasks/clean.yaml +++ b/baker/ansible/roles/image/tasks/clean.yaml @@ -5,22 +5,32 @@ that: - image_keys is defined -- name: Kill containers +- name: Remove containers ansible.builtin.shell: cmd: docker container rm -fv $(docker ps -aq --filter label=image.key={{ item }}) strip_empty_ends: false - register: kill_outcome + register: container_outcome failed_when: false changed_when: - - kill_outcome.rc == 0 + - container_outcome.rc == 0 + loop: "{{ image_keys }}" + +- name: Remove volumes + ansible.builtin.shell: + cmd: docker volume rm -f $(docker volume ls -q --filter label=image.key={{ item }}) + strip_empty_ends: false + register: volume_outcome + failed_when: false + changed_when: + - volume_outcome.rc == 0 loop: "{{ image_keys }}" - name: Remove images ansible.builtin.shell: cmd: docker image rm -f $(docker images -q --filter label=image.key={{ item }}) strip_empty_ends: false - register: remove_outcome + register: image_outcome failed_when: false changed_when: - - remove_outcome.rc == 0 + - image_outcome.rc == 0 loop: "{{ image_keys }}" diff --git a/baker/ansible/tasks/product/check.yaml b/baker/ansible/tasks/product/check.yaml index 95e0bb78..29d5fd23 100644 --- a/baker/ansible/tasks/product/check.yaml +++ b/baker/ansible/tasks/product/check.yaml @@ -12,8 +12,8 @@ --file compose.yaml up --remove-orphans + --pull missing --quiet-pull - --no-deps --detach chdir: "{{ stack_dir }}/target/{{ focus }}" strip_empty_ends: false diff --git a/stack/product/focus/resilience/compose.yaml.jinja b/stack/product/focus/resilience/compose.yaml.jinja index bac5e8fc..5d0de5aa 100644 --- a/stack/product/focus/resilience/compose.yaml.jinja +++ b/stack/product/focus/resilience/compose.yaml.jinja @@ -21,6 +21,11 @@ configs: product: file: {{ ops.config.file_name }} +volumes: + postgres_data: + labels: + image.key: db/{{ ops.storage.vendor }}/{{ devenv }} + services: {{ ops.storage.vendor }}-primary: container_name: {{ project.name }}-{{ ops.storage.vendor }}-primary @@ -34,23 +39,50 @@ services: POSTGRES_PASSWORD: "{{ storage.dba.password }}" configs: - source: postgresql - target: /etc/postgres/postgresql.conf + target: /postgres/conf/postgresql.conf - source: pg_hba - target: /etc/postgres/pg_hba.conf + target: /postgres/conf/pg_hba.conf healthcheck: test: ["CMD-SHELL", "pg_isready"] interval: 10s timeout: 5s retries: 5 command: >- - -c config_file=/etc/postgres/postgresql.conf - -c hba_file=/etc/postgres/pg_hba.conf + -c config_file=/postgres/conf/postgresql.conf + -c hba_file=/postgres/conf/pg_hba.conf + labels: + image.key: db/{{ ops.storage.vendor }}/{{ devenv }} + + {{ ops.storage.vendor }}-backup: + container_name: {{ project.name }}-{{ ops.storage.vendor }}-backup + image: {{ storage.image }}:{{ ops.storage.version }}-alpine + depends_on: + {{ ops.storage.vendor }}-primary: + condition: service_healthy + restart: false + networks: + - product + environment: + PGDATA: /postgres/data + volumes: + - source: postgres_data + target: /postgres/data + type: volume + entrypoint: [] + command: >- + pg_basebackup --write-recovery-conf + --host {{ ops.storage.vendor }}-primary + --username {{ storage.dba.username }} + --pgdata /postgres/data labels: image.key: db/{{ ops.storage.vendor }}/{{ devenv }} {{ ops.storage.vendor }}-secondary: container_name: {{ project.name }}-{{ ops.storage.vendor }}-secondary image: {{ storage.image }}:{{ ops.storage.version }}-alpine + depends_on: + {{ ops.storage.vendor }}-backup: + condition: service_completed_successfully networks: - product ports: @@ -58,19 +90,24 @@ services: environment: POSTGRES_USER: "{{ storage.dba.username }}" POSTGRES_PASSWORD: "{{ storage.dba.password }}" + PGDATA: /postgres/data configs: - source: postgresql - target: /etc/postgres/postgresql.conf + target: /postgres/conf/postgresql.conf - source: pg_hba - target: /etc/postgres/pg_hba.conf + target: /postgres/conf/pg_hba.conf + volumes: + - source: postgres_data + target: /postgres/data + type: volume healthcheck: test: ["CMD-SHELL", "pg_isready"] interval: 10s timeout: 5s retries: 5 command: >- - -c config_file=/etc/postgres/postgresql.conf - -c hba_file=/etc/postgres/pg_hba.conf + -c config_file=/postgres/conf/postgresql.conf + -c hba_file=/postgres/conf/pg_hba.conf labels: image.key: db/{{ ops.storage.vendor }}/{{ devenv }} @@ -78,7 +115,7 @@ services: container_name: {{ project.name }}-database-{{ database.name }} image: {{ image_ns }}/db/{{ ops.storage.vendor }}:{{ hostvars.db.image_tags[ops.storage.vendor] }} depends_on: - {{ ops.storage.vendor }}-primary: + {{ ops.storage.vendor }}-secondary: condition: service_healthy networks: - product diff --git a/stack/product/focus/resilience/pg_hba.conf b/stack/product/focus/resilience/pg_hba.conf index c66b9a9a..55d92975 100644 --- a/stack/product/focus/resilience/pg_hba.conf +++ b/stack/product/focus/resilience/pg_hba.conf @@ -1,3 +1,4 @@ # TYPE DATABASE USER ADDRESS METHOD local all all trust host all all all trust +host replication all all trust diff --git a/stack/product/focus/resilience/postgresql.conf.jinja b/stack/product/focus/resilience/postgresql.conf.jinja index 3d3a9145..517cdf37 100644 --- a/stack/product/focus/resilience/postgresql.conf.jinja +++ b/stack/product/focus/resilience/postgresql.conf.jinja @@ -2,4 +2,4 @@ max_wal_senders = 3 wal_level = replica listen_addresses = '*' -primary_conninfo = 'host={{ ops.storage.vendor }}-primary dbname={{ database.name }}' +primary_conninfo = 'host={{ ops.storage.vendor }}-primary' diff --git a/test/e2e/src/test/java/smecalculus/bezmen/resilience/AbstractResilienceTest.java b/test/e2e/src/test/java/smecalculus/bezmen/resilience/AbstractResilienceTest.java index d19e3124..464e3c07 100644 --- a/test/e2e/src/test/java/smecalculus/bezmen/resilience/AbstractResilienceTest.java +++ b/test/e2e/src/test/java/smecalculus/bezmen/resilience/AbstractResilienceTest.java @@ -1,37 +1,48 @@ package smecalculus.bezmen.resilience; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.groupingBy; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.model.Container; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; import smecalculus.bezmen.construction.StandBeans; +import smecalculus.bezmen.messaging.BezmenClient; import smecalculus.bezmen.testing.Demiurge; @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = StandBeans.class) public abstract class AbstractResilienceTest { + protected static final String POSTGRES_PRIMARY = "postgres-primary"; + protected static final String POSTGRES_SECONDARY = "postgres-secondary"; + + @Autowired + private DockerClient dockerClient; + @Autowired - protected DockerClient dockerClient; + protected BezmenClient bezmenClient; @Autowired protected Demiurge demiurge; - protected Container postgresPrimary; - protected Container postgresSecondary; + protected Map> services; + + static { + Awaitility.pollInSameThread(); + Awaitility.setDefaultPollInterval(Duration.ofSeconds(1)); + } @BeforeAll void abstractBeforeAll() { - var containers = dockerClient.listContainersCmd().withShowAll(true).exec().stream() - .collect(toMap(container -> container.getNames()[0], identity())); - containers.values().forEach(demiurge::starts); - postgresPrimary = containers.get("/bezmen-postgres-primary"); - postgresSecondary = containers.get("/bezmen-postgres-secondary"); + services = dockerClient.listContainersCmd().withShowAll(true).exec().stream() + .collect(groupingBy(container -> container.getLabels().get("com.docker.compose.service"))); } } diff --git a/test/e2e/src/test/java/smecalculus/bezmen/resilience/SepulkaTest.java b/test/e2e/src/test/java/smecalculus/bezmen/resilience/StorageResilienceTest.java similarity index 82% rename from test/e2e/src/test/java/smecalculus/bezmen/resilience/SepulkaTest.java rename to test/e2e/src/test/java/smecalculus/bezmen/resilience/StorageResilienceTest.java index ddf8b225..6fc7ecb8 100644 --- a/test/e2e/src/test/java/smecalculus/bezmen/resilience/SepulkaTest.java +++ b/test/e2e/src/test/java/smecalculus/bezmen/resilience/StorageResilienceTest.java @@ -9,17 +9,13 @@ import java.util.UUID; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import smecalculus.bezmen.messaging.BezmenClient; import smecalculus.bezmen.messaging.SepulkaMessageEmEg; -public class SepulkaTest extends AbstractResilienceTest { - - @Autowired - private BezmenClient bezmenClient; +public class StorageResilienceTest extends AbstractResilienceTest { @BeforeAll void beforeAll() { + demiurge.starts(services.get(POSTGRES_PRIMARY)); await("isReady").atMost(ofSeconds(5)).until(bezmenClient::isReady); } @@ -32,7 +28,7 @@ void shouldViewSepulka() { // and bezmenClient.register(registrationRequest(externalId)); // and - demiurge.kills(postgresPrimary); + demiurge.kills(services.get(POSTGRES_PRIMARY)); // and var expectedResponse = SepulkaMessageEmEg.viewResponse(registrationRequest.getExternalId()); // when diff --git a/tool/pom.xml b/tool/pom.xml index fd6ece70..99de3e32 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -75,6 +75,11 @@ slf4j-api 2.0.12 + + ch.qos.logback + logback-classic + 1.5.1 + com.fasterxml.jackson.core jackson-annotations diff --git a/tool/testing/pom.xml b/tool/testing/pom.xml index 45e9e1c6..6f4abde7 100644 --- a/tool/testing/pom.xml +++ b/tool/testing/pom.xml @@ -47,6 +47,12 @@ compile + + ch.qos.logback + logback-classic + runtime + + com.github.docker-java docker-java-core diff --git a/tool/testing/src/main/java/smecalculus/bezmen/testing/Demiurge.java b/tool/testing/src/main/java/smecalculus/bezmen/testing/Demiurge.java index 087ae4c3..37b52f7c 100644 --- a/tool/testing/src/main/java/smecalculus/bezmen/testing/Demiurge.java +++ b/tool/testing/src/main/java/smecalculus/bezmen/testing/Demiurge.java @@ -6,6 +6,7 @@ import com.github.dockerjava.api.exception.NotModifiedException; import com.github.dockerjava.api.model.Container; import java.time.Duration; +import java.util.List; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.slf4j.Logger; @@ -16,22 +17,31 @@ public class Demiurge { private static final Logger LOG = LoggerFactory.getLogger(Demiurge.class); + private static final Duration SERVICE_START_TIMEOUT = Duration.ofSeconds(20); + @NonNull private DockerClient dockerClient; - public void starts(@NonNull Container container) { + public void starts(@NonNull List containers) { try { - dockerClient.startContainerCmd(container.getId()).exec(); - await().atMost(Duration.ofSeconds(20)).until(() -> isStarted(container)); + containers.forEach(container -> dockerClient.startContainerCmd(container.getId()).exec()); } catch (NotModifiedException e) { LOG.debug("Already started"); } + containers.forEach(container -> + await(container.getNames()[0]) + .atMost(SERVICE_START_TIMEOUT) + .until(() -> isStarted(container))); } public void kills(@NonNull Container container) { dockerClient.killContainerCmd(container.getId()).exec(); } + public void kills(@NonNull List containers) { + containers.forEach(this::kills); + } + private boolean isStarted(@NonNull Container container) { var response = dockerClient.inspectContainerCmd(container.getId()).exec(); if (response.getConfig().getHealthcheck() == null) {