Resilience test passed

baker/ansible/roles/image/tasks/clean.yaml

@@ -5,22 +5,32 @@
     that:
       - image_keys is defined
 
-- name: Kill containers
+- name: Remove containers
   ansible.builtin.shell:
     cmd: docker container rm -fv $(docker ps -aq --filter label=image.key={{ item }})
     strip_empty_ends: false
-  register: kill_outcome
+  register: container_outcome
   failed_when: false
   changed_when:
-    - kill_outcome.rc == 0
+    - container_outcome.rc == 0
+  loop: "{{ image_keys }}"
+
+- name: Remove volumes
+  ansible.builtin.shell:
+    cmd: docker volume rm -f $(docker volume ls -q --filter label=image.key={{ item }})
+    strip_empty_ends: false
+  register: volume_outcome
+  failed_when: false
+  changed_when:
+    - volume_outcome.rc == 0
   loop: "{{ image_keys }}"
 
 - name: Remove images
   ansible.builtin.shell:
     cmd: docker image rm -f $(docker images -q --filter label=image.key={{ item }})
     strip_empty_ends: false
-  register: remove_outcome
+  register: image_outcome
   failed_when: false
   changed_when:
-    - remove_outcome.rc == 0
+    - image_outcome.rc == 0
   loop: "{{ image_keys }}"

baker/ansible/tasks/product/check.yaml

@@ -12,8 +12,8 @@
         --file compose.yaml
         up
         --remove-orphans
+        --pull missing
         --quiet-pull
-        --no-deps
         --detach
     chdir: "{{ stack_dir }}/target/{{ focus }}"
     strip_empty_ends: false

stack/product/focus/resilience/compose.yaml.jinja

@@ -21,6 +21,11 @@ configs:
   product:
     file: {{ ops.config.file_name }}
 
+volumes:
+  postgres_data:
+    labels:
+      image.key: db/{{ ops.storage.vendor }}/{{ devenv }}
+
 services:
   {{ ops.storage.vendor }}-primary:
     container_name: {{ project.name }}-{{ ops.storage.vendor }}-primary
@@ -34,51 +39,83 @@ services:
       POSTGRES_PASSWORD: "{{ storage.dba.password }}"
     configs:
       - source: postgresql
-        target: /etc/postgres/postgresql.conf
+        target: /postgres/conf/postgresql.conf
       - source: pg_hba
-        target: /etc/postgres/pg_hba.conf
+        target: /postgres/conf/pg_hba.conf
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 10s
       timeout: 5s
       retries: 5
     command: >-
-      -c config_file=/etc/postgres/postgresql.conf
-      -c hba_file=/etc/postgres/pg_hba.conf
+      -c config_file=/postgres/conf/postgresql.conf
+      -c hba_file=/postgres/conf/pg_hba.conf
+    labels:
+      image.key: db/{{ ops.storage.vendor }}/{{ devenv }}
+
+  {{ ops.storage.vendor }}-backup:
+    container_name: {{ project.name }}-{{ ops.storage.vendor }}-backup
+    image: {{ storage.image }}:{{ ops.storage.version }}-alpine
+    depends_on:
+      {{ ops.storage.vendor }}-primary:
+        condition: service_healthy
+        restart: false
+    networks:
+      - product
+    environment:
+      PGDATA: /postgres/data
+    volumes:
+      - source: postgres_data
+        target: /postgres/data
+        type: volume
+    entrypoint: []
+    command: >-
+      pg_basebackup --write-recovery-conf
+        --host {{ ops.storage.vendor }}-primary
+        --username {{ storage.dba.username }}
+        --pgdata /postgres/data
     labels:
       image.key: db/{{ ops.storage.vendor }}/{{ devenv }}
 
   {{ ops.storage.vendor }}-secondary:
     container_name: {{ project.name }}-{{ ops.storage.vendor }}-secondary
     image: {{ storage.image }}:{{ ops.storage.version }}-alpine
+    depends_on:
+      {{ ops.storage.vendor }}-backup:
+        condition: service_completed_successfully
     networks:
       - product
     ports:
       - "5433:5432"
     environment:
       POSTGRES_USER: "{{ storage.dba.username }}"
       POSTGRES_PASSWORD: "{{ storage.dba.password }}"
+      PGDATA: /postgres/data
     configs:
       - source: postgresql
-        target: /etc/postgres/postgresql.conf
+        target: /postgres/conf/postgresql.conf
       - source: pg_hba
-        target: /etc/postgres/pg_hba.conf
+        target: /postgres/conf/pg_hba.conf
+    volumes:
+      - source: postgres_data
+        target: /postgres/data
+        type: volume
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 10s
       timeout: 5s
       retries: 5
     command: >-
-      -c config_file=/etc/postgres/postgresql.conf
-      -c hba_file=/etc/postgres/pg_hba.conf
+      -c config_file=/postgres/conf/postgresql.conf
+      -c hba_file=/postgres/conf/pg_hba.conf
     labels:
       image.key: db/{{ ops.storage.vendor }}/{{ devenv }}
 
   database-{{ database.name }}:
     container_name: {{ project.name }}-database-{{ database.name }}
     image: {{ image_ns }}/db/{{ ops.storage.vendor }}:{{ hostvars.db.image_tags[ops.storage.vendor] }}
     depends_on:
-      {{ ops.storage.vendor }}-primary:
+      {{ ops.storage.vendor }}-secondary:
         condition: service_healthy
     networks:
       - product

stack/product/focus/resilience/pg_hba.conf

@@ -1,3 +1,4 @@
 # TYPE DATABASE USER ADDRESS METHOD
 local all all trust
 host all all all trust
+host replication all all trust

stack/product/focus/resilience/postgresql.conf.jinja

@@ -2,4 +2,4 @@
 max_wal_senders = 3
 wal_level = replica
 listen_addresses = '*'
-primary_conninfo = 'host={{ ops.storage.vendor }}-primary dbname={{ database.name }}'
+primary_conninfo = 'host={{ ops.storage.vendor }}-primary'

test/e2e/src/test/java/smecalculus/bezmen/resilience/AbstractResilienceTest.java

@@ -1,37 +1,48 @@
 package smecalculus.bezmen.resilience;
 
-import static java.util.function.Function.identity;
-import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.groupingBy;
 
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.model.Container;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 import smecalculus.bezmen.construction.StandBeans;
+import smecalculus.bezmen.messaging.BezmenClient;
 import smecalculus.bezmen.testing.Demiurge;
 
 @ExtendWith(SpringExtension.class)
 @ContextConfiguration(classes = StandBeans.class)
 public abstract class AbstractResilienceTest {
 
+    protected static final String POSTGRES_PRIMARY = "postgres-primary";
+    protected static final String POSTGRES_SECONDARY = "postgres-secondary";
+
+    @Autowired
+    private DockerClient dockerClient;
+
     @Autowired
-    protected DockerClient dockerClient;
+    protected BezmenClient bezmenClient;
 
     @Autowired
     protected Demiurge demiurge;
 
-    protected Container postgresPrimary;
-    protected Container postgresSecondary;
+    protected Map<String, List<Container>> services;
+
+    static {
+        Awaitility.pollInSameThread();
+        Awaitility.setDefaultPollInterval(Duration.ofSeconds(1));
+    }
 
     @BeforeAll
     void abstractBeforeAll() {
-        var containers = dockerClient.listContainersCmd().withShowAll(true).exec().stream()
-                .collect(toMap(container -> container.getNames()[0], identity()));
-        containers.values().forEach(demiurge::starts);
-        postgresPrimary = containers.get("/bezmen-postgres-primary");
-        postgresSecondary = containers.get("/bezmen-postgres-secondary");
+        services = dockerClient.listContainersCmd().withShowAll(true).exec().stream()
+                .collect(groupingBy(container -> container.getLabels().get("com.docker.compose.service")));
     }
 }

test/e2e/src/test/java/smecalculus/bezmen/resilience/SepulkaTest.java → test/e2e/src/test/java/smecalculus/bezmen/resilience/StorageResilienceTest.java

@@ -9,17 +9,13 @@
 import java.util.UUID;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import smecalculus.bezmen.messaging.BezmenClient;
 import smecalculus.bezmen.messaging.SepulkaMessageEmEg;
 
-public class SepulkaTest extends AbstractResilienceTest {
-
-    @Autowired
-    private BezmenClient bezmenClient;
+public class StorageResilienceTest extends AbstractResilienceTest {
 
     @BeforeAll
     void beforeAll() {
+        demiurge.starts(services.get(POSTGRES_PRIMARY));
         await("isReady").atMost(ofSeconds(5)).until(bezmenClient::isReady);
     }
 
@@ -32,7 +28,7 @@ void shouldViewSepulka() {
         // and
         bezmenClient.register(registrationRequest(externalId));
         // and
-        demiurge.kills(postgresPrimary);
+        demiurge.kills(services.get(POSTGRES_PRIMARY));
         // and
         var expectedResponse = SepulkaMessageEmEg.viewResponse(registrationRequest.getExternalId());
         // when

tool/pom.xml

@@ -75,6 +75,11 @@
         <artifactId>slf4j-api</artifactId>
         <version>2.0.12</version>
       </dependency>
+      <dependency>
+        <groupId>ch.qos.logback</groupId>
+        <artifactId>logback-classic</artifactId>
+        <version>1.5.1</version>
+      </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-annotations</artifactId>

tool/testing/pom.xml

@@ -47,6 +47,12 @@
       <scope>compile</scope>
     </dependency>
 
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.github.docker-java</groupId>
       <artifactId>docker-java-core</artifactId>

tool/testing/src/main/java/smecalculus/bezmen/testing/Demiurge.java

@@ -6,6 +6,7 @@
 import com.github.dockerjava.api.exception.NotModifiedException;
 import com.github.dockerjava.api.model.Container;
 import java.time.Duration;
+import java.util.List;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import org.slf4j.Logger;
@@ -16,22 +17,31 @@ public class Demiurge {
 
     private static final Logger LOG = LoggerFactory.getLogger(Demiurge.class);
 
+    private static final Duration SERVICE_START_TIMEOUT = Duration.ofSeconds(20);
+
     @NonNull
     private DockerClient dockerClient;
 
-    public void starts(@NonNull Container container) {
+    public void starts(@NonNull List<Container> containers) {
         try {
-            dockerClient.startContainerCmd(container.getId()).exec();
-            await().atMost(Duration.ofSeconds(20)).until(() -> isStarted(container));
+            containers.forEach(container -> dockerClient.startContainerCmd(container.getId()).exec());
         } catch (NotModifiedException e) {
             LOG.debug("Already started");
         }
+        containers.forEach(container ->
+                await(container.getNames()[0])
+                        .atMost(SERVICE_START_TIMEOUT)
+                        .until(() -> isStarted(container)));
     }
 
     public void kills(@NonNull Container container) {
         dockerClient.killContainerCmd(container.getId()).exec();
     }
 
+    public void kills(@NonNull List<Container> containers) {
+        containers.forEach(this::kills);
+    }
+
     private boolean isStarted(@NonNull Container container) {
         var response = dockerClient.inspectContainerCmd(container.getId()).exec();
         if (response.getConfig().getHealthcheck() == null) {