OPSEXP-2875: add search enterprise mediation & reindex + fix tests

.github/workflows/cleanup_images.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
     inputs:
       dry-run:
-        description: 'Disable to really delete images'
+        description: Dry run (do not delete images)
         required: false
         type: boolean
         default: true
@@ -21,24 +21,19 @@ jobs:
   cleanup:
     runs-on: ubuntu-latest
     steps:
-      - name: Set package names
+      - name: Checkout repository
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          fetch-depth: 0
+      - name: Get package names from bake definition
+        env:
+          JQ_FILTER: >-
+            .target
+            | map(select(.output | index("type=docker")))
+            | map(.tags[] | split("/")[-1] | split(":")[0])
+            | join(",")
         run: |
-          echo "PACKAGE_NAMES=alfresco-transform-router,\
-          alfresco-shared-file-store,\
-          alfresco-content-repository,\
-          alfresco-imagemagick,\
-          alfresco-misc,\
-          alfresco-libreoffice,\
-          alfresco-pdf-renderer,\
-          alfresco-tika,\
-          alfresco-elasticsearch-live-indexing-content,\
-          alfresco-elasticsearch-live-indexing-path,\
-          alfresco-elasticsearch-live-indexing-metadata,\
-          alfresco-transform-core-aio,\
-          alfresco-elasticsearch-live-indexing,\
-          alfresco-share,\
-          alfresco-ooi-service,\
-          alfresco-ms-teams-service" >> $GITHUB_ENV
+          echo PACKAGE_NAMES=$(docker buildx bake --print | jq -r '${{ env.JQ_FILTER }}') >> $GITHUB_ENV
       - name: Remove tag ${{format('pr-{0}', github.event.pull_request.number)}} after PR is closed
         uses: dataaxiom/ghcr-cleanup-action@98b4022383d6ddb70ccbf6a378b4d8c67a60f066 # v1.0.13
         if: github.event_name == 'pull_request'

docker-bake.hcl

@@ -7,7 +7,7 @@ group "content_service" {
 }
 
 group "enterprise-search" {
-  targets = ["search_liveindexing"]
+  targets = ["search_liveindexing", "search_reindexing"]
 }
 
 group "ats" {
@@ -98,14 +98,6 @@ variable "ALFRESCO_GROUP_NAME" {
   default = "alfresco"
 }
 
-variable "ALFRESCO_REPO_USER_ID" {
-  default = "33000"
-}
-
-variable "ALFRESCO_REPO_USER_NAME" {
-  default = "alfresco"
-}
-
 target "java_base" {
   context = "./java"
   dockerfile = "Dockerfile"
@@ -185,6 +177,14 @@ target "tomcat_base" {
   output = ["type=cacheonly"]
 }
 
+variable "ALFRESCO_REPO_USER_ID" {
+  default = "33000"
+}
+
+variable "ALFRESCO_REPO_USER_NAME" {
+  default = "alfresco"
+}
+
 target "repository" {
   context = "./repository"
   dockerfile = "Dockerfile"
@@ -207,32 +207,53 @@ target "repository" {
   platforms = split(",", "${TARGETARCH}")
 }
 
+variable "ALFRESCO_LIVEINDEX_USER_ID" {
+  default = "33011"
+}
+
+variable "ALFRESCO_LIVEINDEX_USER_NAME" {
+  default = "liveindexer"
+}
+
 target "search_liveindexing" {
   matrix = {
     liveindexing = [
+      {
+        artifact = "alfresco-elasticsearch-live-indexing-mediation",
+        name = "mediation",
+        context = "common"
+      },
       {
         artifact = "alfresco-elasticsearch-live-indexing-metadata",
-        name = "metadata"
+        name = "metadata",
+        context = "common"
       },
       {
         artifact = "alfresco-elasticsearch-live-indexing-path",
-        name = "path"
+        name = "path",
+        context = "common"
       },
       {
         artifact = "alfresco-elasticsearch-live-indexing-content",
-        name = "content"
+        name = "content",
+        context = "common"
       },
       {
         artifact = "alfresco-elasticsearch-live-indexing",
-        name = "all-in-one"
+        name = "all-in-one",
+        context = "all-in-one"
       }
     ]
   }
   name = "${liveindexing.artifact}"
   args = {
     LIVEINDEXING = "${liveindexing.artifact}"
+    ALFRESCO_LIVEINDEX_GROUP_ID = "${ALFRESCO_GROUP_ID}"
+    ALFRESCO_LIVEINDEX_GROUP_NAME = "${ALFRESCO_GROUP_NAME}"
+    ALFRESCO_LIVEINDEX_USER_ID = "${ALFRESCO_LIVEINDEX_USER_ID}"
+    ALFRESCO_LIVEINDEX_USER_NAME = "${ALFRESCO_LIVEINDEX_USER_NAME}"
   }
-  context = "./search/enterprise/common"
+  context = "./search/enterprise/${liveindexing.context}"
   dockerfile = "Dockerfile"
   inherits = ["java_base"]
   contexts = {
@@ -247,6 +268,36 @@ target "search_liveindexing" {
   platforms = split(",", "${TARGETARCH}")
 }
 
+variable "ALFRESCO_REINDEX_USER_ID" {
+  default = "33011"
+}
+
+variable "ALFRESCO_REINDEX_USER_NAME" {
+  default = "liveindexer"
+}
+
+target "search_reindexing" {
+  context = "./search/enterprise/reindexing"
+  dockerfile = "Dockerfile"
+  inherits = ["java_base"]
+  contexts = {
+    java_base = "target:java_base"
+  }
+  args = {
+    ALFRESCO_REINDEX_GROUP_ID = "${ALFRESCO_GROUP_ID}"
+    ALFRESCO_REINDEX_GROUP_NAME = "${ALFRESCO_GROUP_NAME}"
+    ALFRESCO_REINDEX_USER_ID = "${ALFRESCO_REINDEX_USER_ID}"
+    ALFRESCO_REINDEX_USER_NAME = "${ALFRESCO_REINDEX_USER_NAME}"
+  }
+  labels = {
+    "org.opencontainers.image.title" = "${PRODUCT_LINE} Enterprise Search - reindexing"
+    "org.opencontainers.image.description" = "${PRODUCT_LINE} Enterprise Search - reindexing component"
+  }
+  tags = ["${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-elasticsearch-reindexing:${TAG}"]
+  output = ["type=docker"]
+  platforms = split(",", "${TARGETARCH}")
+}
+
 variable "ALFRESCO_TROUTER_USER_NAME" {
   default = "trouter"
 }

search/enterprise/all-in-one/Dockerfile

@@ -0,0 +1,24 @@
+FROM java_base
+
+ARG LIVEINDEXING
+ARG ALFRESCO_LIVEINDEX_USER_ID
+ARG ALFRESCO_LIVEINDEX_GROUP_ID
+ARG ALFRESCO_LIVEINDEX_GROUP_NAME
+ARG ALFRESCO_LIVEINDEX_USER_NAME
+
+ENV PORT=8080
+EXPOSE 8080
+
+COPY *-app.jar /opt/app.jar
+
+HEALTHCHECK --interval=30s --timeout=3s --retries=3 --start-period=1m \
+  CMD curl -f http://localhost:8080/actuator/health || exit 1
+
+COPY entrypoint.sh /entrypoint.sh
+
+RUN groupadd -g ${ALFRESCO_LIVEINDEX_GROUP_ID} ${ALFRESCO_LIVEINDEX_GROUP_NAME} && \
+    useradd -lu ${ALFRESCO_LIVEINDEX_USER_ID} -g ${ALFRESCO_LIVEINDEX_GROUP_NAME} ${ALFRESCO_LIVEINDEX_USER_NAME}
+
+USER ${ALFRESCO_LIVEINDEX_USER_NAME}
+
+CMD ["/entrypoint.sh"]

search/enterprise/all-in-one/artifacts.json

@@ -0,0 +1,14 @@
+{
+  "artifacts": {
+    "acs23": [
+      {
+        "name": "alfresco-elasticsearch-live-indexing",
+        "version": "4.0.1",
+        "path": "search/enterprise/all-in-one",
+        "classifier": "-app.jar",
+        "repository": "enterprise-releases",
+        "group": "org.alfresco"
+      }
+    ]
+  }
+}

search/enterprise/all-in-one/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec java $JAVA_OPTS -jar /opt/app.jar $@

search/enterprise/common/Dockerfile

@@ -1,13 +1,24 @@
 FROM java_base
 
 ARG LIVEINDEXING
+ARG ALFRESCO_LIVEINDEX_USER_ID
+ARG ALFRESCO_LIVEINDEX_GROUP_ID
+ARG ALFRESCO_LIVEINDEX_GROUP_NAME
+ARG ALFRESCO_LIVEINDEX_USER_NAME
+
 ENV PORT=8080
 EXPOSE 8080
 
-ADD ${LIVEINDEXING}*-app.jar /opt/app.jar
+COPY ${LIVEINDEXING}*-app.jar /opt/app.jar
 
 HEALTHCHECK --interval=30s --timeout=3s --retries=3 --start-period=1m \
   CMD curl -f http://localhost:8080/actuator/health || exit 1
 
 COPY entrypoint.sh /entrypoint.sh
+
+RUN groupadd -g ${ALFRESCO_LIVEINDEX_GROUP_ID} ${ALFRESCO_LIVEINDEX_GROUP_NAME} && \
+    useradd -lu ${ALFRESCO_LIVEINDEX_USER_ID} -g ${ALFRESCO_LIVEINDEX_GROUP_NAME} ${ALFRESCO_LIVEINDEX_USER_NAME}
+
+USER ${ALFRESCO_LIVEINDEX_USER_NAME}
+
 CMD ["/entrypoint.sh"]

search/enterprise/common/artifacts.json

@@ -1,6 +1,14 @@
 {
   "artifacts": {
     "acs23": [
+      {
+        "name": "alfresco-elasticsearch-live-indexing-mediation",
+        "version": "4.0.1",
+        "path": "search/enterprise/common",
+        "classifier": "-app.jar",
+        "repository": "enterprise-releases",
+        "group": "org.alfresco"
+      },
       {
         "name": "alfresco-elasticsearch-live-indexing-metadata",
         "version": "4.0.1",

search/enterprise/reindexing/Dockerfile

@@ -0,0 +1,16 @@
+FROM java_base
+
+ARG ALFRESCO_REINDEX_USER_ID
+ARG ALFRESCO_REINDEX_GROUP_ID
+ARG ALFRESCO_REINDEX_GROUP_NAME
+ARG ALFRESCO_REINDEX_USER_NAME
+
+COPY *-app.jar /opt/app.jar
+COPY entrypoint.sh /entrypoint.sh
+
+RUN groupadd -g ${ALFRESCO_REINDEX_GROUP_ID} ${ALFRESCO_REINDEX_GROUP_NAME} && \
+    useradd -lu ${ALFRESCO_REINDEX_USER_ID} -g ${ALFRESCO_REINDEX_GROUP_NAME} ${ALFRESCO_REINDEX_USER_NAME}
+
+USER ${ALFRESCO_REINDEX_USER_NAME}
+
+ENTRYPOINT ["/entrypoint.sh"]

search/enterprise/reindexing/artifacts.json

@@ -0,0 +1,14 @@
+{
+  "artifacts": {
+    "acs23": [
+      {
+        "name": "alfresco-elasticsearch-reindexing",
+        "version": "4.0.1",
+        "path": "search/enterprise/reindexing",
+        "classifier": "-app.jar",
+        "repository": "enterprise-releases",
+        "group": "org.alfresco"
+      }
+    ]
+  }
+}

search/enterprise/reindexing/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec java $JAVA_OPTS -jar /opt/app.jar $@

test/enterprise-docker-compose.yml

@@ -45,8 +45,8 @@ services:
         - http://localhost:8080/alfresco/api/-default-/public/alfresco/versions/1/probes/-ready-
       interval: 30s
       timeout: 3s
-      retries: 3
-      start_period: 1m
+      retries: 5
+      start_period: 2m
   transform-router:
     mem_limit: 512m
     image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-transform-router:${TAG}
@@ -176,11 +176,38 @@ services:
     ports:
       - "9090:9090"
   live-indexing:
-      image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-elasticsearch-live-indexing:${TAG}
-      environment:
-          SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200
-          SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616
-          ALFRESCO_SHAREDFILESTORE_BASEURL: http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file/
+    image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-elasticsearch-live-indexing:${TAG}
+    environment:
+      ALFRESCO_ACCEPTED_CONTENT_MEDIA_TYPES_CACHE_BASE_URL: http://transform-router:8095/transform/config
+      SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200
+      SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616
+      ALFRESCO_SHAREDFILESTORE_BASEURL: http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file/
+    depends_on:
+      activemq:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
+  reindexing:
+    image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-elasticsearch-reindexing:${TAG}
+    mem_limit: 1g
+    restart: on-failure:10
+    environment:
+      ALFRESCO_ACCEPTED_CONTENT_MEDIA_TYPES_CACHE_BASE_URL: >-
+        http://transform-core-aio:8090/transform/config
+      SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616
+      SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/alfresco
+      SPRING_DATASOURCE_USERNAME: alfresco
+      SPRING_DATASOURCE_PASSWORD: alfresco
+      SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200
+    depends_on:
+      activemq:
+        condition: service_started
+      postgres:
+        condition: service_started
+      elasticsearch:
+        condition: service_started
+      alfresco:
+        condition: service_healthy
 volumes:
   shared-file-store-volume:
     driver_opts:

test/helm/test-overrides.yaml

@@ -8,6 +8,10 @@ share:
     repository: localhost/alfresco-share
     tag: latest
 alfresco-search-enterprise:
+  reindexing:
+    image:
+      repository: localhost/alfresco-elasticsearch-reindexing
+      tag: latest
   liveIndexing:
     content:
       image:
@@ -21,6 +25,10 @@ alfresco-search-enterprise:
       image:
         repository: localhost/alfresco-elasticsearch-live-indexing-path
         tag: latest
+    mediation:
+      image:
+        repository: localhost/alfresco-elasticsearch-live-indexing-mediation
+        tag: latest
 alfresco-transform-service:
   pdfrenderer:
     image: