PS-696 stack db (#474)

* add configurator database

* fix default doctrine connection

* composer update

* configurator bundle

* add migrate to configurator

.dockerignore

@@ -3,6 +3,7 @@
 **/client/build
 **/var
 **/vendor
+**/public/bundles
 **/node_modules
 **/.idea
 **/.php-cs-fixer.php

.env

@@ -63,17 +63,18 @@ DISPLAY_SERVICES_MENU=true
 
 # Minio
 MINIO_CONSOLE_PORT=8111
+S3_USE_PATH_STYLE_ENDPOINT=true
 S3_ACCESS_KEY=__CHANGE_ME_LDb7v4BfKXE5vTnslDvNyeWMlMmi
 S3_SECRET_KEY=__CHANGE_ME_ZFA0x96WHJztksXrKUHYUA
 # Can be used to support a single CloudFront distribution with multiple origins
 S3_PATH_PREFIX=
+S3_REGION=eu-west-3
 
 # Auto redirect to identity provider when coming from a client app
 AUTO_CONNECT_IDP=
 
 # Databox
-DATABOX_STORAGE_USE_PATH_STYLE_ENDPOINT=true
-DATABOX_STORAGE_BUCKET_NAME=databox
+DATABOX_S3_BUCKET_NAME=databox
 DATABOX_REQUEST_SIGNATURE_TTL=3600
 DATABOX_UPLOADER_TARGET_SLUG=databox-uploader
 DATABOX_RABBITMQ_VHOST=databox
@@ -95,9 +96,7 @@ DATABOX_CLIENT_ID=databox-app
 UPLOADER_CLIENT_ID=uploader-app
 ASSET_CONSUMER_COMMIT_URI=http://localhost:9999/api/v1/upload/enqueue/
 ASSET_CONSUMER_ACCESS_TOKEN=define-me
-UPLOADER_STORAGE_BUCKET_NAME=uploader-deposit
-UPLOADER_STORAGE_ENDPOINT=http://minio:9000
-UPLOADER_STORAGE_USE_PATH_STYLE_ENDPOINT=true
+UPLOADER_S3_BUCKET_NAME=uploader-deposit
 UPLOADER_REQUEST_SIGNATURE_TTL=600
 UPLOADER_DELETE_ASSET_GRACEFUL_TIME=30
 UPLOADER_RABBITMQ_VHOST=uploader
@@ -122,9 +121,7 @@ KEYCLOAK_REALM_NAME="${KEYCLOAK_REALM_NAME:-phrasea}"
 # Expose
 # OAuth client ID for Expose service
 EXPOSE_CLIENT_ID=expose-app
-EXPOSE_STORAGE_ENDPOINT=http://minio:9000
-EXPOSE_STORAGE_BUCKET_NAME=expose
-EXPOSE_STORAGE_USE_PATH_STYLE_ENDPOINT=true
+EXPOSE_S3_BUCKET_NAME=expose
 EXPOSE_REQUEST_SIGNATURE_TTL=3600
 EXPOSE_SIDEBAR_DEFAULT_OPEN=true
 EXPOSE_CLIENT_LOGO_URL=
@@ -176,6 +173,10 @@ MARIADB_PORT=3306
 MAILER_DSN=smtp://mailhog:1025
 MAIL_FROM=noreply@${PHRASEA_DOMAIN}
 
+# Configurator
+CONFIGURATOR_DB_NAME=configurator
+CONFIGURATOR_S3_BUCKET_NAME=static
+
 # Mailhog
 MAILHOG_PORT=8125
 

.github/workflows/ci.yaml

@@ -36,10 +36,11 @@ jobs:
 
   build_configurator:
     name: 'Build Configurator'
-    uses: ./.github/workflows/build_base.yaml
+    uses: ./.github/workflows/build.yaml
     with:
       image: configurator
       context: ./configurator
+      withLibs: true
     secrets: inherit
     needs:
       - build_php-fpm-base
@@ -52,6 +53,14 @@ jobs:
       context: ./infra/docker/nginx-fpm-base
     secrets: inherit
 
+  build_nginx-client-base:
+    name: 'Build nginx-client-base'
+    uses: ./.github/workflows/build_base.yaml
+    with:
+      image: nginx-client-base
+      context: ./infra/docker/nginx-client-base
+    secrets: inherit
+
   build_nodejs-base:
     name: 'Build nodejs-base'
     uses: ./.github/workflows/build_base.yaml

bin/functions.sh

@@ -39,7 +39,7 @@ function load-env {
   rm -f "${tmp}"
 }
 
-# execute a shell commmand in a container defined in docker-compose.yml
+# execute a shell command in a container defined in docker-compose.yml
 function exec_container() {
   docker compose exec -T "$1" sh -c "$2"
 }
@@ -48,6 +48,10 @@ function exec_container_as() {
   docker compose exec -T "$1" su "$3" sh -c "$2"
 }
 
+function run_container_as() {
+  docker compose run --rm -T "$1" su "$3" sh -c "$2"
+}
+
 function create_db() {
   exec_container db "psql -U \"${POSTGRES_USER}\" -tc \"SELECT 1 FROM pg_database WHERE datname = '$1'\" | grep -q 1 || psql -U \"${POSTGRES_USER}\" -c \"CREATE DATABASE $1\""
 }

bin/ops/configurator-configure.sh

@@ -23,7 +23,7 @@ NS="${1}"
 RELEASE_NAME=phrasea
 CHART_VERSION="${2}"
 
-echo "Running configuration:configure..."
+echo "Running bin/console configure..."
 
 (
   mkdir -p /tmp/phrasea-helm-configure \

bin/optimize-composer-docker-cache

@@ -62,7 +62,7 @@ function updateProject(string $project)
             'export COMPOSER_MEMORY_LIMIT=-1',
             '(cd %s',
             'rm -rf composer.lock vendor',
-            'composer update --no-interaction --no-scripts --ansi --no-plugins',
+            'composer update --no-install --no-interaction --no-scripts --ansi --no-plugins',
             'rm -rf vendor)',
             '(cd %s && composer update --no-interaction --ansi)',
         ]), dirname($composerCacheSrc), $project));
@@ -74,22 +74,26 @@ function updateProject(string $project)
 function flattenSubRequire(string $workingDir, array $composer): array {
     $repositories = $composer['repositories'] ?? null;
     if ($repositories) {
+        $localPackages = [];
         foreach ($repositories as $repository) {
             if ('path' === $repository['type']) {
                 $url = $repository['url'];
                 $path = $workingDir.'/'.preg_replace('#^\./#', '', $url);
 
                 $subComposer = json_decode(file_get_contents($path.'/composer.json'), true, 512, JSON_THROW_ON_ERROR);
 
-                unset($composer['require'][$subComposer['name']]);
-
+                $localPackages[$subComposer['name']] = true;
                 foreach ($subComposer['require'] as $lib => $version) {
                     if (str_contains($lib, '/')) {
-                        $composer['require'][$lib] = $version;
+                        $composer['require'][$lib] ??= $version;
                     }
                 }
             }
         }
+
+        foreach (array_keys($localPackages) as $localPackage) {
+            unset($composer['require'][$localPackage]);
+        }
     }
 
     return $composer;

bin/setup.sh

@@ -15,6 +15,21 @@ docker compose up -d
 # Wait for services to be ready
 docker compose run --rm dockerize
 
+# Setup Report
+## Create DB
+create_db "${REPORT_DB_NAME}"
+create_db "${KEYCLOAK_DB_NAME}"
+create_db "${KEYCLOAK2_DB_NAME}"
+
+run_container_as configurator "bin/setup.sh" app
+## Create minio bucket
+COMPOSE_PROFILES="${COMPOSE_PROFILES},setup" docker compose run --rm -T --entrypoint "sh -c" minio-mc "\
+  while ! nc -z minio 9000; do echo 'Wait minio to startup...' && sleep 0.1; done; \
+  sleep 5 && \
+  mc config host add minio http://minio:9000 \$MINIO_ACCESS_KEY \$MINIO_SECRET_KEY && \
+  mc mb --ignore-existing minio/$CONFIGURATOR_S3_BUCKET_NAME \
+"
+
 # Setup Uploader
 ## Create rabbitmq vhost
 exec_container rabbitmq "rabbitmqctl add_vhost ${UPLOADER_RABBITMQ_VHOST} && rabbitmqctl set_permissions -p ${UPLOADER_RABBITMQ_VHOST} ${RABBITMQ_USER} '.*' '.*' '.*'"
@@ -25,7 +40,7 @@ COMPOSE_PROFILES="${COMPOSE_PROFILES},setup" docker compose run --rm -T --entryp
   while ! nc -z minio 9000; do echo 'Wait minio to startup...' && sleep 0.1; done; \
   sleep 5 && \
   mc config host add minio http://minio:9000 \$MINIO_ACCESS_KEY \$MINIO_SECRET_KEY && \
-  mc mb --ignore-existing minio/$UPLOADER_STORAGE_BUCKET_NAME \
+  mc mb --ignore-existing minio/$UPLOADER_S3_BUCKET_NAME \
 "
 
 # Setup Expose
@@ -48,7 +63,7 @@ COMPOSE_PROFILES="${COMPOSE_PROFILES},setup" docker compose run --rm -T --entryp
   done; \
   sleep 3 \
   && mc config host add minio http://minio:9000 \$MINIO_ACCESS_KEY \$MINIO_SECRET_KEY \
-  && mc mb --ignore-existing minio/$EXPOSE_STORAGE_BUCKET_NAME \
+  && mc mb --ignore-existing minio/$EXPOSE_S3_BUCKET_NAME \
 "
 
 # Setup Notify
@@ -67,18 +82,11 @@ COMPOSE_PROFILES="${COMPOSE_PROFILES},setup" docker compose run --rm -T --entryp
   while ! nc -z minio 9000; do echo 'Wait minio to startup...' && sleep 0.1; done; \
   sleep 5 && \
   mc config host add minio http://minio:9000 \$MINIO_ACCESS_KEY \$MINIO_SECRET_KEY && \
-  mc mb --ignore-existing minio/\$DATABOX_STORAGE_BUCKET_NAME \
+  mc mb --ignore-existing minio/$DATABOX_S3_BUCKET_NAME \
 "
 ## Create Uploader target for client upload
 exec_container uploader-api-php "bin/console app:create-target ${DATABOX_UPLOADER_TARGET_SLUG} 'Databox Uploader' http://databox-api/incoming-uploads"
 
-# Setup Report
-## Create DB
-create_db "${REPORT_DB_NAME}"
-
-create_db "${KEYCLOAK_DB_NAME}"
-create_db "${KEYCLOAK2_DB_NAME}"
-
 ## Setup indexer
 ## Create Databox OAuth client for indexer
 COMPOSE_PROFILES="${COMPOSE_PROFILES},setup" docker compose run --rm -T --entrypoint "sh -c" minio-mc "\
@@ -113,6 +121,6 @@ PRESETS=""
 for p in $@; do
   PRESETS="${PRESETS} --preset $p"
 done
-docker compose run --rm configurator configure -vvv$PRESETS
+docker compose run --rm configurator bin/console configure -vvv$PRESETS
 
 echo "Done."

bin/vars.sh

@@ -21,18 +21,19 @@ databox/indexer
 PHP_LIBS="
 lib/php/admin-bundle
 lib/php/api-test
+lib/php/auth-bundle
+lib/php/configurator-bundle
 lib/php/core-bundle
 lib/php/metadata-manipulator-bundle
 lib/php/notify-bundle
-lib/php/auth-bundle
+lib/php/rendition-factory
+lib/php/rendition-factory-bundle
 lib/php/report-bundle
 lib/php/report-sdk
 lib/php/test-bundle
 lib/php/webhook-bundle
 lib/php/workflow
 lib/php/workflow-bundle
-lib/php/rendition-factory
-lib/php/rendition-factory-bundle
 "
 
 JS_LIBS="

configs/config.dist.json

@@ -1,10 +1,4 @@
 {
-  "available_locales": [
-    "en",
-    "fr",
-    "es",
-    "fr_CA"
-  ],
   "uploader": {
     "max_upload_file_size": null,
     "max_upload_commit_size": null,

configurator/.env

@@ -14,6 +14,8 @@
 # Run "composer dump-env prod" to compile .env files for production use (requires symfony/flex >=1.2).
 # https://symfony.com/doc/current/best_practices.html#use-environment-variables-for-infrastructure-configuration
 
+APP_ID=configurator
+
 ###> symfony/framework-bundle ###
 APP_ENV=dev
 APP_SECRET=fe5922c9df2f78b897cbf2bf988c7204

configurator/Dockerfile

@@ -2,16 +2,19 @@ ARG BASE_TAG=latest
 ARG REGISTRY_NAMESPACE
 FROM ${REGISTRY_NAMESPACE}php-fpm-base:${BASE_TAG} AS api-php
 
-WORKDIR /usr/app
+WORKDIR /srv/app
 
-ENV APP_ENV=prod
+COPY --chown=app:app ./configurator .
+COPY --chown=app:app lib/php /srv/lib/php
 
-COPY . .
-
-RUN composer install --prefer-dist --no-dev --no-progress --classmap-authoritative --no-interaction \
+RUN mkdir -p var/cache var/logs var/sessions \
+    && composer install --prefer-dist --no-dev --no-progress --classmap-authoritative --no-interaction \
     && composer clear-cache \
     && chown -R app: .
 
-ENTRYPOINT ["bin/console"]
+ARG SENTRY_RELEASE
+ENV SENTRY_RELEASE=${SENTRY_RELEASE}
+
+ENTRYPOINT ["/srv/app/docker/php-entrypoint.sh"]
 
-CMD ["configure"]
+CMD ["bin/console", "configure"]

configurator/bin/migrate.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+BASEDIR=$(dirname $0)
+
+"${BASEDIR}/console" doctrine:migrations:sync-metadata-storage
+"${BASEDIR}/console" doctrine:migrations:migrate --no-interaction

configurator/bin/setup.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+BASEDIR=$(dirname $0)
+
+if [ ! -d "${BASEDIR}/../vendor" ]; then
+  (cd "${BASEDIR}/.." && composer install)
+fi
+
+"${BASEDIR}/console" doctrine:database:create --if-not-exists
+"${BASEDIR}/console" doctrine:migrations:sync-metadata-storage
+"${BASEDIR}/console" doctrine:migrations:migrate --no-interaction

configurator/composer.json

@@ -1,13 +1,34 @@
 {
     "type": "project",
     "license": "proprietary",
-    "minimum-stability": "stable",
+    "minimum-stability": "dev",
     "prefer-stable": true,
+    "repositories": [
+        {
+            "type": "path",
+            "url": "../lib/php/configurator-bundle",
+            "options": {
+                "symlink": true
+            }
+        },
+        {
+            "type": "path",
+            "url": "../lib/php/core-bundle",
+            "options": {
+                "symlink": true
+            }
+        }
+    ],
     "require": {
         "php": ">=8.3",
         "ext-ctype": "*",
         "ext-iconv": "*",
-        "doctrine/doctrine-bundle": "*",
+        "alchemy/configurator-bundle": "@dev",
+        "doctrine/doctrine-bundle": "^2.12",
+        "doctrine/doctrine-migrations-bundle": "^3.3",
+        "doctrine/orm": "^3.3",
+        "gedmo/doctrine-extensions": "^3.17",
+        "stof/doctrine-extensions-bundle": "^1.12",
         "symfony/console": "6.3.*",
         "symfony/dotenv": "6.3.*",
         "symfony/flex": "^2",