PHRAS-4010 Add mysql8 as alternative datastore (#4477)

* PHRAS-4010 : add profile and alternative compose file * PHRAS-4010 : add entrypoint * PHRAS-4010 : update .env * fix compatibility base_structures.xml -> mysql8 for sys:up * fix timestamp DEFAULT_GENERATED * fix mysql 8 version * Update .env
alchemy-fr · Feb 23, 2024 · 0feb7e9 · 0feb7e9
1 parent 3bf99cb
commit 0feb7e9
Show file tree

Hide file tree

Showing 3 changed files with 98 additions and 19 deletions.
diff --git a/.env b/.env
@@ -32,6 +32,7 @@
 #
 # - "docker-compose.limits.yml"     : defines containers cpu and memory limits for all Phraseanet and gateway containers only.
 #
+# - "docker-compose.altenatives.yml": all alternative services, used only on evoluation or transition periods 
 #
 # 2/ "COMPOSE_PROFILES" value define which profiles you want to use
 # in docker-compose.
@@ -48,8 +49,8 @@
 #                       choose to launch only some workers, see worker profile list below.
 # - "worker"          : launch one container worker with all jobs run on it.
 # - "cmd"             : launch a container based on worker image, useful for run cmd manualy. 
-# - "db"              : launch a mariadb container, because this is the primary
-#                       datastore for production usage, use your own service.
+# - "db"              : db profile will launch a mariadb container, 
+#                       because this is the primary datastore, you should use you own SGDD service for production needs.
 # - "elastisearch"    : launch a elasticsearch container.
 # - "rabbitmq"        : launch a rabbitmq container.
 # - "redis"           : launch a redis container for app cache.
@@ -59,6 +60,8 @@
 # - "squid"           : reverse proxy for dev only.
 # - "mailhog"         : for catching all email emit by app for dev.
 # - "db-backup"       : launch and run a container to cron database backups and backup file's rotation.
+# - "mysql8"          : launch a mysql8 container (beta), (/!\ do not mix with the "db" profile)
+#                       Because this is the primary datastore, you should use you own SGDD service for production needs.
 #
 # Profiles worker list:
 # - "assetsInjest"

diff --git a/docker-compose.altenatives.yml b/docker-compose.altenatives.yml
@@ -0,0 +1,30 @@
+version: "3.9"
+
+services:
+
+  db:
+    image: mysql:8.0.36-debian
+    # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password
+    # (this is just an example, not intended to be a production configuration)
+    command: --default-authentication-plugin=mysql_native_password --max_allowed_packet=$MYSQL_MAX_ALLOWED_PACKET --max_connections=$MYSQL_MAX_CONNECTION --long_query_time=$MYSQL_LONG_QUERY_TIME --sql_mode="NO_ENGINE_SUBSTITUTION"
+    restart: on-failure
+    profiles: ["mysql8"]
+    entrypoint:
+      sh -c "
+        echo 'CREATE DATABASE IF NOT EXISTS ab_master CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_databox1 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_unitTest CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_dataset1 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_dataset2 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE USER `$PHRASEANET_DB_USER`@`%` IDENTIFIED BY \"$PHRASEANET_DB_PASSWORD\";GRANT ALL PRIVILEGES ON *.* to `$PHRASEANET_DB_USER`@`%`;' > /docker-entrypoint-initdb.d/init.sql;
+        chmod +x /usr/local/bin/docker-entrypoint.sh /docker-entrypoint-initdb.d/init.sql;
+        /usr/local/bin/docker-entrypoint.sh --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --default-authentication-plugin=mysql_native_password --max_allowed_packet=$MYSQL_MAX_ALLOWED_PACKET --max_connections=$MYSQL_MAX_CONNECTION --long_query_time=$MYSQL_LONG_QUERY_TIME --sql_mode="NO_ENGINE_SUBSTITUTION" --slow_query_log=$MYSQL_SLOW_QUERY_LOG
+      "
+    environment:
+    - MYSQL_ROOT_PASSWORD
+    - MYSQL_MAX_ALLOWED_PACKET
+    - MYSQL_MAX_CONNECTION
+    - MYSQL_LONG_QUERY_TIME
+    - MYSQL_SLOW_QUERY_LOG
+    - MYSQL_QUERY_CACHE_LIMIT
+    - MYSQL_QUERY_CACHE_SIZE
+    - MYSQL_KEY_BUFFER_SIZE
+    volumes:
+    - ${PHRASEANET_DB_DIR}_mysql8:/var/lib/mysql:rw
+    networks:
+      - internal
diff --git a/lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php b/lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php
@@ -93,7 +93,7 @@ public function upgradeDatabase(\base $base, $applyPatches, InputInterface $inpu
                     $this->alterTableEngine($tableName, $engine, $recommends);
                 }
 
-                $ret = $this->upgradeTable($allTables[$tableName]);
+                $ret = $this->upgradeTable($allTables[$tableName], $output);
                 $recommends = array_merge($recommends, $ret);
 
                 unset($allTables[$tableName]);
@@ -285,7 +285,7 @@ public function createTable(\SimpleXMLElement $table)
         unset($stmt);
     }
 
-    public function upgradeTable(\SimpleXMLElement $table)
+    public function upgradeTable(\SimpleXMLElement $table, OutputInterface $output)
     {
         $this->reconnect();
 
@@ -296,15 +296,18 @@ public function upgradeTable(\SimpleXMLElement $table)
             $expr = trim((string)$field->type);
 
             $_extra = trim((string)$field->extra);
+            $type = trim(strtolower((string)$field->type));
+            $_default = (string)$field->default;
+
             if ($_extra) {
                 $expr .= ' ' . $_extra;
             }
 
             $collation = trim((string)$field->collation) != '' ? trim((string)$field->collation) : 'utf8_unicode_ci';
 
-            if (in_array(strtolower((string)$field->type), ['text', 'longtext', 'mediumtext', 'tinytext'])
-                || substr(strtolower((string)$field->type), 0, 7) == 'varchar'
-                || in_array(substr(strtolower((string)$field->type), 0, 4), ['char', 'enum'])
+            if (in_array($type, ['text', 'longtext', 'mediumtext', 'tinytext'])
+                || substr($type, 0, 7) == 'varchar'
+                || in_array(substr($type, 0, 4), ['char', 'enum'])
             ) {
                 $collations = array_reverse(explode('_', $collation));
                 $code = array_pop($collations);
@@ -321,14 +324,17 @@ public function upgradeTable(\SimpleXMLElement $table)
                 $expr .= ' NOT NULL';
             }
 
-            $_default = (string)$field->default;
             if ($_default && $_default != 'CURRENT_TIMESTAMP') {
                 $expr .= ' DEFAULT \'' . $_default . '\'';
             } elseif ($_default == 'CURRENT_TIMESTAMP') {
                 $expr .= ' DEFAULT ' . $_default . '';
             }
 
-            $correct_table['fields'][trim((string)$field->name)] = $expr;
+            $expr8 = preg_replace('/^(\\w*int)(\\(\\d+\\))?(.*)?$/', '$1$3', $expr);
+            $correct_table['fields'][trim((string)$field->name)] = [
+                'expr' => $expr,
+                'expr8' => $expr8
+            ];
         }
         if ($table->indexes) {
             foreach ($table->indexes->index as $index) {
@@ -359,6 +365,36 @@ public function upgradeTable(\SimpleXMLElement $table)
 
         foreach ($rs2 as $row2) {
             $f_name = $row2['Field'];
+
+            // accept alias collations as same as in lib/conf.d/bases_structure.xml (utf8_unicode_ci)
+            if(in_array(strtolower($row2['Collation']), ['utf8mb3_unicode_ci', 'utf8mb4_unicode_ci'])) {
+                $row2['Collation'] = 'utf8_unicode_ci';
+            }
+
+            // accept current_timestamp() as result of CURRENT_TIMESTAMP
+            if(strtolower($row2['Default']) === 'current_timestamp()') {
+                $row2['Default'] = "CURRENT_TIMESTAMP";
+            }
+
+            // match integers (https://dev.mysql.com/worklog/task/?id=13127)
+            if(isset($correct_table['fields'][$f_name])) {
+                $matches = [];
+                if(preg_match("/^(\\w*int)(\\(\d+\\))?(.*)?$/", $row2['Type'], $matches) === 1) {
+                    $matches = array_merge($matches, ['', '', '', '']); // set missing matches (easier to test)
+                    if($matches[2] === '') {
+                        // mysql 8 : we must use the expr8
+                        $correct_table['fields'][$f_name] = $correct_table['fields'][$f_name]['expr8'];
+                    }
+                    else {
+                        // mysql 5
+                        $correct_table['fields'][$f_name] = $correct_table['fields'][$f_name]['expr'];
+                    }
+                }
+                else {
+                    $correct_table['fields'][$f_name] = $correct_table['fields'][$f_name]['expr'];
+                }
+            }
+
             $expr_found = trim($row2['Type']);
 
             $_extra = $row2['Extra'];
@@ -395,6 +431,12 @@ public function upgradeTable(\SimpleXMLElement $table)
             }
 
             if (isset($correct_table['fields'][$f_name])) {
+
+                $matches = [];
+                if(preg_match("/^timestamp DEFAULT_GENERATED/", $expr_found, $matches) === 1) {
+                    $correct_table['fields'][$f_name] = preg_replace("/^timestamp/", "timestamp DEFAULT_GENERATED", $correct_table['fields'][$f_name]);
+                }
+
                 if (isset($correct_table['collation'][$f_name]) && $correct_table['collation'][$f_name] != $current_collation) {
                     $old_type = mb_strtolower(trim($row2['Type']));
                     $new_type = false;
@@ -427,11 +469,15 @@ public function upgradeTable(\SimpleXMLElement $table)
                     }
                 }
 
-                if (strtolower($expr_found) !== strtolower($correct_table['fields'][$f_name])) {
-                    $alter[] = "ALTER TABLE `" . $table['name'] . "` CHANGE `$f_name` `$f_name` " . $correct_table['fields'][$f_name];
+                $expected = $correct_table['fields'][$f_name];
+                if (strtolower($expr_found) !== strtolower($expected)) {
+                    $output->writeln(sprintf("expected: <info>%s</info>", $expected));
+                    $output->writeln(sprintf("got     : <info>%s</info>", $expr_found));
+                    $alter[] = "ALTER TABLE `" . $table['name'] . "` CHANGE `$f_name` `$f_name` " . $expected;
                 }
                 unset($correct_table['fields'][$f_name]);
-            } else {
+            }
+            else {
                 $return[] = [
                     'message' => 'Un champ pourrait etre supprime',
                     'sql' => "ALTER TABLE " . $this->connection->getDatabase() . ".`" . $table['name'] . "` DROP `$f_name`;"
@@ -478,6 +524,8 @@ public function upgradeTable(\SimpleXMLElement $table)
             if (isset($correct_table['indexes'][$kIndex])) {
 
                 if (mb_strtolower($expr_found) !== mb_strtolower($correct_table['indexes'][$kIndex])) {
+                    $output->writeln(sprintf("expected: <info>%s</info>", $correct_table['indexes'][$kIndex]));
+                    $output->writeln(sprintf("got     : <info>%s</info>", $expr_found));
                     $alter[] = 'ALTER TABLE `' . $table['name'] . '` DROP ' . $full_name_index . ', ADD ' . $correct_table['indexes'][$kIndex];
                 }
 
@@ -498,6 +546,7 @@ public function upgradeTable(\SimpleXMLElement $table)
             $this->reconnect();
 
             try {
+                $output->writeln(sprintf("%s \n", $a));
                 $this->connection->exec($a);
             } catch (\Exception $e) {
                 $return[] = [
@@ -512,6 +561,7 @@ public function upgradeTable(\SimpleXMLElement $table)
             $this->reconnect();
 
             try {
+                $output->writeln(sprintf("%s \n", $a));
                 $this->connection->exec($a);
             } catch (\Exception $e) {
                 $return[] = [
@@ -541,7 +591,7 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf
 
         foreach ($iterator as $fileinfo) {
             if (!$fileinfo->isDot()) {
-// printf("---- [%d]\n", __LINE__);
+
                 if (substr($fileinfo->getFilename(), 0, 1) == '.') {
                     continue;
                 }
@@ -551,7 +601,6 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf
                 /** @var \patchAbstract $patch */
                 $patch = new $classname();
 
-// printf("---- [%d]\n", __LINE__);
                 if (!in_array($base->get_base_type(), $patch->concern())) {
                     continue;
                 }
@@ -560,18 +609,15 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf
                     continue;
                 }
 
-// printf("---- [%d] %s ; from: %s ; patch: %s; to:%s\n", __LINE__, $classname, $from, $patch->get_release(), $to);
-// printf("---- [%d]\n", __LINE__);
                 // if patch is older than current install
                 if (version::lte($patch->get_release(), $from)) {
                     continue;
                 }
-// printf("---- [%d]\n", __LINE__);
+
                 // if patch is new than current target
                 if (version::gt($patch->get_release(), $to)) {
                     continue;
                 }
-// printf("---- [%d]\n", __LINE__);
 
                 $n = 0;
                 do {
@@ -591,7 +637,7 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf
 
         // disable mail
         $this->app['swiftmailer.transport'] = null;
-// var_dump($list_patches);
+
         foreach ($list_patches as $patch) {
 
             $output->writeln(sprintf(" - patch \"%s\" (release %s) should be applied", get_class($patch), $patch->get_release()));