From 0feb7e9f30b840b550b287e34e898f1fb9aa8645 Mon Sep 17 00:00:00 2001 From: moctardiouf Date: Fri, 23 Feb 2024 13:16:41 +0100 Subject: [PATCH] PHRAS-4010 Add mysql8 as alternative datastore (#4477) * PHRAS-4010 : add profile and alternative compose file * PHRAS-4010 : add entrypoint * PHRAS-4010 : update .env * fix compatibility base_structures.xml -> mysql8 for sys:up * fix timestamp DEFAULT_GENERATED * fix mysql 8 version * Update .env --- .env | 7 +- docker-compose.altenatives.yml | 30 +++++++ .../Database/DatabaseMaintenanceService.php | 80 +++++++++++++++---- 3 files changed, 98 insertions(+), 19 deletions(-) create mode 100644 docker-compose.altenatives.yml diff --git a/.env b/.env index 70ebba3114..6fd77d394a 100644 --- a/.env +++ b/.env @@ -32,6 +32,7 @@ # # - "docker-compose.limits.yml" : defines containers cpu and memory limits for all Phraseanet and gateway containers only. # +# - "docker-compose.altenatives.yml": all alternative services, used only on evoluation or transition periods # # 2/ "COMPOSE_PROFILES" value define which profiles you want to use # in docker-compose. @@ -48,8 +49,8 @@ # choose to launch only some workers, see worker profile list below. # - "worker" : launch one container worker with all jobs run on it. # - "cmd" : launch a container based on worker image, useful for run cmd manualy. -# - "db" : launch a mariadb container, because this is the primary -# datastore for production usage, use your own service. +# - "db" : db profile will launch a mariadb container, +# because this is the primary datastore, you should use you own SGDD service for production needs. # - "elastisearch" : launch a elasticsearch container. # - "rabbitmq" : launch a rabbitmq container. # - "redis" : launch a redis container for app cache. @@ -59,6 +60,8 @@ # - "squid" : reverse proxy for dev only. # - "mailhog" : for catching all email emit by app for dev. # - "db-backup" : launch and run a container to cron database backups and backup file's rotation. +# - "mysql8" : launch a mysql8 container (beta), (/!\ do not mix with the "db" profile) +# Because this is the primary datastore, you should use you own SGDD service for production needs. # # Profiles worker list: # - "assetsInjest" diff --git a/docker-compose.altenatives.yml b/docker-compose.altenatives.yml new file mode 100644 index 0000000000..d709a01829 --- /dev/null +++ b/docker-compose.altenatives.yml @@ -0,0 +1,30 @@ +version: "3.9" + +services: + + db: + image: mysql:8.0.36-debian + # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password + # (this is just an example, not intended to be a production configuration) + command: --default-authentication-plugin=mysql_native_password --max_allowed_packet=$MYSQL_MAX_ALLOWED_PACKET --max_connections=$MYSQL_MAX_CONNECTION --long_query_time=$MYSQL_LONG_QUERY_TIME --sql_mode="NO_ENGINE_SUBSTITUTION" + restart: on-failure + profiles: ["mysql8"] + entrypoint: + sh -c " + echo 'CREATE DATABASE IF NOT EXISTS ab_master CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_databox1 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_unitTest CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_dataset1 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;CREATE DATABASE IF NOT EXISTS db_dataset2 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE USER `$PHRASEANET_DB_USER`@`%` IDENTIFIED BY \"$PHRASEANET_DB_PASSWORD\";GRANT ALL PRIVILEGES ON *.* to `$PHRASEANET_DB_USER`@`%`;' > /docker-entrypoint-initdb.d/init.sql; + chmod +x /usr/local/bin/docker-entrypoint.sh /docker-entrypoint-initdb.d/init.sql; + /usr/local/bin/docker-entrypoint.sh --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --default-authentication-plugin=mysql_native_password --max_allowed_packet=$MYSQL_MAX_ALLOWED_PACKET --max_connections=$MYSQL_MAX_CONNECTION --long_query_time=$MYSQL_LONG_QUERY_TIME --sql_mode="NO_ENGINE_SUBSTITUTION" --slow_query_log=$MYSQL_SLOW_QUERY_LOG + " + environment: + - MYSQL_ROOT_PASSWORD + - MYSQL_MAX_ALLOWED_PACKET + - MYSQL_MAX_CONNECTION + - MYSQL_LONG_QUERY_TIME + - MYSQL_SLOW_QUERY_LOG + - MYSQL_QUERY_CACHE_LIMIT + - MYSQL_QUERY_CACHE_SIZE + - MYSQL_KEY_BUFFER_SIZE + volumes: + - ${PHRASEANET_DB_DIR}_mysql8:/var/lib/mysql:rw + networks: + - internal diff --git a/lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php b/lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php index ee9429500f..28fe32b5f1 100644 --- a/lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php +++ b/lib/Alchemy/Phrasea/Core/Database/DatabaseMaintenanceService.php @@ -93,7 +93,7 @@ public function upgradeDatabase(\base $base, $applyPatches, InputInterface $inpu $this->alterTableEngine($tableName, $engine, $recommends); } - $ret = $this->upgradeTable($allTables[$tableName]); + $ret = $this->upgradeTable($allTables[$tableName], $output); $recommends = array_merge($recommends, $ret); unset($allTables[$tableName]); @@ -285,7 +285,7 @@ public function createTable(\SimpleXMLElement $table) unset($stmt); } - public function upgradeTable(\SimpleXMLElement $table) + public function upgradeTable(\SimpleXMLElement $table, OutputInterface $output) { $this->reconnect(); @@ -296,15 +296,18 @@ public function upgradeTable(\SimpleXMLElement $table) $expr = trim((string)$field->type); $_extra = trim((string)$field->extra); + $type = trim(strtolower((string)$field->type)); + $_default = (string)$field->default; + if ($_extra) { $expr .= ' ' . $_extra; } $collation = trim((string)$field->collation) != '' ? trim((string)$field->collation) : 'utf8_unicode_ci'; - if (in_array(strtolower((string)$field->type), ['text', 'longtext', 'mediumtext', 'tinytext']) - || substr(strtolower((string)$field->type), 0, 7) == 'varchar' - || in_array(substr(strtolower((string)$field->type), 0, 4), ['char', 'enum']) + if (in_array($type, ['text', 'longtext', 'mediumtext', 'tinytext']) + || substr($type, 0, 7) == 'varchar' + || in_array(substr($type, 0, 4), ['char', 'enum']) ) { $collations = array_reverse(explode('_', $collation)); $code = array_pop($collations); @@ -321,14 +324,17 @@ public function upgradeTable(\SimpleXMLElement $table) $expr .= ' NOT NULL'; } - $_default = (string)$field->default; if ($_default && $_default != 'CURRENT_TIMESTAMP') { $expr .= ' DEFAULT \'' . $_default . '\''; } elseif ($_default == 'CURRENT_TIMESTAMP') { $expr .= ' DEFAULT ' . $_default . ''; } - $correct_table['fields'][trim((string)$field->name)] = $expr; + $expr8 = preg_replace('/^(\\w*int)(\\(\\d+\\))?(.*)?$/', '$1$3', $expr); + $correct_table['fields'][trim((string)$field->name)] = [ + 'expr' => $expr, + 'expr8' => $expr8 + ]; } if ($table->indexes) { foreach ($table->indexes->index as $index) { @@ -359,6 +365,36 @@ public function upgradeTable(\SimpleXMLElement $table) foreach ($rs2 as $row2) { $f_name = $row2['Field']; + + // accept alias collations as same as in lib/conf.d/bases_structure.xml (utf8_unicode_ci) + if(in_array(strtolower($row2['Collation']), ['utf8mb3_unicode_ci', 'utf8mb4_unicode_ci'])) { + $row2['Collation'] = 'utf8_unicode_ci'; + } + + // accept current_timestamp() as result of CURRENT_TIMESTAMP + if(strtolower($row2['Default']) === 'current_timestamp()') { + $row2['Default'] = "CURRENT_TIMESTAMP"; + } + + // match integers (https://dev.mysql.com/worklog/task/?id=13127) + if(isset($correct_table['fields'][$f_name])) { + $matches = []; + if(preg_match("/^(\\w*int)(\\(\d+\\))?(.*)?$/", $row2['Type'], $matches) === 1) { + $matches = array_merge($matches, ['', '', '', '']); // set missing matches (easier to test) + if($matches[2] === '') { + // mysql 8 : we must use the expr8 + $correct_table['fields'][$f_name] = $correct_table['fields'][$f_name]['expr8']; + } + else { + // mysql 5 + $correct_table['fields'][$f_name] = $correct_table['fields'][$f_name]['expr']; + } + } + else { + $correct_table['fields'][$f_name] = $correct_table['fields'][$f_name]['expr']; + } + } + $expr_found = trim($row2['Type']); $_extra = $row2['Extra']; @@ -395,6 +431,12 @@ public function upgradeTable(\SimpleXMLElement $table) } if (isset($correct_table['fields'][$f_name])) { + + $matches = []; + if(preg_match("/^timestamp DEFAULT_GENERATED/", $expr_found, $matches) === 1) { + $correct_table['fields'][$f_name] = preg_replace("/^timestamp/", "timestamp DEFAULT_GENERATED", $correct_table['fields'][$f_name]); + } + if (isset($correct_table['collation'][$f_name]) && $correct_table['collation'][$f_name] != $current_collation) { $old_type = mb_strtolower(trim($row2['Type'])); $new_type = false; @@ -427,11 +469,15 @@ public function upgradeTable(\SimpleXMLElement $table) } } - if (strtolower($expr_found) !== strtolower($correct_table['fields'][$f_name])) { - $alter[] = "ALTER TABLE `" . $table['name'] . "` CHANGE `$f_name` `$f_name` " . $correct_table['fields'][$f_name]; + $expected = $correct_table['fields'][$f_name]; + if (strtolower($expr_found) !== strtolower($expected)) { + $output->writeln(sprintf("expected: %s", $expected)); + $output->writeln(sprintf("got : %s", $expr_found)); + $alter[] = "ALTER TABLE `" . $table['name'] . "` CHANGE `$f_name` `$f_name` " . $expected; } unset($correct_table['fields'][$f_name]); - } else { + } + else { $return[] = [ 'message' => 'Un champ pourrait etre supprime', 'sql' => "ALTER TABLE " . $this->connection->getDatabase() . ".`" . $table['name'] . "` DROP `$f_name`;" @@ -478,6 +524,8 @@ public function upgradeTable(\SimpleXMLElement $table) if (isset($correct_table['indexes'][$kIndex])) { if (mb_strtolower($expr_found) !== mb_strtolower($correct_table['indexes'][$kIndex])) { + $output->writeln(sprintf("expected: %s", $correct_table['indexes'][$kIndex])); + $output->writeln(sprintf("got : %s", $expr_found)); $alter[] = 'ALTER TABLE `' . $table['name'] . '` DROP ' . $full_name_index . ', ADD ' . $correct_table['indexes'][$kIndex]; } @@ -498,6 +546,7 @@ public function upgradeTable(\SimpleXMLElement $table) $this->reconnect(); try { + $output->writeln(sprintf("%s \n", $a)); $this->connection->exec($a); } catch (\Exception $e) { $return[] = [ @@ -512,6 +561,7 @@ public function upgradeTable(\SimpleXMLElement $table) $this->reconnect(); try { + $output->writeln(sprintf("%s \n", $a)); $this->connection->exec($a); } catch (\Exception $e) { $return[] = [ @@ -541,7 +591,7 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf foreach ($iterator as $fileinfo) { if (!$fileinfo->isDot()) { -// printf("---- [%d]\n", __LINE__); + if (substr($fileinfo->getFilename(), 0, 1) == '.') { continue; } @@ -551,7 +601,6 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf /** @var \patchAbstract $patch */ $patch = new $classname(); -// printf("---- [%d]\n", __LINE__); if (!in_array($base->get_base_type(), $patch->concern())) { continue; } @@ -560,18 +609,15 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf continue; } -// printf("---- [%d] %s ; from: %s ; patch: %s; to:%s\n", __LINE__, $classname, $from, $patch->get_release(), $to); -// printf("---- [%d]\n", __LINE__); // if patch is older than current install if (version::lte($patch->get_release(), $from)) { continue; } -// printf("---- [%d]\n", __LINE__); + // if patch is new than current target if (version::gt($patch->get_release(), $to)) { continue; } -// printf("---- [%d]\n", __LINE__); $n = 0; do { @@ -591,7 +637,7 @@ public function applyPatches(\base $base, $from, $to, $post_process, InputInterf // disable mail $this->app['swiftmailer.transport'] = null; -// var_dump($list_patches); + foreach ($list_patches as $patch) { $output->writeln(sprintf(" - patch \"%s\" (release %s) should be applied", get_class($patch), $patch->get_release()));