diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f71609b..8cd8519 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -306,6 +306,36 @@ jobs: experimental: false py: "3.12" + - toxenv: "python3.8" + db: "mariadb:11.4" + legacy_db: 0 + experimental: false + py: "3.8" + + - toxenv: "python3.9" + db: "mariadb:11.4" + legacy_db: 0 + experimental: false + py: "3.9" + + - toxenv: "python3.10" + db: "mariadb:11.4" + legacy_db: 0 + experimental: false + py: "3.10" + + - toxenv: "python3.11" + db: "mariadb:11.4" + legacy_db: 0 + experimental: false + py: "3.11" + + - toxenv: "python3.12" + db: "mariadb:11.4" + legacy_db: 0 + experimental: false + py: "3.12" + - toxenv: "python3.8" db: "mysql:5.5" legacy_db: 1 @@ -425,15 +455,46 @@ jobs: legacy_db: 0 experimental: false py: "3.12" + + - toxenv: "python3.8" + db: "mysql:8.4" + legacy_db: 0 + experimental: true + py: "3.8" + + - toxenv: "python3.9" + db: "mysql:8.4" + legacy_db: 0 + experimental: true + py: "3.9" + + - toxenv: "python3.10" + db: "mysql:8.4" + legacy_db: 0 + experimental: true + py: "3.10" + + - toxenv: "python3.11" + db: "mysql:8.4" + legacy_db: 0 + experimental: true + py: "3.11" + + - toxenv: "python3.12" + db: "mysql:8.4" + legacy_db: 0 + experimental: true + py: "3.12" continue-on-error: ${{ matrix.experimental }} services: mysql: - image: "${{ matrix.db }}" + image: ${{ matrix.db }} ports: - 3306:3306 env: MYSQL_ALLOW_EMPTY_PASSWORD: yes - options: "--name=mysqld" + options: >- + --name=mysqld steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.py }} @@ -461,31 +522,52 @@ jobs: MYSQL_PORT: 3306 run: | set -e + while : do sleep 1 mysql -h127.0.0.1 -uroot -e 'select version()' && break done + + case "$DB" in + 'mysql:8.0'|'mysql:8.4') + mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" + docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + ;; + esac + + USER_CREATION_COMMANDS='' + WITH_PLUGIN='' + if [ "$DB" == 'mysql:8.0' ]; then WITH_PLUGIN='with mysql_native_password' - mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" - docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - mysql -uroot -h127.0.0.1 -e ' + USER_CREATION_COMMANDS=' CREATE USER user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", nopass_sha256 IDENTIFIED WITH "sha256_password", user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER;' - mysql -uroot -h127.0.0.1 -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' - else - WITH_PLUGIN='' + PASSWORD EXPIRE NEVER; + GRANT RELOAD ON *.* TO user_caching_sha2;' + elif [ "$DB" == 'mysql:8.4' ]; then + WITH_PLUGIN='with caching_sha2_password' + USER_CREATION_COMMANDS=' + CREATE USER + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER; + GRANT RELOAD ON *.* TO user_caching_sha2;' + fi + + if [ ! -z "$USER_CREATION_COMMANDS" ]; then + mysql -uroot -h127.0.0.1 -e "$USER_CREATION_COMMANDS" fi - mysql -h127.0.0.1 -uroot -e "create database $MYSQL_DATABASE DEFAULT CHARACTER SET utf8mb4" + + mysql -h127.0.0.1 -uroot -e "create database $MYSQL_DATABASE DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" mysql -h127.0.0.1 -uroot -e "create user $MYSQL_USER identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER};" mysql -h127.0.0.1 -uroot -e "create user ${MYSQL_USER}@localhost identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER}@localhost;" - name: Create db_credentials.json diff --git a/README.md b/README.md index 202ce97..0ee34a2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ [![PyPI](https://img.shields.io/pypi/v/sqlite3-to-mysql)](https://pypi.org/project/sqlite3-to-mysql/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/sqlite3-to-mysql)](https://pypistats.org/packages/sqlite3-to-mysql) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlite3-to-mysql)](https://pypi.org/project/sqlite3-to-mysql/) -[![MySQL Support](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0&color=2b5d80)](https://img.shields.io/static/v1?label=MySQL&message=5.6+|+5.7+|+8.0&color=2b5d80) -[![MariaDB Support](https://img.shields.io/static/v1?label=MariaDB&message=5.5+|+10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5+|+10.6|+10.11&color=C0765A)](https://img.shields.io/static/v1?label=MariaDB&message=10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5&color=C0765A) +[![MySQL Support](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80)](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80) +[![MariaDB Support](https://img.shields.io/static/v1?label=MariaDB&message=5.5+|+10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5+|+10.6|+10.11+|+11.4&color=C0765A)](https://img.shields.io/static/v1?label=MariaDB&message=5.5|+10.0+|+10.1+|+10.2+|+10.3+|+10.4+|+10.5|+11.4&color=C0765A) [![GitHub license](https://img.shields.io/github/license/techouse/sqlite3-to-mysql)](https://github.com/techouse/sqlite3-to-mysql/blob/master/LICENSE) [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE-OF-CONDUCT.md) [![PyPI - Format](https://img.shields.io/pypi/format/sqlite3-to-mysql)]((https://pypi.org/project/sqlite3-to-mysql/)) diff --git a/pyproject.toml b/pyproject.toml index 3375238..a608a19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ classifiers = [ ] dependencies = [ "Click>=8.1.3", - "mysql-connector-python>=8.2.0", + "mysql-connector-python>=9.0.0", "pytimeparse2", "python-dateutil>=2.9.0.post0", "types_python_dateutil", diff --git a/requirements_dev.txt b/requirements_dev.txt index d878abd..d93218c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,7 +2,7 @@ Click>=8.1.3 docker>=6.1.3 factory-boy Faker>=18.10.0 -mysql-connector-python>=8.2.0 +mysql-connector-python>=9.0.0 PyMySQL>=1.0.3 pytest>=7.3.1 pytest-cov diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index fd2e492..1c41266 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -107,7 +107,7 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0] ) if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci": - self._mysql_collation = "utf8mb4_general_ci" + self._mysql_collation = "utf8mb4_unicode_ci" self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) or False @@ -144,6 +144,8 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): port=self._mysql_port, ssl_disabled=self._mysql_ssl_disabled, use_pure=True, + charset=self._mysql_charset, + collation=self._mysql_collation, ) if isinstance(_mysql_connection, mysql.connector.MySQLConnection): self._mysql = _mysql_connection diff --git a/tests/conftest.py b/tests/conftest.py index 292e755..5dfc344 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -293,6 +293,8 @@ def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) -> password=mysql_credentials.password, host=mysql_credentials.host, port=mysql_credentials.port, + charset="utf8mb4", + collation="utf8mb4_unicode_ci", ) except mysql.connector.Error as err: if err.errno == errorcode.CR_SERVER_LOST: diff --git a/tests/func/sqlite3_to_mysql_test.py b/tests/func/sqlite3_to_mysql_test.py index eb8f831..5593799 100644 --- a/tests/func/sqlite3_to_mysql_test.py +++ b/tests/func/sqlite3_to_mysql_test.py @@ -348,6 +348,8 @@ def test_transfer_transfers_all_tables_in_sqlite_file( host=mysql_credentials.host, port=mysql_credentials.port, database=mysql_credentials.database, + charset="utf8mb4", + collation="utf8mb4_unicode_ci", ) ) server_version: t.Tuple[int, ...] = mysql_connector_connection.get_server_version() diff --git a/tests/func/test_cli.py b/tests/func/test_cli.py index c6d7d11..5cea609 100644 --- a/tests/func/test_cli.py +++ b/tests/func/test_cli.py @@ -77,6 +77,10 @@ def test_invalid_database_name( "_".join(faker.words(nb=3)), "-u", faker.first_name().lower(), + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result.exit_code > 0 @@ -99,6 +103,10 @@ def test_invalid_database_user( mysql_credentials.database, "-u", faker.first_name().lower(), + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result.exit_code > 0 @@ -123,6 +131,10 @@ def test_invalid_database_password( mysql_credentials.user, "--mysql-password", faker.password(length=16), + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result.exit_code > 0 @@ -145,6 +157,10 @@ def test_database_password_prompt( "-u", mysql_credentials.user, "-p", + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], input=mysql_credentials.password, ) @@ -168,6 +184,10 @@ def test_invalid_database_password_prompt( "-u", mysql_credentials.user, "-p", + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], input=faker.password(length=16), ) @@ -230,6 +250,10 @@ def test_mysql_skip_transfer_data( "--mysql-password", mysql_credentials.password, "--mysql-skip-transfer-data", + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result.exit_code == 0 @@ -254,6 +278,10 @@ def test_mysql_skip_create_tables( "--mysql-password", mysql_credentials.password, "--mysql-skip-transfer-data", + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result1.exit_code == 0 @@ -270,6 +298,10 @@ def test_mysql_skip_create_tables( "--mysql-password", mysql_credentials.password, "--mysql-skip-create-tables", + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result2.exit_code == 0 @@ -294,6 +326,10 @@ def test_mysql_skip_create_tables_and_transfer_data( mysql_credentials.password, "--mysql-skip-create-tables", "--mysql-skip-transfer-data", + "-h", + mysql_credentials.host, + "-P", + str(mysql_credentials.port), ], ) assert result.exit_code > 0