-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into duckdb-arrow-integration
- Loading branch information
Showing
43 changed files
with
5,819 additions
and
183 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| name: MySQL Copy Instance Test | ||
|
|
||
| on: | ||
| push: | ||
| branches: [ "main" ] | ||
| pull_request: | ||
| branches: [ "main" ] | ||
|
|
||
| jobs: | ||
| copy-instance-test: | ||
| runs-on: ubuntu-latest | ||
| services: | ||
| source: | ||
| image: mysql:lts | ||
| env: | ||
| MYSQL_ROOT_PASSWORD: root | ||
| ports: | ||
| - 13306:3306 | ||
| options: >- | ||
| --health-cmd="mysqladmin ping" | ||
| --health-interval=10s | ||
| --health-timeout=5s | ||
| --health-retries=3 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| - name: Set up Go | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version: '1.23' | ||
|
|
||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.13' | ||
|
|
||
| - name: Install dependencies | ||
| run: | | ||
| go get . | ||
| pip3 install "sqlglot[rs]" | ||
| curl -LJO https://dev.mysql.com/get/Downloads/MySQL-Shell/mysql-shell_9.1.0-1debian12_amd64.deb | ||
| sudo apt-get install -y ./mysql-shell_9.1.0-1debian12_amd64.deb | ||
| - name: Setup test data in source MySQL | ||
| run: | | ||
| mysqlsh -hlocalhost -P13306 -uroot -proot --sql -e " | ||
| CREATE DATABASE testdb; | ||
| USE testdb; | ||
| CREATE TABLE users ( | ||
| id INT AUTO_INCREMENT PRIMARY KEY, | ||
| name VARCHAR(100) | ||
| ); | ||
| INSERT INTO users (name) VALUES ('test1'), ('test2'), ('test3'); | ||
| -- Make a gap in the id sequence | ||
| INSERT INTO users VALUES (100, 'test100'); | ||
| INSERT INTO users (name) VALUES ('test101'); | ||
| -- A table with non-default starting auto_increment value | ||
| CREATE TABLE items ( | ||
| id INT AUTO_INCREMENT PRIMARY KEY, | ||
| v BIGINT check (v > 0), | ||
| name VARCHAR(100) | ||
| ) AUTO_INCREMENT=1000; | ||
| INSERT INTO items (v, name) VALUES (1, 'item1'), (2, 'item2'), (3, 'item3'); | ||
| " | ||
| - name: Build and start MyDuck Server | ||
| run: | | ||
| go build -v | ||
| ./myduckserver & | ||
| sleep 5 | ||
| - name: Run copy-instance test | ||
| run: | | ||
| # Set local_infile to true to allow loading data from files | ||
| mysqlsh -uroot --no-password --sql -e "SET GLOBAL local_infile = 1;" | ||
| # Copy the data from source MySQL to MyDuck | ||
| mysqlsh -hlocalhost -P13306 -uroot -proot \ | ||
| -- util copy-instance "mysql://root:@127.0.0.1:3306" \ | ||
| --users false --ignore-version true | ||
| # Verify the data was copied | ||
| for table in users items; do | ||
| mysqlsh -hlocalhost -P13306 -uroot -proot --sql -e " | ||
| SELECT * FROM testdb.$table ORDER BY id; | ||
| " | tee source_data_$table.tsv | ||
| mysqlsh -uroot --no-password --sql -e " | ||
| SELECT * FROM testdb.$table ORDER BY id; | ||
| " | tee copied_data_$table.tsv | ||
| diff source_data_$table.tsv copied_data_$table.tsv | ||
| done | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -90,23 +90,29 @@ jobs: | |
| docker exec source-db dolt sql -q " | ||
| CREATE DATABASE test; | ||
| CREATE TABLE test.items (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO test.items VALUES (1, 'test1'), (2, 'test2');" | ||
| INSERT INTO test.items VALUES (1, 'test1'), (2, 'test2'); | ||
| CREATE TABLE test.skip (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO test.skip VALUES (1, 'abc'), (2, 'def');" | ||
| elif [ "${{ matrix.source }}" = "mariadb" ]; then | ||
| docker exec source-db mariadb -uroot -proot test -e " | ||
| CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2');" | ||
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2'); | ||
| CREATE TABLE skip (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO skip VALUES (1, 'abc'), (2, 'def');" | ||
| else | ||
| docker exec source-db mysql -uroot -proot test -e " | ||
| CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2');" | ||
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2'); | ||
| CREATE TABLE skip (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO skip VALUES (1, 'abc'), (2, 'def');" | ||
| fi | ||
| - name: Start MyDuck Server in replica mode | ||
| run: | | ||
| if [ "${{ matrix.source }}" = "postgres" ]; then | ||
| SOURCE_DSN="postgres://postgres:[email protected]:5432/test" | ||
| else | ||
| SOURCE_DSN="mysql://root:[email protected]:3306" | ||
| SOURCE_DSN="mysql://root:[email protected]:3306/test?skip-tables=test.skip" | ||
| fi | ||
| docker run -d --name myduck \ | ||
|
|
@@ -203,6 +209,31 @@ jobs: | |
| exit 1 | ||
| fi | ||
| # Print the logs | ||
| docker logs myduck | ||
| - name: Verify skip tables | ||
| run: | | ||
| # Verify skipped table is empty (for MySQL-compatible databases only) | ||
| if [ "${{ matrix.source }}" != "postgres" ]; then | ||
| # Check if skip table exists and has any rows | ||
| TABLE_EXISTS=$(docker exec myduck psql -t -U postgres -h 127.0.0.1 -c \ | ||
| "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${SCHEMA}' AND table_name = 'skip';" | tr -d ' ') | ||
| if [ "$TABLE_EXISTS" -ne "0" ]; then | ||
| COUNT=$(docker exec myduck psql -t -U postgres -h 127.0.0.1 -c \ | ||
| "SELECT COUNT(*) FROM ${SCHEMA}.skip;" | tr -d ' ') | ||
| if [ "$COUNT" -eq "0" ]; then | ||
| echo "Successfully verified that skipped table exists but is empty" | ||
| else | ||
| echo "Error: Skipped table 'skip' contains $COUNT rows when it should be empty" | ||
| exit 1 | ||
| fi | ||
| else | ||
| echo "Successfully verified that skipped table does not exist in destination" | ||
| fi | ||
| fi | ||
| - name: Cleanup | ||
| if: always() | ||
| run: | | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.