From 7f2561252d7b53bae094fc99abaee4d80b0c3111 Mon Sep 17 00:00:00 2001 From: Richan Fongdasen Date: Sun, 14 Apr 2024 20:37:06 +0700 Subject: [PATCH] Add support for Turso's embedded replica feature --- .gitignore | 2 + README.md | 67 ++++++++++++- config/turso-laravel.php | 8 +- package.json | 9 ++ src/Commands/TursoSyncCommand.php | 45 +++++++++ src/Database/TursoConnection.php | 52 ++++++++++ src/Facades/Turso.php | 8 +- src/Jobs/TursoSyncJob.php | 28 ++++++ src/{TursoClient.php => TursoHttpClient.php} | 36 +++++-- src/TursoLaravelServiceProvider.php | 23 ++++- src/TursoManager.php | 71 ++++++++++++++ tests/Feature/ReadReplicaTest.php | 53 +++++++++++ tests/TestCase.php | 3 +- ...ClientTest.php => TursoHttpClientTest.php} | 0 tests/Unit/TursoManagerTest.php | 95 +++++++++++++++++++ tests/Unit/TursoSyncCommandTest.php | 39 ++++++++ turso-sync.mjs | 21 ++++ 17 files changed, 536 insertions(+), 24 deletions(-) create mode 100644 package.json create mode 100644 src/Commands/TursoSyncCommand.php create mode 100644 src/Jobs/TursoSyncJob.php rename src/{TursoClient.php => TursoHttpClient.php} (86%) create mode 100644 src/TursoManager.php create mode 100644 tests/Feature/ReadReplicaTest.php rename tests/Unit/{TursoClientTest.php => TursoHttpClientTest.php} (100%) create mode 100644 tests/Unit/TursoManagerTest.php create mode 100644 tests/Unit/TursoSyncCommandTest.php create mode 100644 turso-sync.mjs diff --git a/.gitignore b/.gitignore index 96cda9d..6b932e1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,10 @@ phpunit.xml phpstan.neon testbench.yaml vendor +package-lock.json node_modules *.swp *.sqlite *.sqlite-shm *.sqlite-wal +*.sqlite-client_wal_index diff --git a/README.md b/README.md index 9aed751..77b7e28 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,36 @@ To use Turso as your database driver in Laravel, you need to append the followin ```php 'turso' => [ 'driver' => 'turso', - 'turso_url' => env('DB_URL', 'http://localhost:8080'), - 'database' => null, - 'prefix' => env('DB_PREFIX', ''), + 'db_url' => env('DB_URL', 'http://localhost:8080'), 'access_token' => env('DB_ACCESS_TOKEN', null), + 'db_replica' => env('DB_REPLICA'), + 'database' => null, // Leave this null + 'prefix' => env('DB_PREFIX', ''), 'foreign_key_constraints' => env('DB_FOREIGN_KEYS', true), ], ``` +### Publishing Configuration and Sync Script + +You can publish the configuration file and sync script by running the following command: + +```bash +php artisan vendor:publish --provider="RichanFongdasen\Turso\TursoLaravelServiceProvider" +``` + +The above command will publish the following files: + +- `config/turso-laravel.php` +- `turso-sync.mjs` + +### Installing Node.js Dependencies + +The Turso database driver requires Node.js to run the sync script. You can install the Node.js dependencies by running the following command: + +```bash +npm install @libsql/client +``` + ## Configuration In Laravel application, The database driver configuration is stored in your `.env` file. Here is the list of available configuration for Turso database driver: @@ -54,16 +76,53 @@ In Laravel application, The database driver configuration is stored in your `.en ```bash DB_CONNECTION=turso DB_URL=http://localhost:8080 -DB_PREFIX= DB_ACCESS_TOKEN= +DB_REPLICA= +DB_PREFIX= +DB_FOREIGN_KEYS=true ``` +| ENV Variable Name | Description | +| :---------------- | :--------------------------------------------------------------------------------------------- | +| DB_URL | The Turso database server URL. E.g: `https://[databaseName]-[organizationName].turso.io` | +| DB_ACCESS_TOKEN | (Optional) The access token to access the Turso database server. | +| DB_REPLICA | (Optional) The full path to the local embedded replica database file. E.g: `/tmp/turso.sqlite` | +| DB_PREFIX | (Optional) The database table prefix. | +| DB_FOREIGN_KEYS | Enable or disable foreign key constraints, default is `true`. | + ## Usage For local development, you can use the local Turso database server that is provided by the Turso database team for development purposes. You can find the instruction to run the local Turso database server in the [Turso CLI documentation](https://docs.turso.tech/local-development#turso-cli). The Turso database driver should work as expected with Laravel Query Builder and Eloquent ORM. +The database driver supports the embedded replica feature. If you're not familiar with the embedded replica feature, you can read the [Turso embedded replica](https://turso.tech/blog/introducing-embedded-replicas-deploy-turso-anywhere-2085aa0dc242) article. + +### Running the sync script from artisan command + +You can run the sync script manually from the artisan command by using the following command: + +```bash +php artisan turso:sync +``` + +### Running the sync script programmatically + +You can run the sync script programmatically by using the following code: + +```php +use Illuminate\Support\Facades\DB; +use RichanFongdasen\Turso\Facades\Turso; + +if ( DB::hasUpdated() ) { + // Run the sync script immediately + Turso::sync(); + + // Run the sync script in the background + Turso::backgroundSync(); +} +``` + ## Debugging There is a way to debug the HTTP request and response that is sent and received by the Turso database client. Here is the example of how to enable the debugging feature: diff --git a/config/turso-laravel.php b/config/turso-laravel.php index 41cc03e..4486310 100644 --- a/config/turso-laravel.php +++ b/config/turso-laravel.php @@ -1,4 +1,10 @@ [ + 'script_filename' => 'turso-sync.mjs', + 'script_path' => realpath(__DIR__ . '/..'), + 'timeout' => 60, + ], +]; diff --git a/package.json b/package.json new file mode 100644 index 0000000..e3f66e6 --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "private": true, + "scripts": { + "sync": "node turso-sync.js" + }, + "devDependencies": { + "@libsql/client": "^0.6.0" + } +} diff --git a/src/Commands/TursoSyncCommand.php b/src/Commands/TursoSyncCommand.php new file mode 100644 index 0000000..56539e5 --- /dev/null +++ b/src/Commands/TursoSyncCommand.php @@ -0,0 +1,45 @@ +path(config('turso-laravel.sync.script_path') ?? base_path()) + ->run($this->compileRunProcess()); + + if ($result->failed()) { + $this->error($result->errorOutput()); + + return self::FAILURE; + } + + $this->info($result->output()); + + return self::SUCCESS; + } +} diff --git a/src/Database/TursoConnection.php b/src/Database/TursoConnection.php index 0e46a86..ee81e08 100644 --- a/src/Database/TursoConnection.php +++ b/src/Database/TursoConnection.php @@ -9,6 +9,8 @@ class TursoConnection extends SQLiteConnection { + protected bool $hasUpdated = false; + public function __construct(TursoPDO $pdo, string $database = ':memory:', string $tablePrefix = '', array $config = []) { parent::__construct($pdo, $database, $tablePrefix, $config); @@ -67,4 +69,54 @@ protected function getDefaultPostProcessor(): TursoQueryProcessor { return new TursoQueryProcessor(); } + + /** + * Run an insert statement against the database. + * + * @param string $query + * @param array $bindings + * + * @return bool + */ + public function insert($query, $bindings = []) + { + $this->hasUpdated = true; + + return parent::insert($query, $bindings); + } + + /** + * Run an update statement against the database. + * + * @param string $query + * @param array $bindings + * + * @return int + */ + public function update($query, $bindings = []) + { + $this->hasUpdated = true; + + return parent::update($query, $bindings); + } + + /** + * Run a delete statement against the database. + * + * @param string $query + * @param array $bindings + * + * @return int + */ + public function delete($query, $bindings = []) + { + $this->hasUpdated = true; + + return parent::delete($query, $bindings); + } + + public function hasUpdated(): bool + { + return $this->hasUpdated; + } } diff --git a/src/Facades/Turso.php b/src/Facades/Turso.php index ca7dacd..1a2284d 100644 --- a/src/Facades/Turso.php +++ b/src/Facades/Turso.php @@ -5,17 +5,17 @@ namespace RichanFongdasen\Turso\Facades; use Illuminate\Support\Facades\Facade; -use RichanFongdasen\Turso\TursoClient; +use RichanFongdasen\Turso\TursoManager; /** - * @see \RichanFongdasen\Turso\TursoClient + * @see \RichanFongdasen\Turso\TursoHttpClient * - * @mixin \RichanFongdasen\Turso\TursoClient + * @mixin \RichanFongdasen\Turso\TursoHttpClient */ class Turso extends Facade { protected static function getFacadeAccessor(): string { - return TursoClient::class; + return TursoManager::class; } } diff --git a/src/Jobs/TursoSyncJob.php b/src/Jobs/TursoSyncJob.php new file mode 100644 index 0000000..331cfdb --- /dev/null +++ b/src/Jobs/TursoSyncJob.php @@ -0,0 +1,28 @@ +config = config('database.connections.turso', []); + $this->config = $config; $this->queryLog = new Collection(); @@ -35,7 +37,9 @@ public function __construct() public function __destruct() { - $this->close(); + if ($this->isOpen) { + $this->close(); + } } public function close(): void @@ -93,6 +97,15 @@ public function enableQueryLog(): void $this->loggingQueries = true; } + public function freshRequest(): PendingRequest + { + $this->resetClientState(); + + $this->request = $this->createRequest(); + + return $this->request; + } + public function getBaseUrl(): ?string { return $this->baseUrl; @@ -119,6 +132,10 @@ public function query(string $statement, array $bindingValues = []): array $response->throw(); } + if (! $this->isOpen) { + $this->isOpen = true; + } + $jsonResponse = $response->json(); if ($this->loggingQueries) { @@ -152,16 +169,15 @@ public function query(string $statement, array $bindingValues = []): array public function request(): PendingRequest { - if ($this->request === null) { - $this->request = $this->createRequest(); - } - - return $this->request; + return ($this->request === null) + ? $this->freshRequest() + : $this->request; } public function resetClientState(): void { $this->baton = null; - $this->baseUrl = data_get($this->config, 'turso_url'); + $this->baseUrl = data_get($this->config, 'db_url'); + $this->isOpen = false; } } diff --git a/src/TursoLaravelServiceProvider.php b/src/TursoLaravelServiceProvider.php index 45d0237..ca689c9 100644 --- a/src/TursoLaravelServiceProvider.php +++ b/src/TursoLaravelServiceProvider.php @@ -6,6 +6,8 @@ use Illuminate\Database\Connection; use Illuminate\Database\DatabaseManager; +use PDO; +use RichanFongdasen\Turso\Commands\TursoSyncCommand; use RichanFongdasen\Turso\Database\TursoConnection; use RichanFongdasen\Turso\Database\TursoConnector; use Spatie\LaravelPackageTools\Package; @@ -22,15 +24,20 @@ public function configurePackage(Package $package): void */ $package ->name('turso-laravel') - ->hasConfigFile(); + ->hasConfigFile() + ->hasCommand(TursoSyncCommand::class); + + $this->publishes([ + realpath(dirname(__DIR__) . '/turso-sync.mjs') => base_path('turso-sync.mjs'), + ], 'sync-script'); } public function register(): void { parent::register(); - $this->app->scoped(TursoClient::class, function () { - return new TursoClient(); + $this->app->scoped(TursoManager::class, function () { + return new TursoManager(config('database.connections.turso')); }); $this->app->extend(DatabaseManager::class, function (DatabaseManager $manager) { @@ -38,7 +45,15 @@ public function register(): void $connector = new TursoConnector(); $pdo = $connector->connect($config); - return new TursoConnection($pdo, $database ?? ':memory:', $prefix, $config); + $connection = new TursoConnection($pdo, $database ?? 'turso', $prefix, $config); + + $replicaPath = (string) data_get($config, 'db_replica'); + + if ($replicaPath !== '') { + $connection->setReadPdo(new PDO('sqlite:' . $replicaPath)); + } + + return $connection; }); return $manager; diff --git a/src/TursoManager.php b/src/TursoManager.php new file mode 100644 index 0000000..0bab933 --- /dev/null +++ b/src/TursoManager.php @@ -0,0 +1,71 @@ +config = new Collection($config); + $this->client = new TursoHttpClient($config); + } + + public function backgroundSync(): void + { + if ($this->config->get('db_replica', false) !== false) { + TursoSyncJob::dispatch(); + } + } + + public function disableReadReplica(): bool + { + $this->readPdo = DB::connection('turso')->getReadPdo(); + + DB::connection('turso')->setReadPdo(null); + + return true; + } + + public function enableReadReplica(): bool + { + if ($this->readPdo === null) { + return false; + } + + DB::connection('turso')->setReadPdo($this->readPdo); + + return true; + } + + public function sync(): void + { + if ($this->config->get('db_replica', false) !== false) { + Artisan::call('turso:sync'); + } + } + + public function __call(string $methodName, array $arguments = []): mixed + { + if (! method_exists($this->client, $methodName)) { + throw new BadMethodCallException('Call to undefined method ' . static::class . '::' . $methodName . '()'); + } + + // @phpstan-ignore-next-line + return call_user_func_array([$this->client, $methodName], $arguments); + } +} diff --git a/tests/Feature/ReadReplicaTest.php b/tests/Feature/ReadReplicaTest.php new file mode 100644 index 0000000..6d54bf0 --- /dev/null +++ b/tests/Feature/ReadReplicaTest.php @@ -0,0 +1,53 @@ +pdo = new \PDO('sqlite::memory:'); + $this->pdo->exec('CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)'); + $this->pdo->exec('INSERT INTO users (name) VALUES ("John Doe")'); + $this->pdo->exec('INSERT INTO users (name) VALUES ("Jane Doe")'); + + DB::connection('turso')->setReadPdo($this->pdo); +}); + +test('it can retrieve data from read replica', function () { + $users = DB::table('users')->get(); + + expect($users)->toHaveCount(2) + ->and($users[0]->name)->toBe('John Doe') + ->and($users[1]->name)->toBe('Jane Doe'); +})->group('ReadReplicaTest', 'FeatureTest'); + +test('it will use the primary database connection for data manipulation operation', function () { + Http::fake([ + '*' => Http::response(), + ]); + + Turso::freshRequest(); + + DB::table('users')->insert([ + 'name' => 'June Monroe', + ]); + + Http::assertSent(function (Request $request) { + expect($request->url())->toBe('http://127.0.0.1:8080/v3/pipeline') + ->and($request->data())->toBe([ + 'requests' => [[ + 'type' => 'execute', + 'stmt' => [ + 'sql' => 'insert into "users" ("name") values (?)', + 'args' => [[ + 'type' => 'text', + 'value' => 'June Monroe', + ]], + ], + ]], + ]); + + return true; + }); +})->group('ReadReplicaTest', 'FeatureTest'); diff --git a/tests/TestCase.php b/tests/TestCase.php index 321275d..7caeb8c 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -28,7 +28,8 @@ public function getEnvironmentSetUp($app) { config()->set('database.connections.turso', [ 'driver' => 'turso', - 'turso_url' => env('DB_URL', 'http://127.0.0.1:8080'), + 'db_url' => env('DB_URL', 'http://127.0.0.1:8080'), + 'db_replica' => env('DB_REPLICA'), 'database' => null, 'prefix' => env('DB_PREFIX', ''), 'access_token' => 'your-access-token', diff --git a/tests/Unit/TursoClientTest.php b/tests/Unit/TursoHttpClientTest.php similarity index 100% rename from tests/Unit/TursoClientTest.php rename to tests/Unit/TursoHttpClientTest.php diff --git a/tests/Unit/TursoManagerTest.php b/tests/Unit/TursoManagerTest.php new file mode 100644 index 0000000..186a7e5 --- /dev/null +++ b/tests/Unit/TursoManagerTest.php @@ -0,0 +1,95 @@ +toBeFalse(); +})->group('TursoManagerTest', 'UnitTest'); + +test('it can disable the read replica database connection', function () { + DB::connection('turso')->setReadPdo(new \PDO('sqlite::memory:')); + + Http::fake(); + + Turso::disableReadReplica(); + Turso::resetClientState(); + + Turso::query('SELECT * FROM sqlite_master'); + + Http::assertSent(function (Request $request) { + expect($request->url())->toBe('http://127.0.0.1:8080/v3/pipeline') + ->and($request->data())->toBe([ + 'requests' => [[ + 'type' => 'execute', + 'stmt' => [ + 'sql' => 'SELECT * FROM sqlite_master', + ], + ]], + ]); + + return true; + }); +})->group('TursoManagerTest', 'UnitTest'); + +test('it can reenable the read replica database connection', function () { + Turso::disableReadReplica(); + + expect(Turso::enableReadReplica())->toBeTrue(); +})->group('TursoManagerTest', 'UnitTest'); + +test('it raises exception on calling an undefined method', function () { + Turso::undefinedMethod(); +})->throws(\BadMethodCallException::class)->group('TursoManagerTest', 'UnitTest'); + +test('it can trigger the sync command immediately', function () { + Process::fake(); + + config(['database.connections.turso.db_replica' => '/tmp/turso.sqlite']); + + Turso::sync(); + + Process::assertRan(function (PendingProcess $process) { + $expectedPath = realpath(__DIR__ . '/../..'); + + expect($process->command)->toBe('node turso-sync.mjs "http://127.0.0.1:8080" "your-access-token" "/tmp/turso.sqlite"') + ->and($process->timeout)->toBe(60) + ->and($process->path)->toBe($expectedPath); + + return true; + }); +})->group('TursoManagerTest', 'UnitTest'); + +test('it can dispatch the sync background job', function () { + Bus::fake(); + + config(['database.connections.turso.db_replica' => '/tmp/turso.sqlite']); + + Turso::backgroundSync(); + + Bus::assertDispatched(TursoSyncJob::class); +})->group('TursoManagerTest', 'UnitTest'); + +test('it can run the sync background job and call the sync artisan command', function () { + Process::fake(); + + config(['database.connections.turso.db_replica' => '/tmp/turso.sqlite']); + + Turso::backgroundSync(); + + Process::assertRan(function (PendingProcess $process) { + $expectedPath = realpath(__DIR__ . '/../..'); + + expect($process->command)->toBe('node turso-sync.mjs "http://127.0.0.1:8080" "your-access-token" "/tmp/turso.sqlite"') + ->and($process->timeout)->toBe(60) + ->and($process->path)->toBe($expectedPath); + + return true; + }); +})->group('TursoManagerTest', 'UnitTest'); diff --git a/tests/Unit/TursoSyncCommandTest.php b/tests/Unit/TursoSyncCommandTest.php new file mode 100644 index 0000000..0ede505 --- /dev/null +++ b/tests/Unit/TursoSyncCommandTest.php @@ -0,0 +1,39 @@ + '/tmp/turso.sqlite']); + + Artisan::call('turso:sync'); + + Process::assertRan(function (PendingProcess $process) { + $expectedPath = realpath(__DIR__ . '/../..'); + + expect($process->command)->toBe('node turso-sync.mjs "http://127.0.0.1:8080" "your-access-token" "/tmp/turso.sqlite"') + ->and($process->timeout)->toBe(60) + ->and($process->path)->toBe($expectedPath); + + return true; + }); +})->group('TursoSyncCommandTest', 'UnitTest'); + +test('it can handle process error output', function () { + Process::fake([ + '*' => Process::result( + output: 'Whooops! Something went wrong!', + errorOutput: 'Error: Something went wrong!', + exitCode: 500 + ), + ]); + + config(['database.connections.turso.db_replica' => '/tmp/turso.sqlite']); + + $result = Artisan::call('turso:sync'); + + expect($result)->toBe(1); +})->group('TursoSyncCommandTest', 'UnitTest'); diff --git a/turso-sync.mjs b/turso-sync.mjs new file mode 100644 index 0000000..e83b8fd --- /dev/null +++ b/turso-sync.mjs @@ -0,0 +1,21 @@ +import { createClient } from "@libsql/client"; + +let databaseURL = process.argv.slice(2)[0]; +const accessToken = process.argv.slice(2)[1]; +const replicaPath = process.argv.slice(2)[2]; + +if (databaseURL.startsWith('https://')) { + databaseURL = databaseURL.replace('https://', 'libsql://'); +} + +const client = createClient({ + url: `file:${replicaPath}`, + syncUrl: databaseURL, + authToken: accessToken, +}); + +console.log('Syncing database to replica ' + replicaPath + ' from ' + databaseURL); + +await client.sync(); + +console.log('Sync completed.');