From b5da0d7a3186c0877a788a93312272757b5f3565 Mon Sep 17 00:00:00 2001 From: Daniel Arbuckle Date: Wed, 29 Jan 2025 07:11:23 -0800 Subject: [PATCH 1/4] SQLite extension loading via sqlx.toml for CLI and query macros --- sqlx-cli/src/lib.rs | 2 +- sqlx-core/src/any/connection/mod.rs | 14 +++++++++++ sqlx-core/src/any/options.rs | 15 +++++++++++ sqlx-core/src/config/common.rs | 39 +++++++++++++++++++++++++++++ sqlx-core/src/config/reference.toml | 6 +++++ sqlx-sqlite/src/any.rs | 8 ++++++ sqlx-sqlite/src/lib.rs | 9 ++++++- 7 files changed, 91 insertions(+), 2 deletions(-) diff --git a/sqlx-cli/src/lib.rs b/sqlx-cli/src/lib.rs index 43b301e4c5..ff9cd41ae2 100644 --- a/sqlx-cli/src/lib.rs +++ b/sqlx-cli/src/lib.rs @@ -131,7 +131,7 @@ pub async fn run(opt: Opt) -> Result<()> { /// Attempt to connect to the database server, retrying up to `ops.connect_timeout`. async fn connect(opts: &ConnectOpts) -> anyhow::Result { - retry_connect_errors(opts, AnyConnection::connect).await + retry_connect_errors(opts, AnyConnection::connect_with_config).await } /// Attempt an operation that may return errors like `ConnectionRefused`, diff --git a/sqlx-core/src/any/connection/mod.rs b/sqlx-core/src/any/connection/mod.rs index b6f795848a..f27dad05d3 100644 --- a/sqlx-core/src/any/connection/mod.rs +++ b/sqlx-core/src/any/connection/mod.rs @@ -39,6 +39,20 @@ impl AnyConnection { }) } + /// UNSTABLE: for use with `sqlx-cli` + /// + /// Connect to the database, and instruct the nested driver to + /// read options from the sqlx.toml file as appropriate. + #[doc(hidden)] + pub fn connect_with_config(url: &str) -> BoxFuture<'static, Result> + where + Self: Sized, + { + let options: Result = url.parse(); + + Box::pin(async move { Self::connect_with(&options?.allow_config_file()).await }) + } + pub(crate) fn connect_with_db( options: &AnyConnectOptions, ) -> BoxFuture<'_, crate::Result> diff --git a/sqlx-core/src/any/options.rs b/sqlx-core/src/any/options.rs index bb29d817c9..5ed68efec5 100644 --- a/sqlx-core/src/any/options.rs +++ b/sqlx-core/src/any/options.rs @@ -19,6 +19,7 @@ use url::Url; pub struct AnyConnectOptions { pub database_url: Url, pub log_settings: LogSettings, + pub enable_config: bool, } impl FromStr for AnyConnectOptions { type Err = Error; @@ -29,6 +30,7 @@ impl FromStr for AnyConnectOptions { .parse::() .map_err(|e| Error::Configuration(e.into()))?, log_settings: LogSettings::default(), + enable_config: false, }) } } @@ -40,6 +42,7 @@ impl ConnectOptions for AnyConnectOptions { Ok(AnyConnectOptions { database_url: url.clone(), log_settings: LogSettings::default(), + enable_config: false, }) } @@ -63,3 +66,15 @@ impl ConnectOptions for AnyConnectOptions { self } } + +impl AnyConnectOptions { + /// UNSTABLE: for use with `sqlx-cli` + /// + /// Allow nested drivers to extract configuration information from + /// the sqlx.toml file. + #[doc(hidden)] + pub fn allow_config_file(mut self) -> Self { + self.enable_config = true; + self + } +} diff --git a/sqlx-core/src/config/common.rs b/sqlx-core/src/config/common.rs index d2bf639e5f..9a17c7d0ef 100644 --- a/sqlx-core/src/config/common.rs +++ b/sqlx-core/src/config/common.rs @@ -40,6 +40,14 @@ pub struct Config { /// The query macros used in `foo` will use `FOO_DATABASE_URL`, /// and the ones used in `bar` will use `BAR_DATABASE_URL`. pub database_url_var: Option, + + /// Settings for specific database drivers. + /// + /// These settings apply when checking queries, or when applying + /// migrations via `sqlx-cli`. These settings *do not* apply when + /// applying migrations via the macro, as that uses the run-time + /// database connection configured by the application. + pub drivers: Drivers, } impl Config { @@ -47,3 +55,34 @@ impl Config { self.database_url_var.as_deref().unwrap_or("DATABASE_URL") } } + +/// Configuration for specific database drivers. +#[derive(Debug, Default)] +#[cfg_attr( + feature = "sqlx-toml", + derive(serde::Deserialize), + serde(default, rename_all = "kebab-case") +)] +pub struct Drivers { + /// Specify options for the SQLite driver. + pub sqlite: SQLite, +} + +/// Configuration for the SQLite database driver. +#[derive(Debug, Default)] +#[cfg_attr( + feature = "sqlx-toml", + derive(serde::Deserialize), + serde(default, rename_all = "kebab-case") +)] +pub struct SQLite { + /// Specify extensions to load. + /// + /// ### Example: Load the "uuid" and "vsv" extensions + /// `sqlx.toml`: + /// ```toml + /// [common.drivers.sqlite] + /// load-extensions = ["uuid", "vsv"] + /// ``` + pub load_extensions: Vec, +} diff --git a/sqlx-core/src/config/reference.toml b/sqlx-core/src/config/reference.toml index 77833fb5a8..787c3456db 100644 --- a/sqlx-core/src/config/reference.toml +++ b/sqlx-core/src/config/reference.toml @@ -15,6 +15,12 @@ # If not specified, defaults to `DATABASE_URL` database-url-var = "FOO_DATABASE_URL" +[common.drivers.sqlite] +# Load extensions into SQLite when running macros or migrations +# +# Defaults to an empty list, which has no effect. +load-extensions = ["uuid", "vsv"] + ############################################################################################### # Configuration for the `query!()` family of macros. diff --git a/sqlx-sqlite/src/any.rs b/sqlx-sqlite/src/any.rs index 01600d9931..f12009730f 100644 --- a/sqlx-sqlite/src/any.rs +++ b/sqlx-sqlite/src/any.rs @@ -191,6 +191,14 @@ impl<'a> TryFrom<&'a AnyConnectOptions> for SqliteConnectOptions { fn try_from(opts: &'a AnyConnectOptions) -> Result { let mut opts_out = SqliteConnectOptions::from_url(&opts.database_url)?; opts_out.log_settings = opts.log_settings.clone(); + + if opts.enable_config { + let config = sqlx_core::config::Config::from_crate(); + for extension in config.common.drivers.sqlite.load_extensions.iter() { + opts_out = opts_out.extension(extension); + } + } + Ok(opts_out) } } diff --git a/sqlx-sqlite/src/lib.rs b/sqlx-sqlite/src/lib.rs index 3bcb6d148d..a5d4f8661f 100644 --- a/sqlx-sqlite/src/lib.rs +++ b/sqlx-sqlite/src/lib.rs @@ -124,8 +124,15 @@ pub static CREATE_DB_WAL: AtomicBool = AtomicBool::new(true); /// UNSTABLE: for use by `sqlite-macros-core` only. #[doc(hidden)] pub fn describe_blocking(query: &str, database_url: &str) -> Result, Error> { - let opts: SqliteConnectOptions = database_url.parse()?; + let mut opts: SqliteConnectOptions = database_url.parse()?; + + let config = sqlx_core::config::Config::from_crate(); + for extension in config.common.drivers.sqlite.load_extensions.iter() { + opts = opts.extension(extension); + } + let params = EstablishParams::from_options(&opts)?; + let mut conn = params.establish()?; // Execute any ancillary `PRAGMA`s From f3d2d38a5a94103db286d8ace4328cb7770f217b Mon Sep 17 00:00:00 2001 From: Daniel Arbuckle Date: Fri, 31 Jan 2025 03:50:57 -0800 Subject: [PATCH 2/4] fix: allow start_database to function when the SQLite database file does not already exist --- tests/docker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/docker.py b/tests/docker.py index b1b81b07fb..5e8c74fb1f 100644 --- a/tests/docker.py +++ b/tests/docker.py @@ -17,9 +17,10 @@ def start_database(driver, database, cwd): database = path.join(cwd, database) (base_path, ext) = path.splitext(database) new_database = f"{base_path}.test{ext}" - shutil.copy(database, new_database) + if path.exists(database): + shutil.copy(database, new_database) # short-circuit for sqlite - return f"sqlite://{path.join(cwd, new_database)}" + return f"sqlite://{path.join(cwd, new_database)}?mode=rwc" res = subprocess.run( ["docker-compose", "up", "-d", driver], From 22d0d2c6c3dcbb1b1a4ee19b6ba8ef9aa39638e1 Mon Sep 17 00:00:00 2001 From: Daniel Arbuckle Date: Mon, 3 Feb 2025 03:03:15 -0800 Subject: [PATCH 3/4] Added example demonstrating migration and compile-time checking with SQLite extensions --- Cargo.toml | 1 + examples/sqlite/extension/Cargo.toml | 17 +++++++++ .../sqlite/extension/download-extension.sh | 9 +++++ examples/sqlite/extension/extension.test.db | Bin 0 -> 16384 bytes .../migrations/20250203094951_addresses.sql | 25 +++++++++++++ examples/sqlite/extension/sqlx.toml | 12 +++++++ examples/sqlite/extension/src/main.rs | 33 ++++++++++++++++++ examples/x.py | 1 + 8 files changed, 98 insertions(+) create mode 100644 examples/sqlite/extension/Cargo.toml create mode 100755 examples/sqlite/extension/download-extension.sh create mode 100644 examples/sqlite/extension/extension.test.db create mode 100644 examples/sqlite/extension/migrations/20250203094951_addresses.sql create mode 100644 examples/sqlite/extension/sqlx.toml create mode 100644 examples/sqlite/extension/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index b75b3c3bd7..7fd47239a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "examples/postgres/mockable-todos", "examples/postgres/transaction", "examples/sqlite/todos", + "examples/sqlite/extension", ] [workspace.package] diff --git a/examples/sqlite/extension/Cargo.toml b/examples/sqlite/extension/Cargo.toml new file mode 100644 index 0000000000..bf20add4b3 --- /dev/null +++ b/examples/sqlite/extension/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sqlx-example-sqlite-extension" +version = "0.1.0" +license.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +authors.workspace = true + +[dependencies] +sqlx = { path = "../../../", features = [ "sqlite", "runtime-tokio", "tls-native-tls" ] } +tokio = { version = "1.20.0", features = ["rt", "macros"]} +anyhow = "1.0" + +[lints] +workspace = true diff --git a/examples/sqlite/extension/download-extension.sh b/examples/sqlite/extension/download-extension.sh new file mode 100755 index 0000000000..ce7f23a486 --- /dev/null +++ b/examples/sqlite/extension/download-extension.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# This grabs a pre-compiled version of the extension used in this +# example, and stores it in a temporary directory. That's a bit +# unusual. Normally, any extensions you need will be installed into a +# directory on the library search path, either by using the system +# package manager or by compiling and installing it yourself. + +mkdir /tmp/sqlite3-lib && wget -O /tmp/sqlite3-lib/ipaddr.so https://github.com/nalgeon/sqlean/releases/download/0.15.2/ipaddr.so diff --git a/examples/sqlite/extension/extension.test.db b/examples/sqlite/extension/extension.test.db new file mode 100644 index 0000000000000000000000000000000000000000..a7f1325883ccd36ba1e882a496bf923277394637 GIT binary patch literal 16384 zcmeI%!Ee$~7y$6Llx|?xHO^0VCQm5Qd6*&OsNX)rt#R% zOKg#*HIB=Ls!`d%M$IBCq#oP~(tRVL5dFY=w5)F4sxx*Qq^|ZsF_xw zQXwV1RZq_=c`Z4=&>e)&D;JVWs)py5ayccV>D?ji>1Lx?v}JHAn^l!$raP})EnX+Z zn`WbmQ))KVQzK1MY_xWzTSk?1w%^&DKRCm1Bl!?b8MP8=+EDgyc)ifDR*z!67ekkO zGama<>u)`719qp;Q_4P|fdB}A00@8p2!H?xfB*=900@8p2>dO9t8||?5b;&mp&JLm zd(XNrQ^HbsG9-mVQUuF#LXs1a=zH(0`s~5-;c(ys+1NG%Zy$Y|ejKid_eT@Y7cY(O z$R|JTp4q!IxV3ls?tW}%hFSj-@Y~;uMxQDBfCd5}00JNY0w4eaAOHd&00JNY0wC~T z3HUr-U;BMR>;L~D;RS_0qJ8uR?V=~>A=*Smq@r_3LZ{HM@Lf0*-U+YmLC`<|1V8`; zKmY_l00ck)1V8`;K;X{_@I#&xv@A`uK5 anyhow::Result<()> { + let opts = SqliteConnectOptions::from_str(&std::env::var("DATABASE_URL")?)? + // The sqlx.toml file controls loading extensions for the CLI + // and for the query checking macros, *not* for the + // application while it's running. Thus, if we want the + // extension to be available during program execution, we need + // to load it. + // + // Note that while in this case the extension path is the same + // when checking the program (sqlx.toml) and when running it + // (here), this is not required. The runtime environment can + // be entirely different from the development one. + // + // The extension can be described with a full path, as seen + // here, but in many cases that will not be necessary. As long + // as the extension is installed in a directory on the library + // search path, it is sufficient to just provide the extension + // name, like "ipaddr" + .extension("/tmp/sqlite3-lib/ipaddr"); + + let db = SqlitePool::connect_with(opts).await?; + + query!("insert into addresses (address, family) values (?1, ipfamily(?1))", "10.0.0.10").execute(&db).await?; + + println!("Query which requires the extension was successfully executed."); + + Ok(()) +} diff --git a/examples/x.py b/examples/x.py index 79f6fda1ba..aaf4170c77 100755 --- a/examples/x.py +++ b/examples/x.py @@ -85,3 +85,4 @@ def project(name, database=None, driver=None): project("mysql/todos", driver="mysql_8", database="todos") project("postgres/todos", driver="postgres_12", database="todos") project("sqlite/todos", driver="sqlite", database="todos.db") +project("sqlite/extension", driver="sqlite", database="extension.db") From 0229e1e0c0efb0339287ddabd4adb61b91b78c1b Mon Sep 17 00:00:00 2001 From: Daniel Arbuckle Date: Mon, 3 Feb 2025 03:04:19 -0800 Subject: [PATCH 4/4] remove accidentally included db file --- examples/sqlite/extension/extension.test.db | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/sqlite/extension/extension.test.db diff --git a/examples/sqlite/extension/extension.test.db b/examples/sqlite/extension/extension.test.db deleted file mode 100644 index a7f1325883ccd36ba1e882a496bf923277394637..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI%!Ee$~7y$6Llx|?xHO^0VCQm5Qd6*&OsNX)rt#R% zOKg#*HIB=Ls!`d%M$IBCq#oP~(tRVL5dFY=w5)F4sxx*Qq^|ZsF_xw zQXwV1RZq_=c`Z4=&>e)&D;JVWs)py5ayccV>D?ji>1Lx?v}JHAn^l!$raP})EnX+Z zn`WbmQ))KVQzK1MY_xWzTSk?1w%^&DKRCm1Bl!?b8MP8=+EDgyc)ifDR*z!67ekkO zGama<>u)`719qp;Q_4P|fdB}A00@8p2!H?xfB*=900@8p2>dO9t8||?5b;&mp&JLm zd(XNrQ^HbsG9-mVQUuF#LXs1a=zH(0`s~5-;c(ys+1NG%Zy$Y|ejKid_eT@Y7cY(O z$R|JTp4q!IxV3ls?tW}%hFSj-@Y~;uMxQDBfCd5}00JNY0w4eaAOHd&00JNY0wC~T z3HUr-U;BMR>;L~D;RS_0qJ8uR?V=~>A=*Smq@r_3LZ{HM@Lf0*-U+YmLC`<|1V8`; zKmY_l00ck)1V8`;K;X{_@I#&xv@A`uK5