diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f36d3ae5..9cec04b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,5 +25,3 @@ repos: rev: v1.0 hooks: - id: fmt - - id: clippy - args: ["--all-targets", "--all-features", "--", "-D", "warnings"] diff --git a/Cargo.lock b/Cargo.lock index 5e7ecfc7..84f55faf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,7 +66,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand", - "sha1", + "sha1 0.10.6", "smallvec", "tokio", "tokio-util", @@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -151,7 +151,7 @@ dependencies = [ "pin-project-lite", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tracing", ] @@ -218,14 +218,14 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -242,7 +242,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "generic-array", ] @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -323,9 +323,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -338,36 +338,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "flate2", "futures-core", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -413,13 +413,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -430,7 +430,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -465,9 +465,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.7" +version = "1.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" dependencies = [ "aws-credential-types", "aws-runtime", @@ -476,7 +476,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -507,21 +507,20 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.9.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3" dependencies = [ "aws-lc-sys", - "mirai-annotations", "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.21.2" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62" +checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7" dependencies = [ "bindgen", "cc", @@ -534,9 +533,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -560,11 +559,10 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.52.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f571deb0a80c20d21d9f3e8418c1712af9ff4bf399d057e5549a934eca4844e2" +checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" dependencies = [ - "ahash", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -572,7 +570,7 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -581,29 +579,29 @@ dependencies = [ "bytes", "fastrand", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http-body 0.4.6", "lru", "once_cell", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.10.8", "tracing", "url", ] [[package]] name = "aws-sdk-secretsmanager" -version = "1.49.0" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2ac19e43e100834e7b9e6f838af7506a5cb8ee7531d1104cb207b3d927e0c2" +checksum = "450b2e8cb5f0ee102e4a04c5f8e923aff8187ae9323058707c6cec238cf51699" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -618,15 +616,15 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.44.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -640,15 +638,15 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.45.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -662,15 +660,15 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.44.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -685,9 +683,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -698,14 +696,14 @@ dependencies = [ "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "once_cell", "p256", "percent-encoding", "ring", - "sha2", + "sha2 0.10.8", "subtle", "time", "tracing", @@ -714,9 +712,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "8aa8ff1492fd9fb99ae28e8467af0dbbb7c31512b16fabf1a0f10d7bb6ef78bb" dependencies = [ "futures-util", "pin-project-lite", @@ -725,9 +723,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.12" +version = "0.60.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -739,8 +737,8 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.8", "tracing", ] @@ -785,6 +783,15 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + [[package]] name = "aws-smithy-mocks-experimental" version = "0.2.1" @@ -797,21 +804,21 @@ dependencies = [ [[package]] name = "aws-smithy-protocol-test" -version = "0.62.0" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495c940cd5c7232ac3f0945ff559096deadd2fc73e4418a0e98fe5836788bb39" +checksum = "3b92b62199921f10685c6b588fdbeb81168ae4e7950ae3e5f50145a01bb5f1ad" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", "base64-simd", "cbor-diag", + "ciborium", "http 0.2.12", "pretty_assertions", "regex-lite", "roxmltree", - "serde_cbor", "serde_json", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -826,9 +833,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +checksum = "431a10d0e07e09091284ef04453dae4069283aa108d209974d67e77ae1caa658" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -842,9 +849,9 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "hyper 0.14.30", + "hyper 0.14.32", "hyper-rustls 0.24.2", - "indexmap 2.5.0", + "indexmap 2.7.0", "once_cell", "pin-project-lite", "pin-utils", @@ -858,15 +865,15 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "tokio", "tracing", @@ -875,16 +882,16 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +checksum = "8ecbf4d5dfb169812e2b240a4350f15ad3c6b03a54074e5712818801615f2dc5" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -940,7 +947,7 @@ checksum = "7319a086b79c3ff026a33a61e80f04fd3885fbb73237981ea080d21944e1cb1c" dependencies = [ "base64 0.22.1", "bytes", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-serde", "query_map", @@ -950,18 +957,18 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.2", "hyper-util", "itoa", "matchit", @@ -974,9 +981,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -991,13 +998,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -1005,25 +1012,27 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.4" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c3220b188aea709cf1b6c5f9b01c3bd936bb08bd2b5184a12b35ac8131b1f9" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" dependencies = [ "axum", "axum-core", "bytes", + "fastrand", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", + "multer", "pin-project-lite", "serde", "serde_json", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", - "tracing", + "typed-json", ] [[package]] @@ -1083,7 +1092,7 @@ checksum = "6aeac2e1fe888769f34f05ac343bbef98b14d1ffb292ab69d4608b3abc86f2a2" dependencies = [ "blowfish", "pbkdf2", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -1097,9 +1106,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags", "cexpr", @@ -1114,7 +1123,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.79", + "syn 2.0.90", "which", ] @@ -1136,7 +1145,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1148,6 +1157,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.11.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd016a0ddc7cb13661bf5576073ce07330a693f8608a1320b4e20561cc12cdc" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1199,9 +1217,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" dependencies = [ "memchr", "serde", @@ -1215,9 +1233,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -1227,9 +1245,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" dependencies = [ "serde", ] @@ -1246,9 +1264,9 @@ dependencies = [ [[package]] name = "bytestring" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f" dependencies = [ "bytes", ] @@ -1298,7 +1316,7 @@ dependencies = [ "bs58", "chrono", "data-encoding", - "half 2.4.1", + "half", "nom", "num-bigint", "num-rational", @@ -1310,9 +1328,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.22" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" dependencies = [ "jobserver", "libc", @@ -1334,6 +1352,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -1360,9 +1384,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1395,7 +1419,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 2.4.1", + "half", ] [[package]] @@ -1404,7 +1428,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "inout", "zeroize", ] @@ -1422,9 +1446,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -1432,9 +1456,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -1451,29 +1475,29 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" dependencies = [ "cc", ] [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "const-oid" @@ -1481,6 +1505,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ff6be19477a1bd5441f382916a89bc2a0b2c35db6d41e0f6e8538bf6d6463f" + [[package]] name = "convert_case" version = "0.4.0" @@ -1516,9 +1546,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -1581,18 +1611,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1609,9 +1639,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1645,7 +1675,7 @@ dependencies = [ "rpassword", "scrypt", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1681,6 +1711,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b8ce8218c97789f16356e7896b3714f26c2ee1079b79c0b7ae7064bb9089fa" +dependencies = [ + "getrandom", + "hybrid-array", + "rand_core", +] + [[package]] name = "crypto_kx" version = "0.2.1" @@ -1724,7 +1765,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1748,7 +1789,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1759,7 +1800,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1780,7 +1821,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -1804,7 +1845,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1819,11 +1860,34 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.0-pre.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2e3d6615d99707295a9673e889bf363a04b2a466bd320c65a72536f7577379" +dependencies = [ + "block-buffer 0.11.0-rc.3", + "const-oid 0.10.0-rc.3", + "crypto-common 0.2.0-rc.1", "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1863,7 +1927,7 @@ dependencies = [ "base16ct", "crypto-bigint 0.4.9", "der", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -1876,9 +1940,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -1904,12 +1968,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1926,9 +1990,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -1964,9 +2028,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -1978,6 +2042,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1995,9 +2065,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -2010,9 +2080,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -2020,15 +2090,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -2037,38 +2107,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -2099,15 +2169,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -2138,7 +2210,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2147,29 +2219,23 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.1.0", - "indexmap 2.5.0", + "http 1.2.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - [[package]] name = "half" version = "2.4.1" @@ -2188,12 +2254,13 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "ahash", "allocator-api2", + "equivalent", + "foldhash", ] [[package]] @@ -2202,12 +2269,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hermit-abi" version = "0.4.0" @@ -2236,16 +2297,25 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4b1fb14e4df79f9406b434b60acef9f45c26c50062cccf1346c6103b8c47d58" +dependencies = [ + "digest 0.11.0-pre.9", ] [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2264,10 +2334,10 @@ dependencies = [ "htsget-search", "htsget-test", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "reqwest", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.20", + "rustls-pemfile 2.2.0", "serde", "serde_json", "tempfile", @@ -2289,16 +2359,16 @@ dependencies = [ "htsget-http", "htsget-search", "htsget-test", - "http 1.1.0", - "hyper 1.4.1", + "http 1.2.0", + "hyper 1.5.2", "hyper-util", "reqwest", - "rustls 0.23.13", + "rustls 0.23.20", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.0", - "tower 0.5.1", + "tokio-rustls 0.26.1", + "tower 0.5.2", "tower-http", "tracing", ] @@ -2316,21 +2386,21 @@ dependencies = [ "crypt4gh", "figment", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-serde", "noodles", "rcgen", "regex", "reqwest", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.20", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_regex", "serde_with", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "toml", "tracing", @@ -2345,9 +2415,10 @@ dependencies = [ "htsget-config", "htsget-search", "htsget-test", - "http 1.1.0", + "http 1.2.0", "serde", - "thiserror", + "serde_json", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -2368,7 +2439,7 @@ dependencies = [ "mime", "query_map", "regex", - "rustls 0.23.13", + "rustls 0.23.20", "serde", "serde_json", "tempfile", @@ -2389,10 +2460,10 @@ dependencies = [ "htsget-config", "htsget-storage", "htsget-test", - "http 1.1.0", + "http 1.2.0", "noodles", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -2416,11 +2487,11 @@ dependencies = [ "futures-util", "htsget-config", "htsget-test", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "reqwest", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-util", "tower-http", @@ -2440,7 +2511,7 @@ dependencies = [ "crypt4gh", "futures", "htsget-config", - "http 1.1.0", + "http 1.2.0", "mime", "noodles", "rcgen", @@ -2451,7 +2522,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", ] @@ -2468,9 +2539,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -2495,7 +2566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -2506,16 +2577,16 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "http-range-header" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" [[package]] name = "http-serde" @@ -2523,15 +2594,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http 1.1.0", + "http 1.2.0", "serde", ] [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -2545,11 +2616,20 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hybrid-array" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d35805454dc9f8662a98d6d61886ffe26bd465f5960e0e55345c70d5c0d2a9" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -2571,15 +2651,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", - "http 1.1.0", + "h2 0.4.7", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", @@ -2598,7 +2678,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.30", + "hyper 0.14.32", "log", "rustls 0.21.12", "rustls-native-certs", @@ -2613,29 +2693,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.4.1", + "http 1.2.0", + "hyper 1.5.2", "hyper-util", - "rustls 0.23.13", + "rustls 0.23.20", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tower-service", "webpki-roots", ] [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -2666,6 +2746,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2674,19 +2872,30 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] name = "impl-more" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" [[package]] name = "indexmap" @@ -2701,12 +2910,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "serde", ] @@ -2728,9 +2937,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-terminal" @@ -2738,7 +2947,7 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi 0.4.0", + "hermit-abi", "libc", "windows-sys 0.52.0", ] @@ -2778,9 +2987,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -2793,10 +3002,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2812,10 +3022,10 @@ dependencies = [ "encoding_rs", "futures", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.2", "lambda_runtime", "mime", "percent-encoding", @@ -2837,11 +3047,11 @@ dependencies = [ "base64 0.22.1", "bytes", "futures", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "http-serde", - "hyper 1.4.1", + "hyper 1.5.2", "hyper-util", "lambda_runtime_api_client", "pin-project", @@ -2864,10 +3074,10 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.2", "hyper-util", "tokio", "tower 0.4.13", @@ -2896,9 +3106,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lexical-core" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2909,9 +3119,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -2920,9 +3130,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ "lexical-util", "static_assertions", @@ -2930,18 +3140,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" dependencies = [ "lexical-util", "lexical-write-integer", @@ -2950,9 +3160,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" dependencies = [ "lexical-util", "static_assertions", @@ -2960,15 +3170,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.159" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -2980,6 +3190,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + [[package]] name = "local-channel" version = "0.1.5" @@ -3015,11 +3231,11 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -3055,7 +3271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -3097,11 +3313,10 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi 0.3.9", "libc", "log", "wasi", @@ -3109,10 +3324,21 @@ dependencies = [ ] [[package]] -name = "mirai-annotations" -version = "1.12.0" +name = "multer" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.2.0", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] [[package]] name = "mutually_exclusive_features" @@ -3160,7 +3386,7 @@ dependencies = [ "byteorder", "bytes", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3176,7 +3402,7 @@ checksum = "e465d9d332ee4f5366cb1861aae94177767504f4a490277ed33a16ffb372b47e" dependencies = [ "byteorder", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3223,7 +3449,7 @@ dependencies = [ "bzip2", "flate2", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "md-5", "noodles-bam", "noodles-core", @@ -3242,7 +3468,7 @@ checksum = "a0f41004636fb4232155421cbf4706565073623838a8252875085fa670b8185c" dependencies = [ "bit-vec", "byteorder", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "tokio", @@ -3281,7 +3507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca42cb034ccc595d963de2ff2fd9cf2c3f563bcb65c1337e90f9d73724743d84" dependencies = [ "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3298,7 +3524,7 @@ dependencies = [ "bitflags", "bstr", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "lexical-core", "memchr", "noodles-bgzf", @@ -3314,7 +3540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5c5ed1fa0b9ae083c2e1e1cedc07715861400fdd943fdb1ed6c593719be187" dependencies = [ "byteorder", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3328,7 +3554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be568315676d38294c4b00bd608bd19d04394e6106c2db08c4b351b275741df" dependencies = [ "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "memchr", "noodles-bgzf", "noodles-core", @@ -3411,21 +3637,18 @@ checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -3465,7 +3688,7 @@ checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" dependencies = [ "ecdsa", "elliptic-curve", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -3532,8 +3755,8 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", - "hmac", + "digest 0.10.7", + "hmac 0.12.1", ] [[package]] @@ -3556,7 +3779,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -3577,29 +3800,29 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -3662,12 +3885,6 @@ dependencies = [ "universal-hash", ] -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - [[package]] name = "powerfmt" version = "0.2.0" @@ -3705,12 +3922,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -3739,9 +3956,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -3754,7 +3971,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", "version_check", "yansi", ] @@ -3772,9 +3989,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "memchr", "serde", @@ -3782,45 +3999,49 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustc-hash 2.1.0", + "rustls 0.23.20", "socket2", - "thiserror", + "thiserror 2.0.7", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", - "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustc-hash 2.1.0", + "rustls 0.23.20", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.7", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -3902,22 +4123,22 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -3932,9 +4153,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -3961,19 +4182,19 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.2", "hyper-rustls 0.27.3", "hyper-util", "ipnet", @@ -3984,15 +4205,15 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.20", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tower-service", "url", @@ -4011,7 +4232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ "crypto-bigint 0.4.9", - "hmac", + "hmac 0.12.1", "zeroize", ] @@ -4074,9 +4295,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc_version" @@ -4089,15 +4310,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4114,9 +4335,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "aws-lc-rs", "log", @@ -4151,19 +4372,21 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -4189,9 +4412,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -4201,9 +4424,8 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "s3s" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa54e3b4b4791c8c62291516997866b4f265c3fcbfdbcdd0b8da62896fba8bfa" +version = "0.11.0-dev" +source = "git+https://github.com/Nugine/s3s#787319c5a6c6df905dc8d1b4a140635fef75d42d" dependencies = [ "arrayvec", "async-trait", @@ -4214,14 +4436,14 @@ dependencies = [ "chrono", "crc32c", "crc32fast", - "digest", + "digest 0.11.0-pre.9", "futures", "hex-simd", - "hmac", + "hmac 0.13.0-pre.4", "http-body 1.0.1", "http-body-util", "httparse", - "hyper 1.4.1", + "hyper 1.5.2", "itoa", "memchr", "mime", @@ -4232,11 +4454,12 @@ dependencies = [ "quick-xml", "serde", "serde_urlencoded", - "sha1", - "sha2", + "sha1 0.11.0-pre.4", + "sha2 0.11.0-pre.4", "smallvec", - "sync_wrapper 1.0.1", - "thiserror", + "std-next", + "sync_wrapper", + "thiserror 2.0.7", "time", "tokio", "tracing", @@ -4247,27 +4470,25 @@ dependencies = [ [[package]] name = "s3s-aws" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13db1822175997bba0d1e398a772085b75791aa14e793819bec728ed8b394ff2" +version = "0.11.0-dev" +source = "git+https://github.com/Nugine/s3s#787319c5a6c6df905dc8d1b4a140635fef75d42d" dependencies = [ "async-trait", "aws-sdk-s3", "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-types-convert", - "hyper 1.4.1", + "hyper 1.5.2", "s3s", - "sync_wrapper 1.0.1", + "sync_wrapper", "tracing", "transform-stream", ] [[package]] name = "s3s-fs" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b045f5ab67e8536d147ae48e038f6f533717fe5ddb0a68d0ba8f922ce7fb293f" +version = "0.11.0-dev" +source = "git+https://github.com/Nugine/s3s#787319c5a6c6df905dc8d1b4a140635fef75d42d" dependencies = [ "async-trait", "base64-simd", @@ -4278,12 +4499,12 @@ dependencies = [ "hex-simd", "md-5", "mime", - "nugine-rust-utils", "numeric_cast", "path-absolutize", "s3s", "serde_json", - "thiserror", + "std-next", + "thiserror 2.0.7", "time", "tokio", "tokio-util", @@ -4313,9 +4534,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -4335,7 +4556,7 @@ dependencies = [ "password-hash", "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -4377,9 +4598,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -4387,9 +4608,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "separator" @@ -4399,41 +4620,31 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half 1.8.3", - "serde", -] - [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.7.0", "itoa", "memchr", "ryu", @@ -4483,15 +4694,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.7.0", "serde", "serde_derive", "serde_json", @@ -4501,14 +4712,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -4519,7 +4730,18 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9540978cef7a8498211c1b1c14e5ce920fe5bd524ea84f4a3d72d4602515ae93" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.0-pre.9", ] [[package]] @@ -4530,7 +4752,18 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "540c0893cce56cdbcfebcec191ec8e0f470dd1889b6e7a0b503e310a94a168f5" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.0-pre.9", ] [[package]] @@ -4563,7 +4796,7 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest", + "digest 0.10.7", "rand_core", ] @@ -4590,9 +4823,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4614,12 +4847,28 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "std-next" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87854dde78837ff867561b22d873c53c25938d4b152afba2fa6fc48cb406c099" +dependencies = [ + "simdutf8", + "thiserror 2.0.7", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4645,9 +4894,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -4656,24 +4905,29 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] -name = "sync_wrapper" -version = "1.0.1" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ - "futures-core", + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -4693,22 +4947,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.7", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -4723,9 +4997,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -4744,14 +5018,24 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4779,9 +5063,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -4803,7 +5087,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -4818,20 +5102,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.13", - "rustls-pki-types", + "rustls 0.23.20", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -4840,9 +5123,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -4879,7 +5162,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -4903,14 +5186,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -4919,14 +5202,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "http-range-header", @@ -4956,9 +5239,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -4968,9 +5251,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15bc0cd5f72e837e310f4d978a90abf202a7f7d8ef3272246bae381d0086d3bf" +checksum = "54a9f5c1aca50ebebf074ee665b9f99f2e84906dcf6b993a0d0090edb835166d" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -4981,20 +5264,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -5002,9 +5285,9 @@ dependencies = [ [[package]] name = "tracing-error" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" dependencies = [ "tracing", "tracing-subscriber", @@ -5023,9 +5306,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ "serde", "tracing-core", @@ -5033,9 +5316,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -5067,6 +5350,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6024a8d0025400b3f6b189366e9aa92012cf9c4fe1cd2620848dd61425c49eed" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "typenum" version = "1.17.0" @@ -5084,33 +5377,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.15" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-ident" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" - -[[package]] -name = "unicode-normalization" -version = "0.1.24" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "universal-hash" @@ -5118,7 +5393,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "subtle", ] @@ -5130,9 +5405,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -5145,6 +5420,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -5153,9 +5440,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", ] @@ -5205,9 +5492,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -5216,36 +5503,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5253,28 +5540,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -5285,9 +5572,19 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -5295,9 +5592,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -5541,6 +5838,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xmlparser" version = "0.13.6" @@ -5571,6 +5880,30 @@ dependencies = [ "time", ] +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -5589,7 +5922,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", + "synstructure", ] [[package]] @@ -5598,6 +5952,28 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/README.md b/README.md index ba3bc122..5e1e990f 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Other directories contain further applications or data: - [deploy]: Deployments for htsget-rs. [axum]: https://github.com/tokio-rs/axum +[htsget-axum]: htsget-axum [htsget-config]: htsget-config [htsget-actix]: htsget-actix [htsget-http]: htsget-http diff --git a/htsget-actix/README.md b/htsget-actix/README.md index c68665fb..826c5221 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -53,8 +53,8 @@ are exposed in the public API. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Benchmarks diff --git a/htsget-actix/benches/request_benchmarks.rs b/htsget-actix/benches/request_benchmarks.rs index 1dd6b5d9..60af6b82 100644 --- a/htsget-actix/benches/request_benchmarks.rs +++ b/htsget-actix/benches/request_benchmarks.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use htsget_config::types::{Headers, JsonResponse}; use htsget_http::{PostRequest, Region}; use htsget_test::http::default_config_fixed_port; -use htsget_test::util::{default_dir, default_dir_data}; +use htsget_test::util::default_dir; const REFSERVER_DOCKER_IMAGE: &str = "ga4gh/htsget-refserver:1.5.0"; const BENCHMARK_DURATION_SECONDS: u64 = 30; @@ -144,14 +144,14 @@ fn start_htsget_rs() -> (DropGuard, String) { .arg("-p") .arg("htsget-actix") .arg("--no-default-features") - .env("HTSGET_PATH", default_dir_data()) - .env("RUST_LOG", "warn") .spawn() .unwrap(); let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let htsget_rs_ticket_url = format!("http://{}", config.data_server().addr()); + + let data_server = config.data_server().as_data_server_config().unwrap(); + let htsget_rs_ticket_url = format!("http://{}", data_server.addr()); query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); (DropGuard(child), htsget_rs_url) diff --git a/htsget-actix/src/handlers/service_info.rs b/htsget-actix/src/handlers/service_info.rs index 8df7c6dc..f69e2a6b 100644 --- a/htsget-actix/src/handlers/service_info.rs +++ b/htsget-actix/src/handlers/service_info.rs @@ -21,7 +21,7 @@ pub fn get_service_info_json( PrettyJson(get_base_service_info_json( endpoint, app_state.htsget.clone(), - &app_state.config_service_info, + app_state.config_service_info.clone(), )) } diff --git a/htsget-actix/src/lib.rs b/htsget-actix/src/lib.rs index 2d7efb2b..02a37c63 100644 --- a/htsget-actix/src/lib.rs +++ b/htsget-actix/src/lib.rs @@ -5,9 +5,10 @@ use tracing::info; use tracing::instrument; use tracing_actix_web::TracingLogger; -use htsget_config::config::cors::CorsConfig; -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig, USAGE}; -pub use htsget_config::storage::Storage; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::service_info::ServiceInfo; +use htsget_config::config::ticket_server::TicketServerConfig; +pub use htsget_config::config::{Config, USAGE}; use htsget_search::HtsGet; use crate::handlers::{get, post, reads_service_info, variants_service_info, HttpVersionCompat}; @@ -214,14 +215,17 @@ mod tests { #[async_trait(?Send)] impl TestServer> for ActixTestServer { async fn get_expected_path(&self) -> String { - let mut bind_data_server = BindServer::from(self.get_config().data_server().clone()); - let server = bind_data_server - .bind_data_server("/data".to_string()) - .await + let data_server = self + .get_config() + .data_server() + .as_data_server_config() .unwrap(); + + let path = data_server.local_path().to_path_buf(); + let mut bind_data_server = BindServer::from(data_server.clone()); + let server = bind_data_server.bind_data_server().await.unwrap(); let addr = server.local_addr(); - let path = self.get_config().data_server().local_path().to_path_buf(); tokio::spawn(async move { server.serve(path).await.unwrap() }); expected_url_path(self.get_config(), addr.unwrap()) @@ -278,7 +282,7 @@ mod tests { .configure(|service_config: &mut web::ServiceConfig| { configure_server( service_config, - self.config.clone().owned_resolvers(), + self.config.clone().into_locations(), self.config.service_info().clone(), ); }) diff --git a/htsget-actix/src/main.rs b/htsget-actix/src/main.rs index 7a904370..6695e662 100644 --- a/htsget-actix/src/main.rs +++ b/htsget-actix/src/main.rs @@ -7,6 +7,7 @@ use htsget_actix::run_server; use htsget_actix::Config; use htsget_axum::server::data; use htsget_config::command; +use htsget_config::config::data_server::DataServerEnabled; #[actix_web::main] async fn main() -> io::Result<()> { @@ -21,8 +22,8 @@ async fn main() -> io::Result<()> { debug!(config = ?config, "config parsed"); - if config.data_server().enabled() { - let local_server = data::join_handle(config.data_server().clone()).await?; + if let DataServerEnabled::Some(data_server) = config.data_server() { + let local_server = data::join_handle(data_server.clone()).await?; let ticket_server_config = config.ticket_server().clone(); let service_info = config.service_info().clone(); @@ -30,7 +31,7 @@ async fn main() -> io::Result<()> { select! { local_server = local_server => Ok(local_server??), actix_server = run_server( - config.owned_resolvers(), + config.into_locations(), ticket_server_config, service_info )? => actix_server @@ -39,7 +40,7 @@ async fn main() -> io::Result<()> { let ticket_server_config = config.ticket_server().clone(); let service_info = config.service_info().clone(); - run_server(config.owned_resolvers(), ticket_server_config, service_info)?.await + run_server(config.into_locations(), ticket_server_config, service_info)?.await } } else { Ok(()) diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 0311e9a7..886e3f1d 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -178,8 +178,8 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-axum/src/handlers/service_info.rs b/htsget-axum/src/handlers/service_info.rs index 737203ba..89dc7716 100644 --- a/htsget-axum/src/handlers/service_info.rs +++ b/htsget-axum/src/handlers/service_info.rs @@ -16,7 +16,7 @@ pub fn get_service_info_json( ErasedJson::pretty(get_base_service_info_json( endpoint, app_state.htsget, - &app_state.service_info, + app_state.service_info, )) } diff --git a/htsget-axum/src/main.rs b/htsget-axum/src/main.rs index 291d861f..bac6a3e9 100644 --- a/htsget-axum/src/main.rs +++ b/htsget-axum/src/main.rs @@ -5,6 +5,7 @@ use tracing::debug; use htsget_axum::server::{data, ticket}; use htsget_config::command; +use htsget_config::config::data_server::DataServerEnabled; use htsget_config::config::Config; #[tokio::main] @@ -22,8 +23,8 @@ async fn main() -> io::Result<()> { debug!(config = ?config, "config parsed"); - if config.data_server().enabled() { - let local_server = data::join_handle(config.data_server().clone()).await?; + if let DataServerEnabled::Some(data_server) = config.data_server() { + let local_server = data::join_handle(data_server.clone()).await?; let ticket_server = ticket::join_handle(config).await?; select! { diff --git a/htsget-axum/src/server/data.rs b/htsget-axum/src/server/data.rs index ea269ee5..fd4240c4 100644 --- a/htsget-axum/src/server/data.rs +++ b/htsget-axum/src/server/data.rs @@ -4,8 +4,8 @@ use crate::error::Result; use crate::server::{configure_cors, BindServer, Server}; use axum::Router; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::DataServerConfig; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::data_server::DataServerConfig; use std::net::SocketAddr; use std::path::Path; use tokio::task::JoinHandle; @@ -17,32 +17,24 @@ use tracing::info; #[derive(Debug)] pub struct DataServer { server: Server, - serve_at: String, cors: CorsConfig, } impl DataServer { /// Create a new data server. - pub fn new(server: Server, serve_at: String, cors: CorsConfig) -> Self { - Self { - server, - serve_at, - cors, - } + pub fn new(server: Server, cors: CorsConfig) -> Self { + Self { server, cors } } /// Run the data server, using the provided path, key and certificate. pub async fn serve>(self, path: P) -> Result<()> { - self - .server - .serve(Self::router(self.cors, &self.serve_at, path)) - .await + self.server.serve(Self::router(self.cors, path)).await } /// Create the router for the data server. - pub fn router>(cors: CorsConfig, serve_at: &str, path: P) -> Router { + pub fn router>(cors: CorsConfig, path: P) -> Router { Router::new() - .nest_service(serve_at, ServeDir::new(path)) + .nest_service("/", ServeDir::new(path)) .layer(configure_cors(cors)) .layer(TraceLayer::new_for_http()) } @@ -69,11 +61,8 @@ impl From for BindServer { /// Spawn a task to run the data server. pub async fn join_handle(config: DataServerConfig) -> Result>> { - let serve_at = config.serve_at().to_string(); let local_path = config.local_path().to_path_buf(); - let data_server = BindServer::from(config.clone()) - .bind_data_server(serve_at) - .await?; + let data_server = BindServer::from(config.clone()).bind_data_server().await?; info!(address = ?data_server.local_addr()?, "data server address bound to"); @@ -213,8 +202,12 @@ mod tests { let _ = aws_lc_rs::default_provider().install_default(); let (_, base_path) = create_local_test_files().await; - let config = config_with_tls(base_path.path()).data_server().clone(); - let server_config = config.into_tls().unwrap(); + let data_server = config_with_tls(base_path.path()) + .data_server() + .as_data_server_config() + .unwrap() + .clone(); + let server_config = data_server.into_tls().unwrap(); test_server("https", Some(server_config), base_path.path().to_path_buf()).await; } @@ -245,7 +238,7 @@ mod tests { test_cors_simple_request_uri( &DataTestServer::default(), - &format!("http://localhost:{port}/data/key1"), + &format!("http://localhost:{port}/key1"), ) .await; } @@ -258,7 +251,7 @@ mod tests { test_cors_preflight_request_uri( &DataTestServer::default(), - &format!("http://localhost:{port}/data/key1"), + &format!("http://localhost:{port}/key1"), ) .await; } @@ -267,8 +260,12 @@ mod tests { let _ = aws_lc_rs::default_provider().install_default(); let tmp_dir = tempdir().unwrap(); - let config = config_with_tls(tmp_dir.path()).data_server().clone(); - let server_config = config.into_tls().unwrap(); + let data_server = config_with_tls(tmp_dir.path()) + .data_server() + .as_data_server_config() + .unwrap() + .clone(); + let server_config = data_server.clone().into_tls().unwrap(); BindServer::new_with_tls( "127.0.0.1:8080".parse().unwrap(), @@ -285,7 +282,7 @@ mod tests { let server = Server::bind_addr(addr, cert_key_pair).await.unwrap(); let port = server.local_addr().unwrap().port(); - let data_server = DataServer::new(server, "/data".to_string(), default_cors_config()); + let data_server = DataServer::new(server, default_cors_config()); tokio::spawn(async move { data_server.serve(path).await.unwrap() }); port @@ -301,7 +298,7 @@ mod tests { let request = test_server .request() .method(Method::GET) - .uri(format!("{scheme}://localhost:{port}/data/key1")); + .uri(format!("{scheme}://localhost:{port}/key1")); let response = test_server.test_server(request, "".to_string()).await; assert!(response.is_success()); diff --git a/htsget-axum/src/server/mod.rs b/htsget-axum/src/server/mod.rs index 3fbe7290..eabacce3 100644 --- a/htsget-axum/src/server/mod.rs +++ b/htsget-axum/src/server/mod.rs @@ -10,6 +10,11 @@ use std::time::Duration; use axum::extract::Request; use axum::Router; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::service_info::ServiceInfo; +use htsget_config::tls::TlsServerConfig; +use htsget_config::types::Scheme; +use htsget_search::HtsGet; use http::HeaderValue; use hyper::body::Incoming; use hyper::service::service_fn; @@ -22,12 +27,6 @@ use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, CorsLayer, Expos use tracing::trace; use tracing::{error, warn}; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::ServiceInfo; -use htsget_config::tls::TlsServerConfig; -use htsget_config::types::Scheme; -use htsget_search::HtsGet; - use crate::error::Error::ServerError; use crate::error::Result; use crate::server::data::DataServer; @@ -79,6 +78,10 @@ pub fn configure_cors(cors: CorsConfig) -> CorsLayer { |cors_layer| cors_layer.allow_headers(AllowHeaders::mirror_request()), cors_layer, ); + cors_layer = cors.allow_origins().apply_mirror( + |cors_layer| cors_layer.allow_headers(AllowHeaders::mirror_request()), + cors_layer, + ); cors_layer = cors.allow_headers().apply_list( |cors_layer, headers| cors_layer.allow_headers(headers.clone()), cors_layer, @@ -88,6 +91,10 @@ pub fn configure_cors(cors: CorsConfig) -> CorsLayer { |cors_layer| cors_layer.allow_methods(AllowMethods::mirror_request()), cors_layer, ); + cors_layer = cors.allow_origins().apply_mirror( + |cors_layer| cors_layer.allow_methods(AllowMethods::mirror_request()), + cors_layer, + ); cors_layer = cors.allow_methods().apply_list( |cors_layer, methods| cors_layer.allow_methods(methods.clone()), cors_layer, @@ -150,10 +157,10 @@ impl BindServer { } /// Eagerly bind the address by returning a `DataServer`. - pub async fn bind_data_server(&mut self, serve_at: String) -> Result { + pub async fn bind_data_server(&mut self) -> Result { let server = self.bind_server().await?; - Ok(DataServer::new(server, serve_at, self.cors.clone())) + Ok(DataServer::new(server, self.cors.clone())) } /// Eagerly bind the address by returning a `TicketServer`. @@ -216,7 +223,7 @@ impl Server { let tls_acceptor = tls_acceptor.clone(); trace!("accepting connection"); - let (cnx, addr) = self.listener.accept().await.unwrap(); + let (cnx, addr) = self.listener.accept().await?; tokio::spawn(async move { let Ok(stream) = tls_acceptor.accept(cnx).await else { diff --git a/htsget-axum/src/server/ticket.rs b/htsget-axum/src/server/ticket.rs index 7f965cf5..211759f1 100644 --- a/htsget-axum/src/server/ticket.rs +++ b/htsget-axum/src/server/ticket.rs @@ -6,8 +6,10 @@ use crate::handlers::{get, post, reads_service_info, variants_service_info}; use crate::server::{configure_cors, AppState, BindServer, Server}; use axum::routing::get; use axum::Router; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::{Config, ServiceInfo, TicketServerConfig}; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::service_info::ServiceInfo; +use htsget_config::config::ticket_server::TicketServerConfig; +use htsget_config::config::Config; use htsget_search::HtsGet; use std::net::SocketAddr; use tokio::task::JoinHandle; @@ -91,7 +93,7 @@ where pub async fn join_handle(config: Config) -> Result>> { let service_info = config.service_info().clone(); let ticket_server = BindServer::from(config.ticket_server().clone()) - .bind_ticket_server(config.owned_resolvers(), service_info) + .bind_ticket_server(config.into_locations(), service_info) .await?; info!(address = ?ticket_server.local_addr()?, "ticket server address bound to"); @@ -168,14 +170,17 @@ mod tests { #[async_trait(?Send)] impl TestServer>> for AxumTestServer { async fn get_expected_path(&self) -> String { - let mut bind_data_server = BindServer::from(self.get_config().data_server().clone()); - let server = bind_data_server - .bind_data_server("/data".to_string()) - .await + let data_server = self + .get_config() + .data_server() + .as_data_server_config() .unwrap(); + + let path = data_server.local_path().to_path_buf(); + let mut bind_data_server = BindServer::from(data_server.clone()); + let server = bind_data_server.bind_data_server().await.unwrap(); let addr = server.local_addr(); - let path = self.get_config().data_server().local_path().to_path_buf(); tokio::spawn(async move { server.serve(path).await.unwrap() }); expected_url_path(self.get_config(), addr.unwrap()) @@ -218,7 +223,7 @@ mod tests { async fn get_response(&self, request: Request) -> result::Result { let app = TicketServer::router( - self.config.clone().owned_resolvers(), + self.config.clone().into_locations(), self.config.service_info().clone(), self.config.ticket_server().cors().clone(), ); diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 00e7f3ba..c12cf737 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -22,6 +22,7 @@ async-trait = "0.1" noodles = { version = "0.83", features = ["core"] } serde = { version = "1", features = ["derive"] } serde_with = "3" +serde_json = "1" serde_regex = "1" regex = "1" figment = { version = "0.10", features = ["env", "toml"] } @@ -35,7 +36,7 @@ rustls-pemfile = "2" rustls = "0.23" rustls-pki-types = "1" -# url-storage +# url reqwest = { version = "0.12", features = ["rustls-tls"], default-features = false, optional = true } cfg-if = { version = "1", optional = true } @@ -50,7 +51,6 @@ aws-config = { version = "1", optional = true } tempfile = { version = "3", optional = true } [dev-dependencies] -serde_json = "1" figment = { version = "0.10", features = ["test"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } tempfile = "3" diff --git a/htsget-config/README.md b/htsget-config/README.md index 4f9ec7d2..560448f5 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -8,142 +8,145 @@ [actions-badge]: https://github.com/umccr/htsget-rs/actions/workflows/action.yml/badge.svg [actions-url]: https://github.com/umccr/htsget-rs/actions?query=workflow%3Atests+branch%3Amain +## Overview + Configuration for [htsget-rs]. [htsget-rs]: https://github.com/umccr/htsget-rs -## Overview +## Quickstart +The simplest way to use htsget-rs is to create a [toml] config file and specify a storage location: -This crate is used to configure htsget-rs using a config file or environment variables. +```toml +locations = "file://data" +``` -## Usage +Then launch the server using the config file: -To configure htsget-rs, a TOML config file can be defined. There is also support for reading config from environment variables. -Any config options set by environment variables override values in the config file. +```sh +cargo run --all-features -p htsget-axum -- --config +``` -The configuration consists of TOML tables, such as config for the ticket server, data server, service-info, or resolvers. +This will serve files under the [`data`][data] directory: -As a starting point, see the [basic TOML][basic] example file which should work for many use-cases. +```sh +curl 'http://localhost:8080/reads/bam/htsnexus_test_NA12878' +``` -#### Ticket server config +Locations allow htsget-rs access to bioinformatics files and indexes. Instead of local files, htsget-rs can access +files on s3, which returns pre-signed URLs for tickets: -The ticket server responds to htsget requests by returning a set of URL tickets that the client must fetch and concatenate. -To configure the ticket server, set the following options: +```toml +locations = "s3://bucket" +``` -| Option | Description | Type | Default | -|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|-----------------------------| -| `ticket_server_addr` | The address for the ticket server. | Socket address | `'127.0.0.1:8080'` | -| `ticket_server_tls` | Enable TLS for the ticket server. See [TLS](#tls) for more details. | TOML table | Not enabled | -| `ticket_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the ticket server. | Boolean | `false` | -| `ticket_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the ticket server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | -| `ticket_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the ticket server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | -| `ticket_server_cors_allow_methods` | Set the CORS Access-Control-Allow-Methods returned by the ticket server, this can be set to `All` to allow all methods, or a specific array of methods. | `'All'`, or a array of methods | `'All'` | -| `ticket_server_cors_max_age` | Set the CORS Access-Control-Max-Age for the ticket server which controls how long a preflight request can be cached for. | Seconds | `86400` | -| `ticket_server_cors_expose_headers` | Set the CORS Access-Control-Expose-Headers returned by the ticket server, this can be set to `All` to expose all headers, or a specific array of headers. | `'All'`, or a array of headers | `[]` | +or on a remote HTTP server (either `http://` or `https://`): -TLS is supported by setting the `ticket_server_key` and `ticket_server_cert` options. An example of config for the ticket server: ```toml -ticket_server_addr = '127.0.0.1:8080' -ticket_server_cors_allow_credentials = false -ticket_server_cors_allow_origins = 'Mirror' -ticket_server_cors_allow_headers = ['Content-Type'] -ticket_server_cors_allow_methods = ['GET', 'POST'] -ticket_server_cors_max_age = 86400 -ticket_server_cors_expose_headers = [] +locations = "https://example.com" ``` -#### Data server config - -The local data server responds to tickets produced by the ticket server by serving local filesystem data. -To configure the data server, set the following options: - -| Option | Description | Type | Default | -|-------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|-----------------------------| -| `data_server_addr` | The address for the data server. | Socket address | `'127.0.0.1:8081'` | -| `data_server_local_path` | The local path which the data server can access to serve files. | Filesystem path | `'./'` | -| `data_server_serve_at` | The path which the data server will prefix to all response URLs for tickets. | URL path | `''` | -| `data_server_tls` | Enable TLS for the data server. See [TLS](#tls) for more details. | TOML table | Not enabled | -| `data_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the data server. | Boolean | `false` | -| `data_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the data server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | -| `data_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the data server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | -| `data_server_cors_allow_methods` | Set the CORS Access-Control-Allow-Methods returned by the data server, this can be set to `All` to allow all methods, or a specific array of methods. | `'All'`, or a array of methods | `'All'` | -| `data_server_cors_max_age` | Set the CORS Access-Control-Max-Age for the data server which controls how long a preflight request can be cached for. | Seconds | `86400` | -| `data_server_cors_expose_headers` | Set the CORS Access-Control-Expose-Headers returned by the data server, this can be set to `All` to expose all headers, or a specific array of headers. | `'All'`, or a array of headers | `[]` | - -TLS is supported by setting the `data_server_key` and `data_server_cert` options. An example of config for the data server: +Multiple locations can be specified by providing a list and an id prefix after the location: + ```toml -data_server_addr = '127.0.0.1:8081' -data_server_local_path = './' -data_server_serve_at = '' -data_server_key = 'key.pem' -data_server_cert = 'cert.pem' -data_server_cors_allow_credentials = false -data_server_cors_allow_origins = 'Mirror' -data_server_cors_allow_headers = ['Content-Type'] -data_server_cors_allow_methods = ['GET', 'POST'] -data_server_cors_max_age = 86400 -data_server_cors_expose_headers = [] +locations = ["file://data/bam", "file://data/cram"] ``` -Sometimes it may be useful to disable the data server as all responses to the ticket server will be handled elsewhere, such as with an AWS S3 data server. +This allows htsget-rs to serve data only when the request also contains the prefix: -To disable the data server, set the following option: +```sh +curl 'http://localhost:8080/reads/bam/htsnexus_test_NA12878' +curl 'http://localhost:8080/reads/cram/htsnexus_test_NA12878?format=CRAM' +``` -
-data_server_enabled = false
-
+Locations can be mixed, and don't all need to have the same directory or resource: -#### Service info config +```toml +data_server.local_path = "root" +locations = ["file://dir_two/bam", "file://dir_one/cram", "s3://bucket/vcf"] +``` -The service info config controls what is returned when the [`service-info`][service-info] path is queried.
-To configure the service-info, set the following options: +htsget-rs spawns a separate server process to respond to htsget tickets for file locations, +so setting `data_server.local_path` to the root directory which contains all subdirectories is +required to give this server access to the local directory. -| Option | Description | Type | Default | -|---------------------------------------------------------|---------------------------------------------|-----------|----------| -| `id` | Service ID. | String | Not set | -| `name` | Service name. | String | Not set | -| `version` | Service version. | String | Not set | -| `organization_name` | Organization name. | String | Not set | -| `organization_url` | Organization URL. | String | Not set | -| `contact_url` | Service contact URL | String | Not set | -| `documentation_url` | Service documentation URL. | String | Not set | -| `created_at` | When the service was created. | String | Not set | -| `updated_at` | When the service was last updated. | String | Not set | -| `environment` | The environment the service is running in. | String | Not set | +The data server process can be disabled by setting it to `None` if no file locations are being used: -An example of config for the service info: ```toml -id = 'id' -name = 'name' -version = '0.1' -organization_name = 'name' -organization_url = 'https://example.com/' -contact_url = 'mailto:nobody@example.com' -documentation_url = 'https://example.com/' -created_at = '2022-01-01T12:00:00Z' -updated_at = '2022-01-01T12:00:00Z' -environment = 'dev' +data_server = "None" ``` -#### Resolvers +> [!NOTE] +> For S3 locations, the bucket is not included in the request to htsget-rs. To include the bucket as well, +> see deriving the bucket from the first capture group in [advanced config](#bucket). + +> [!IMPORTANT] +> Some parts of htsget-rs require extra feature flags for conditional compilation, that's why the examples specify +> using `--all-features`. Notably, `--features s3-storage` enables the `S3` location type, and `--features url-storage` +> enabled the remote HTTP server location type. If using a subset of features, for example S3 locations only, then +> a single feature can be enabled instead of using `--all-features`. -The resolvers component of htsget-rs is used to map query IDs to the location of the resource. This is the component of the -code that takes the [`id`][id], which is everything after `reads/` or `variants/` in the http path, and maps it to a data location. +### Server config -For example, if the request to htsget-rs is: +htsget-rs spawn up to two server instances - the ticket server responds to the initial htsget request, and optionally, +the data server, which responds to the htsget tickets. -```sh -curl 'http://localhost:8080/reads/some_id/file' +The socket address of the servers can be changed by specifying `addr`: + +```toml +ticket_server.addr = "127.0.0.1:8000" +data_server.addr = "127.0.0.1:8001" ``` -Then the resolvers controls how the server finds `some_id/file`, which may be stored locally, in the cloud, or at an arbitrary URL location. -The resolvers maps `some_id/file` to a location using regexes and substitution strings. The location of the file does not -need to have the same name as the id. +TLS can be configured to enabled HTTPS support by providing a certificate and private key: -A query ID is matched with a regex, and is then mapped with a substitution string that has access to the regex capture groups. -Resolvers are configured in an array, where the first matching resolver is resolver used to map the ID. +```toml +ticket_server.tls.key = "key.pem" +ticket_server.tls.cert = "cert.pem" + +data_server.tls.key = "key.pem" +data_server.tls.cert = "cert.pem" +``` + +### Service info config + +The service info config controls what is returned when the [`service-info`][service-info] path is queried. The following +option accepts any nested value, which gets converted to a JSON response: + +```toml +service_info.environment = "dev" +service_info.organization = { name = "name", url = "https://example.com/" } +``` + +### Environment variables -To create a resolver, add a `[[resolvers]]` array of tables, and set the following options: +Most options can also be set using environment variables. Any environment variables will override options set in the +config file. Arrays are delimited with `[` and `]`, and items are separated by commas: + +| Variable | Description | Example | +|---------------------------------|----------------------------------------------------------------|----------------------------------------------------| +| `HTSGET_TICKET_SERVER_ADDR` | Set the ticket server socket address. | "127.0.0.1:8080" | +| `HTSGET_TICKET_SERVER_TLS_KEY` | See [server config](#server-config) | "key.pem" | +| `HTSGET_TICKET_SERVER_TLS_CERT` | See [server config](#server-config) | "cert.pem" | +| `HTSGET_DATA_SERVER_ADDR` | Set the data server socket address. | "127.0.0.1:8081" | +| `HTSGET_DATA_SERVER_LOCAL_PATH` | Set the path that the data server has access to. | "dir/path" | +| `HTSGET_DATA_SERVER_TLS_KEY` | See [server config](#server-config) | "key.pem" | +| `HTSGET_DATA_SERVER_TLS_CERT` | See `server config](#server-config) | "cert.pem" | +| `HTSGET_SERVICE_INFO` | Set the service info, see [service info](#service-info-config) | "{ organization = { name = name, url = url }}" | +| `HTSGET_LOCATIONS` | Set the locations. | "[file://data/prefix_one, s3://bucket/prefix_two]" | +| `HTSGET_CONFIG` | Set the config file location. | "dir/config.toml" | + +## Advanced config + +The following section describes advanced configuration which is more flexible, but adds complexity. + +### Regex-based location + +Instead of the simple path-based locations described above, htsget-rs supports arbitrary regex-based id resolution. +This allows matching an [`id`][id], which is everything after `reads/` or `variants/` in the http path, and mapping +it to a location using regex substitution. + +To create a regex location, add a `[[locations]]` array of tables, and set the following options: | Option | Description | Type | Default | |-----------------------|-------------------------------------------------------------------------------------------------------------------------|---------------------------------------|---------| @@ -151,106 +154,114 @@ To create a resolver, add a `[[resolvers]]` array of tables, and set the followi | `substitution_string` | The replacement expression used to map the matched query ID. This has access to the match groups in the `regex` option. | String with access to capture groups | `'$0'` | For example, below is a `regex` option which matches a `/` between two groups, and inserts an additional `data` -in between the groups with the `substitution_string`. +in between the groups with the `substitution_string`: ```toml -[[resolvers]] +[[locations]] regex = '(?P.*?)/(?P.*)' substitution_string = '$group1/data/$group2' ``` -This would mean that a request to `http://localhost:8080/reads/some_id/file` would search for files at `some_id/data/file.bam` and `some_id/data/file.bam.bai`. +This would mean that a request to `http://localhost:8080/reads/some_id/file` would search for files at `some_id/data/file.bam`. -For more information about regex options see the [regex crate](https://docs.rs/regex/). +The regex locations also have access to further configuration of storage locations for `file://`, `s3://`, or `http://` +locations. These are called `File`, `S3`, and `Url` respectively. -Each resolver also maps to a certain storage backend. This storage backend can be used to set query IDs which are served from local storage, from S3-style bucket storage, or from HTTP URLs. -To set the storage backend for a resolver, add a `[resolvers.storage]` table. Some storage backends require feature flags to be set when compiling htsget-rs. +To manually configure `File` locations, set `backend.kind = "File"`, and specify any additional options from below the `backend` table: -To use `LocalStorage`, set `backend = 'Local'` under `[resolvers.storage]`, and specify any additional options from below: +| Option | Description | Type | Default | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| +| `scheme` | The scheme present on URL tickets. | Either `'Http'` or `'Https'` | `'Http'` | +| `authority` | The authority present on URL tickets. This should likely match the `data_server.addr`. | URL authority | `'127.0.0.1:8081'` | +| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server.local_path`. | Filesystem path | `'./'` | -| Option | Description | Type | Default | -|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| -| `scheme` | The scheme present on URL tickets. | Either `'Http'` or `'Https'` | `'Http'` | -| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | URL authority | `'127.0.0.1:8081'` | -| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Filesystem path | `'./'` | -| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `''` | -| `use_data_server_config` | Whether to use the data server config to fill in the above values. This overrides any other options specified from this table. | Boolean | `false` | - -By default, if the above options are left unspecified, they inherit values from the [`data_server`][data-server] config. -For example, the following sets the `scheme`, `authority`, `local_path` and `path_prefix` to values used by the `data_server`. +For example: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' +data_server.addr = "127.0.0.1:8000" -[resolvers.storage] -backend = 'Local' +[[locations]] +regex = ".*" +substitution_string = "$0" + +backend.kind = "Local" +backend.scheme = "Http" +backend.authority = "127.0.0.1:8000" +backend.local_path = "path" ``` -To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `backend = 'S3'` under `[resolvers.storage]`, and specify: +To manually configure `S3` locations, set `backend.kind = "S3"`, and specify options from below under the `backend` table: + +| Option | Description | Type | Default | +|------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------| +| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `location` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. | +| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. | +| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | -| Option | Description | Type | Default | -|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------| -| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `resolvers` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. | -| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. | -| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | +For example, the following backend manually sets the `bucket` and uses path style requests: -For example, a `resolvers` value of: ```toml -[[resolvers]] -regex = '^(example_bucket)/(?P.*)$' -substitution_string = '$key' +[[locations]] +regex = "prefix/(?P.*)$" +substitution_string = "$key" -[resolvers.storage] -backend = 'S3' -# Uses the first capture group in the regex as the bucket. +backend.kind = "S3" +backend.bucket = "bucket" +backend.path_style = true ``` -Will use "example_bucket" as the S3 bucket if that resolver matches, because this is the first capture group in the `regex`. -Note, to use this feature, at least one capture group must be defined in the `regex`. +To manually configure `Url` locations, set `backend.kind = "Url"`, specify any additional options from below under the `backend` table: -`UrlStorage` is a storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. -To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `backend = 'Url'` under `[resolvers.storage]`, and specify any additional options from below: +| Option | Description | Type | Default | +|--------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------| +| `url` | The URL to fetch data from. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `response_url` | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. | Boolean | `true` | +| `header_blacklist` | List of headers that should not be forwarded. | Array of headers | `[]` | +| `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [server configuration](#server-configuration) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. | -| Option | Description | Type | Default | -|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------| -| `url` | The URL to fetch data from. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `response_url` | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. | Boolean | `true` | -| `header_blacklist` | List of headers that should not be forwarded. | Array of headers | `[]` | -| `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [TLS](#tls) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. | +For example, the following forwards all headers to response tickets except `Host`, and constructs tickets using `https://example.com` instead of `http://localhost:8080`: -When using `UrlStorage`, the following requests will be made to the `url`. -* `GET` request to fetch only the headers of the data file (e.g. `GET /data.bam`, with `Range: bytes=0-`). -* `GET` request to fetch the entire index file (e.g. `GET /data.bam.bai`). -* `HEAD` request on the data file to get its length (e.g. `HEAD /data.bam`). +```toml +[[locations]] +regex = ".*" +substitution_string = "$0" -By default, all headers received in the initial query will be included when making these requests. To exclude certain headers from being forwarded, set the `header_blacklist` option. Note that the blacklisted headers are removed from the requests made to `url` and from the URL tickets as well. +backend.kind = "Url" +backend.url = "http://localhost:8080" +backend.response_url = "https://example.com" +backend.forward_headers = true +backend.header_blacklist = ["Host"] +``` -Example of a resolver with `UrlStorage`: +Regex-based locations also support multiple locations: ```toml -[[resolvers]] +[[locations]] +regex = "prefix/(?P.*)$" +substitution_string = "$key" +backend.kind = "S3" +backend.bucket = "bucket" +backend.path_style = true + +[[locations]] regex = ".*" substitution_string = "$0" - -[resolvers.storage] -backend = 'Url' -url = "http://localhost:8080" -response_url = "https://example.com" -forward_headers = true -header_blacklist = ["Host"] +backend.kind = "Url" +backend.url = "http://localhost:8080" +forward_headers = false ``` -There are additional examples of config files located under [`examples/config-files`][examples-config-files]. +If there is an overlap in regex matches, the first location specified will be the one used. + +Additional config file examples are available under [`example/config-files`][examples-config-files]. -#### Allow guard -Additionally, the resolver component has a feature, which allows resolving IDs based on the other fields present in a query. -This is useful as it allows the resolver to match an ID only if a particular set of query parameters are also present. For example, -a resolver can be set to only resolve IDs if the format is also BAM. +### Allow guard -This component can be configured by setting the `[resolver.allow_guard]` table with. The following options are available to restrict which queries are resolved by a resolver: +Additionally, locations support resolving IDs based on the other fields present in a query. +This is useful to allow the location to match an ID only if a particular set of query parameters are also present. + +This component can be configured by setting the `guard` table with: | Option | Description | Type | Default | |-------------------------|-----------------------------------------------------------------------------------------|-----------------------------------------------------------------------|-------------------------------------| @@ -259,59 +270,42 @@ This component can be configured by setting the `[resolver.allow_guard]` table w | `allow_tags` | Resolve the query ID if the query also contains the tags set by this option. | Array of tags or `'All'` | `'All'` | | `allow_formats` | Resolve the query ID if the query is one of the formats specified by this option. | An array of formats containing `'BAM'`, `'CRAM'`, `'VCF'`, or `'BCF'` | `['BAM', 'CRAM', 'VCF', 'BCF']` | | `allow_classes` | Resolve the query ID if the query is one of the classes specified by this option. | An array of classes containing eithr `'body'` or `'header'` | `['body', 'header']` | -| `allow_interval_start` | Resolve the query ID if the query reference start position is at least this option. | Unsigned 32-bit integer start position, 0-based, inclusive | Not set, allows all start positions | -| `allow_interval_end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive. | Not set, allows all end positions | +| `allow_interval.start` | Resolve the query ID if the query reference start position is at least this option. | Unsigned 32-bit integer start position, 0-based, inclusive | Not set, allows all start positions | +| `allow_interval.end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive | Not set, allows all end positions | -An example of a fully configured resolver: +For example, match only if the request queries `chr1` with positions between `100` and `1000`: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' - -[resolvers.storage] -backend = 'S3' -bucket = 'bucket' - -[resolvers.allow_guard] -allow_reference_names = ['chr1'] -allow_fields = ['QNAME'] -allow_tags = ['RG'] -allow_formats = ['BAM'] -allow_classes = ['body'] -allow_interval_start = 100 -allow_interval_end = 1000 -``` - -In this example, the resolver will only match the query ID if the query is for `chr1` with positions between `100` and `1000`. +[[locations]] +regex = ".*" +substitution_string = "$0" -#### TLS +backend.kind = "S3" +backend.bucket = "bucket" -TLS can be configured for the ticket server, data server, or the url storage client. These options read private keys and -certificates from PEM-formatted files. Certificates must be in X.509 format and private keys can be RSA, PKCS8, or SEC1 (EC) encoded. -The following options are available: +guard.allow_reference_names = ["chr1"] +guard.allow_interval.start = 100 +guard.allow_interval.end = 1000 +``` -| Option | Description | Type | Default | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| -| `key` | The path to the PEM formatted X.509 certificate. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | -| `cert` | The path to the PEM formatted RSA, PKCS8, or SEC1 encoded EC private key. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | -| `root_store` | The path to the PEM formatted root certificate store. Only used to specify non-native root certificates for the HTTP client in `UrlStorage`. | Filesystem path | Not Set | +### Server configuration -When used by the ticket and data servers, `key` and `cert` enable TLS, and when used with the url storage client, they enable client authentication. -The root store is only used by the url storage client. Note, the url storage client always allows TLS, however the default configuration performs no client authentication -and uses the native root certificate store. +To use custom root certificates for `Url` locations, set the following: -For example, TLS for the ticket server can be enabled by specifying the key and cert options: ```toml -ticket_server_tls.cert = "cert.pem" -ticket_server_tls.key = "key.pem" +[[locations]] +regex = ".*" +substitution_string = "$0" + +backend.kind = "Url" +backend.tls.root_store = "root.crt" ``` This project uses [rustls] for all TLS logic, and it does not depend on OpenSSL. The rustls library can be more strict when accepting certificates and keys. If generating certificates for `root_store` using OpenSSL, the correct extensions, such as `subjectAltName` should be included. -An example of generating a custom root CA and certificates for a `UrlStorage` backend: +An example of generating a custom root CA and certificates for a `Url` backend: ```sh # Create a root CA @@ -320,173 +314,43 @@ openssl req -x509 -noenc -subj '/CN=localhost' -newkey rsa -keyout root.key -out # Create a certificate signing request openssl req -noenc -newkey rsa -keyout server.key -out server.csr -subj '/CN=localhost' -addext subjectAltName=DNS:localhost -# Create the `UrlStorage` server's certificate +# Create the `Url` server's certificate openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -days 365 -out server.crt -copy_extensions copy # An additional client certificate signing request and certificate can be created in the same way as the server # certificate if using client authentication. ``` -The `root.crt` can then be used in htsget-rs to allow authenticating to a `UrlStorage` backend using `server.crt`: +CORS can also be configured for the data and ticket servers by specifying the `cors` option: ```toml -# Trust the root CA that signed the server's certificate. -tls.root_store = "root.crt" -``` - -Alternatively, projects such as [mkcert] can be used to simplify this process. - -Further TLS examples are available under [`examples/config-files`][examples-config-files]. - -[examples-config-files]: examples/config-files -[rustls]: https://github.com/rustls/rustls -[mkcert]: https://github.com/FiloSottile/mkcert - -#### Config file location - -The htsget-rs binaries ([htsget-axum], [htsget-actix] and [htsget-lambda]) support some command line options. The config file location can -be specified by setting the `--config` option: - -```shell -cargo run -p htsget-axum -- --config "config.toml" +ticket_server.cors.allow_credentials = false +ticket_server.cors.allow_origins = "Mirror" +ticket_server.cors.allow_headers = "All" +ticket_server.cors.allow_methods = ["GET", "POST"] +ticket_server.cors.max_age = 86400 +ticket_server.cors.expose_headers = [] ``` -The config can also be read from an environment variable: - -```shell -export HTSGET_CONFIG="config.toml" -``` - -If no config file is specified, the default configuration is used. Further, the default configuration file can be printed to stdout by passing -the `--print-default-config` flag: - -```shell -cargo run -p htsget-axum -- --print-default-config -``` - -Use the `--help` flag to see more details on command line options. - -[htsget-actix]: ../htsget-actix -[htsget-axum]: ../htsget-axum -[htsget-lambda]: ../htsget-lambda - -#### Log formatting - -The [Tracing][tracing] crate is used by htsget-rs is for logging functionality. The `RUST_LOG` variable is -read to configure the level that trace logs are emitted. - -For example, the following indicates trace level for all htsget crates, and info level for all other crates: - -```sh -export RUST_LOG='info,htsget_lambda=trace,htsget_lambda=trace,htsget_config=trace,htsget_http=trace,htsget_search=trace,htsget_test=trace' -``` - -See [here][rust-log] for more information on setting this variable. - -The style of formatting can be configured by setting the following option: - -| Option | Description | Type | Default | -|---------------------------------------------------------|--------------------------------------|--------------------------------------------------------|----------| -| `formatting_style` | The style of log formatting to use. | One of `'Full'`, `'Compact'`, `'Pretty'`, or `'Json'` | `'Full'` | - -See [here][formatting-style] for more information on how these values look. - -[tracing]: https://github.com/tokio-rs/tracing -[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html -[formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters - -#### Environment variables - -All the htsget-rs config options can be set using environment variables, which is convenient for runtimes such as AWS Lambda. -The ticket server, data server and service info options are flattened and can be set directly using -environment variable. It is not recommended to set the resolvers using environment variables, however it can be done by setting a single environment variable which -contains a list of structures, where a key name and value pair is used to set the nested options. - -Environment variables will override options set in the config file. Note, arrays are delimited with `[` and `]` in environment variables, and items are separated by commas. - -The following environment variables - corresponding to the TOML config - are available: - -| Variable | Description | -|-----------------------------------------------|-------------------------------------------------------------------------------------| -| `HTSGET_TICKET_SERVER_ADDR` | See [`ticket_server_addr`](#ticket_server_addr) | -| `HTSGET_TICKET_SERVER_TLS_KEY` | See [`TLS`](#tls) | -| `HTSGET_TICKET_SERVER_TLS_CERT` | See [`TLS`](#tls) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS` | See [`ticket_server_cors_allow_credentials`](#ticket_server_cors_allow_credentials) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS` | See [`ticket_server_cors_allow_origins`](#ticket_server_cors_allow_origins) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_HEADERS` | See [`ticket_server_cors_allow_headers`](#ticket_server_cors_allow_headers) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_METHODS` | See [`ticket_server_cors_allow_methods`](#ticket_server_cors_allow_methods) | -| `HTSGET_TICKET_SERVER_CORS_MAX_AGE` | See [`ticket_server_cors_max_age`](#ticket_server_cors_max_age) | -| `HTSGET_TICKET_SERVER_CORS_EXPOSE_HEADERS` | See [`ticket_server_cors_expose_headers`](#ticket_server_cors_expose_headers) | -| `HTSGET_DATA_SERVER_ADDR` | See [`data_server_addr`](#data_server_addr) | -| `HTSGET_DATA_SERVER_LOCAL_PATH` | See [`data_server_local_path`](#data_server_local_path) | -| `HTSGET_DATA_SERVER_SERVE_AT` | See [`data_server_serve_at`](#data_server_serve_at) | -| `HTSGET_DATA_SERVER_TLS_KEY` | See [`TLS`](#tls) | -| `HTSGET_DATA_SERVER_TLS_CERT` | See [`TLS`](#tls) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_CREDENTIALS` | See [`data_server_cors_allow_credentials`](#data_server_cors_allow_credentials) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_ORIGINS` | See [`data_server_cors_allow_origins`](#data_server_cors_allow_origins) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_HEADERS` | See [`data_server_cors_allow_headers`](#data_server_cors_allow_headers) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_METHODS` | See [`data_server_cors_allow_methods`](#data_server_cors_allow_methods) | -| `HTSGET_DATA_SERVER_CORS_MAX_AGE` | See [`data_server_cors_max_age`](#data_server_cors_max_age) | -| `HTSGET_DATA_SERVER_CORS_EXPOSE_HEADERS` | See [`data_server_cors_expose_headers`](#data_server_cors_expose_headers) | -| `HTSGET_ID` | See [`id`](#id) | -| `HTSGET_NAME` | See [`name`](#name) | -| `HTSGET_VERSION` | See [`version`](#version) | -| `HTSGET_ORGANIZATION_NAME` | See [`organization_name`](#organization_name) | -| `HTSGET_ORGANIZATION_URL` | See [`organization_url`](#organization_url) | -| `HTSGET_CONTACT_URL` | See [`contact_url`](#contact_url) | -| `HTSGET_DOCUMENTATION_URL` | See [`documentation_url`](#documentation_url) | -| `HTSGET_CREATED_AT` | See [`created_at`](#created_at) | -| `HTSGET_UPDATED_AT` | See [`updated_at`](#updated_at) | -| `HTSGET_ENVIRONMENT` | See [`environment`](#environment) | -| `HTSGET_RESOLVERS` | See [`resolvers`](#resolvers) | -| `HTSGET_FORMATTING_STYLE` | See [`formatting_style`](#formatting_style) | - -In order to use `HTSGET_RESOLVERS`, the entire resolver config array must be set. The nested array of resolvers structure can be set using name key and value pairs, for example: - -```shell -export HTSGET_RESOLVERS="[{ - regex=regex, - substitution_string=substitution_string, - storage={ - type=S3, - bucket=bucket - }, - allow_guard={ - allow_reference_names=[chr1], - allow_fields=[QNAME], - allow_tags=[RG], - allow_formats=[BAM], - allow_classes=[body], - allow_interval_start=100, - allow_interval_end=1000 - } -}]" -``` - -Similar to the [data_server](#data_server) option, the data server can be disabled by setting the equivalent environment variable: - -```shell -export HTSGET_DATA_SERVER_ENABLED=false -``` -[service-info]: https://samtools.github.io/hts-specs/htsget.html#ga4gh-service-info +Use `"Mirror"` to mirror CORS requests, and `"All"` to allow all methods, headers, or origins. The `ticket_server` table +above can be replaced with `data_server` to configure CORS for the data server. ### MinIO -Operating a local object storage like [MinIO][minio] can be achieved by leveraging the `endpoint` directive as shown below: +Operating a local object storage like [MinIO][minio] can be achieved by using `endpoint` under `"S3"` locations as shown below: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' - -[resolvers.storage] -backend = 'S3' -bucket = 'bucket' -endpoint = 'http://127.0.0.1:9000' -path_style = true +[[locations]] +regex = ".*" +substitution_string = "$0" + +backend.kind = 'S3' +backend.bucket = 'bucket' +backend.endpoint = 'http://127.0.0.1:9000' +backend.path_style = true ``` -Care must be taken to ensure that the [correct][env-variables] `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY` is set to allow +Care must be taken to ensure that the [correct][env-variables] `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY` are set to allow the AWS sdk to reach the endpoint. Additional configuration of the MinIO server is required to use [virtual-hosted][virtual-addressing] style addressing by setting the `MINIO_DOMAIN` environment variable. [Path][path-addressing] style addressing can be forced using `path_style = true`. @@ -494,73 +358,110 @@ See the MinIO deployment [example][minio-deployment] for more information on how ### Crypt4GH -There is experimental support for serving [Crypt4GH][c4gh] encrypted files. This can be enabled by compiling with the -`experimental` feature flag. +There is experimental support for serving [Crypt4GH][c4gh] encrypted files. This allows htsget-rs to read Crypt4GH files and serve them encrypted, directly to the client. In the process of serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and re-encrypt them so that the client can read them. When the client receives byte ranges from htsget-rs and concatenates them, the output bytes will be Crypt4GH encrypted, and will need to be decrypted before they can be read. All file formats (BAM, CRAM, VCF, and BCF) are supported using Crypt4GH. -To use this feature, set `location = 'Local'` under `resolvers.storage.keys` to specify the private and public keys: +To use this feature, set `keys.kind = "File"` under the `location` table to specify the private and public keys: -| Option | Description | Type | Default | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| -| `private_key` | The path to PEM formatted private key which htsget-rs uses to decrypt Crypt4GH data. | Filesystem path | Not Set | -| `recipient_public_key` | The path to the PEM formatted public key which the recipient of the data will use. This is what the client will use to decrypt the returned data, using the corresponding private key. | Filesystem path | Not Set | +| Option | Description | Type | Default | +|-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| +| `private` | The path to PEM formatted private key which htsget-rs uses to decrypt Crypt4GH data. | Filesystem path | Not Set | +| `public` | The path to the PEM formatted public key which the recipient of the data will use. This is what the client will use to decrypt the returned data, using the corresponding private key. | Filesystem path | Not Set | For example: ```toml [[resolvers]] -regex = '.*' -substitution_string = '$0' +regex = ".*" +substitution_string = "$0" -[resolvers.storage.keys] -location = 'Local' -private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret -recipient_public_key = 'data/c4gh/keys/alice.pub' +location.keys.kind = "File" +location.keys.private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret +location.keys.public = "data/c4gh/keys/alice.pub" ``` -Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `location = 'SecretsManager'` under -`resolvers.storage.keys` to fetch keys from Secrets Manager. When using Secrets Manager, the `private_key` and `recipient_public_key` +Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `keys.kind = "SecretsManager"` under +`location` to fetch keys from Secrets Manager. When using Secrets Manager, the `private` and `public` correspond to ARNs or secret names in Secrets Manager storing PEM formatted keys. For example: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' +[[locations]] +regex = ".*" +substitution_string = "$0" -[resolvers.storage.keys] -location = 'SecretsManager' -private_key = 'private_key_secret_name' # pragma: allowlist secret -recipient_public_key = 'public_key_secret_name' +location.keys.kind = "SecretsManager" +location.keys.private = "private_key_secret_name" # pragma: allowlist secret +location.keys.public = "public_key_secret_name" ``` The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure. Any of the storage types are supported, i.e. `Local`, `S3`, or `Url`. +### Log formatting + +The `RUST_LOG` variable is read to configure the level that trace logs are emitted. + +For example, the following indicates trace level for all htsget crates, and info level for all other crates: + +```sh +export RUST_LOG='info,htsget_lambda=trace,htsget_lambda=trace,htsget_config=trace,htsget_http=trace,htsget_search=trace,htsget_test=trace' +``` + +See [here][rust-log] for more information on setting this variable. + +The style of formatting can be configured by setting the following option: + +| Option | Description | Type | Default | +|---------------------------------------------------------|--------------------------------------|--------------------------------------------------------|----------| +| `formatting_style` | The style of log formatting to use. | One of `'Full'`, `'Compact'`, `'Pretty'`, or `'Json'` | `'Full'` | + +See [here][formatting-style] for more information on how these values look. + +### Environment variables + +Advanced configuration options also support environment variables. Generally, options separated by `.` in a config file +are separated by `_` in the corresponding environment variable. For example, to set the ticket server allow origins, +use `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS`. It is not recommended to set regex-based locations using environment +variables because the variables needs to contain the nested array structure of storage backends. + ### As a library This crate reads config files and environment variables using [figment], and accepts command-line arguments using clap. The main function for this is `from_config`, -which is used to obtain the `Config` struct. The crate also contains the `regex_resolver` abstraction, which is used for matching a query ID with -regex, and changing it by using a substitution string. +which is used to obtain the `Config` struct. The crate also contains the `resolver` abstraction, which is used for matching a query ID with +regex, and changing it by using a substitution string. Advanced configuration options are specified in the [`advanced.rs`][advanced] submodule. +[advanced]: src/config/advanced/mod.rs [figment]: https://github.com/SergioBenitez/Figment -#### Feature flags +### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License This project is licensed under the [MIT license][license]. +[tracing]: https://github.com/tokio-rs/tracing +[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html +[formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters +[examples-config-files]: examples/config-files +[rustls]: https://github.com/rustls/rustls +[htsget-actix]: ../htsget-actix +[htsget-axum]: ../htsget-axum +[htsget-lambda]: ../htsget-lambda +[tracing]: https://github.com/tokio-rs/tracing +[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html +[formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters +[service-info]: https://samtools.github.io/hts-specs/htsget.html#ga4gh-service-info [path-addressing]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access [env-variables]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html [virtual-addressing]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access @@ -571,5 +472,5 @@ This project is licensed under the [MIT license][license]. [data-c4gh]: ../data/c4gh [secrets-manager]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html [id]: https://samtools.github.io/hts-specs/htsget.html#url-parameters -[basic]: examples/config-files/basic.toml -[data-server]: README.md#data-server-config \ No newline at end of file +[toml]: https://toml.io/en/ +[data]: ../data \ No newline at end of file diff --git a/htsget-config/examples/config-files/basic.toml b/htsget-config/examples/config-files/basic.toml deleted file mode 100644 index 49457147..00000000 --- a/htsget-config/examples/config-files/basic.toml +++ /dev/null @@ -1,18 +0,0 @@ -# An example of running htsget-rs. -# Run with `cargo run --all-features -- --config htsget-config/examples/config-files/basic.toml` - -ticket_server_addr = "127.0.0.1:8080" -data_server_addr = "127.0.0.1:8081" - -# Serve data locally from the `data` directory. -[[resolvers]] -regex = '.*' -substitution_string = '$0' -storage.backend = 'Local' - -# Serve data from S3 if the id is prefixed with `example_bucket`. -[[resolvers]] -regex = '^(example_bucket)/(?P.*)$' -substitution_string = '$key' -storage.backend = 'S3' -# Uses the first capture group in the regex as the bucket. diff --git a/htsget-config/examples/config-files/c4gh.toml b/htsget-config/examples/config-files/c4gh.toml index 4a350fec..00ccd1e1 100644 --- a/htsget-config/examples/config-files/c4gh.toml +++ b/htsget-config/examples/config-files/c4gh.toml @@ -1,23 +1,23 @@ # An example of running htsget-rs with Crypt4GH enabled. # Run with `cargo run -p htsget-axum --features experimental -- --config htsget-config/examples/config-files/c4gh.toml` -ticket_server_addr = "127.0.0.1:8080" -data_server_addr = "127.0.0.1:8081" +ticket_server.addr = "127.0.0.1:8080" +data_server.addr = "127.0.0.1:8081" [[resolvers]] regex = ".*" substitution_string = "$0" -[resolvers.storage] -backend = 'Local' +[locations.backend] +kind = "File" -[resolvers.storage.keys] -location = "Local" -private_key = "data/c4gh/keys/bob.sec" # pragma: allowlist secret -recipient_public_key = "data/c4gh/keys/alice.pub" +[locations.backend.keys] +kind = "Local" +private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret +public = "data/c4gh/keys/alice.pub" # Or, use AWS secrets manager to store keys. -#[resolvers.storage.keys] -#location = "SecretsManager" -#private_key = "htsget/test_c4gh_private_key" # pragma: allowlist secret -#recipient_public_key = "htsget/test_c4gh_public_key" +#[locations.backend.keys] +#kind = "SecretsManager" +#private = "htsget/test_c4gh_private_key" # pragma: allowlist secret +#public = "htsget/test_c4gh_public_key" diff --git a/htsget-config/examples/config-files/default.toml b/htsget-config/examples/config-files/default.toml index 93ea3608..6845e790 100644 --- a/htsget-config/examples/config-files/default.toml +++ b/htsget-config/examples/config-files/default.toml @@ -1,40 +1,37 @@ # Config generated by running `cargo run -p htsget-axum -- -p` formatting_style = "Full" -ticket_server_addr = "127.0.0.1:8080" -ticket_server_cors_allow_credentials = false -ticket_server_cors_allow_origins = ["http://localhost:8080"] -ticket_server_cors_allow_headers = "All" -ticket_server_cors_allow_methods = "All" -ticket_server_cors_max_age = 86400 -ticket_server_cors_expose_headers = [] -data_server_enabled = true -data_server_addr = "127.0.0.1:8081" -data_server_local_path = "./" -data_server_serve_at = "" -data_server_cors_allow_credentials = false -data_server_cors_allow_origins = ["http://localhost:8080"] -data_server_cors_allow_headers = "All" -data_server_cors_allow_methods = "All" -data_server_cors_max_age = 86400 -data_server_cors_expose_headers = [] - -[[resolvers]] -regex = ".*" -substitution_string = "$0" -storage = "Local" - -[resolvers.allow_guard] -allow_reference_names = "All" -allow_fields = "All" -allow_tags = "All" -allow_formats = [ - "BAM", - "CRAM", - "VCF", - "BCF", -] -allow_classes = [ - "body", - "header", -] + +[ticket_server] +addr = "127.0.0.1:8080" + +[ticket_server.cors] +allow_credentials = false +allow_origins = "Mirror" +allow_headers = "Mirror" +allow_methods = "Mirror" +max_age = 2592000 +expose_headers = "All" + +[data_server] +addr = "127.0.0.1:8081" +local_path = "./" + +[data_server.cors] +allow_credentials = false +allow_origins = "Mirror" +allow_headers = "Mirror" +allow_methods = "Mirror" +max_age = 2592000 +expose_headers = "All" + +[service_info] + +[[locations]] +prefix = "" + +[locations.backend] +kind = "File" +scheme = "HTTP" +authority = "127.0.0.1:8081" +local_path = "./" diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 5cde4dff..7948e457 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -1,19 +1,18 @@ # An example for a server which uses s3 storage with data located in "bucket". # Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/s3_storage.toml` -ticket_server_cors_allow_headers = "All" -ticket_server_cors_allow_methods = "All" -ticket_server_cors_allow_credentials = true -ticket_server_cors_max_age = 300 +ticket_server.cors.allow_headers = "All" +ticket_server.cors.allow_methods = "All" +ticket_server.cors.allow_credentials = true +ticket_server.cors.max_age = 300 -data_server_enabled = false +data_server = "None" -[[resolvers]] -regex = '^(bucket)/(?P.*)$' -substitution_string = '$key' -storage.backend = 'S3' +[[locations]] +regex = "^(bucket)/(?P.*)$" +substitution_string = "$key" +backend.kind = "S3" # Or, set the bucket manually -#[resolvers.storage] -#backend = 'S3' -#bucket = 'bucket' +#backend.kind = "S3" +#backend.bucket = "bucket" diff --git a/htsget-config/examples/config-files/tls_data_server.toml b/htsget-config/examples/config-files/tls_data_server.toml index d2e4316e..5d8987d8 100644 --- a/htsget-config/examples/config-files/tls_data_server.toml +++ b/htsget-config/examples/config-files/tls_data_server.toml @@ -1,16 +1,13 @@ # An example config file for a TLS data server that uses a local storage backend. # Run with `cargo run -p htsget-axum -- --config htsget-config/examples/config-files/tls_data_server.toml` -ticket_server_addr = "0.0.0.0:8080" -data_server_addr = "0.0.0.0:8081" -data_server_cors_allow_origins = "All" -data_server_tls.cert = "cert.pem" -data_server_tls.key = "key.pem" +ticket_server.addr = "0.0.0.0:8080" +data_server.addr = "0.0.0.0:8081" +data_server.cors.allow_origins = "All" +data_server.tls.cert = "cert.pem" +data_server.tls.key = "key.pem" -[[resolvers]] +[[locations]] regex = ".*" substitution_string = "$0" - -[resolvers.storage] -backend = 'Local' -use_data_server_config = true +backend.kind = "Local" diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index 9bd196ff..d73bb596 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -1,16 +1,14 @@ # An example config file for a TLS ticket server that uses S3 as a storage backend. # Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/tls_ticket_server.toml` -ticket_server_addr = "0.0.0.0:8080" -ticket_server_cors_allow_origins = "All" -ticket_server_tls.cert = "cert.pem" -ticket_server_tls.key = "key.pem" -data_server_addr = "0.0.0.0:8081" +ticket_server.addr = "0.0.0.0:8080" +ticket_server.cors_allow_origins = "All" +ticket_server.tls.cert = "cert.pem" +ticket_server.tls.key = "key.pem" +data_server.addr = "0.0.0.0:8081" -[[resolvers]] +[[locations]] regex = ".*" substitution_string = "$0" - -[resolvers.storage] -backend = 'S3' -bucket = "bucket" +backend.kind = "S3" +backend.bucket = "bucket" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index 372b0080..078a41b4 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -3,27 +3,26 @@ # `cargo run -p htsget-axum --features url-storage -- --config htsget-config/examples/config-files/url_storage.toml` # in the project directory. -ticket_server_addr = "127.0.0.1:8082" -ticket_server_cors_allow_origins = "All" +ticket_server.addr = "127.0.0.1:8082" +ticket_server.cors.allow_origins = "All" -ticket_server_cert = "cert.pem" -ticket_server_key = "key.pem" +ticket_server.cert = "cert.pem" +ticket_server.key = "key.pem" -data_server_enabled = false +data_server = "None" -[[resolvers]] +[[locations]] regex = ".*" substitution_string = "$0" -[resolvers.storage] -backend = 'Url' -url = "http://127.0.0.1:8081" -response_url = "https://127.0.0.1:8081" -forward_headers = true +backend.kind = "Url" +backend.url = "http://127.0.0.1:8081" +backend.response_url = "https://127.0.0.1:8081" +backend.forward_headers = true # Set client authentication -#tls.key = "key.pem" -#tls.cert = "cert.pem" +#backend.tls.key = "key.pem" +#backend.tls.cert = "cert.pem" # Set root certificates -#tls.root_store = "cert.pem" +#backend.tls.root_store = "cert.pem" diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs new file mode 100644 index 00000000..3394ce93 --- /dev/null +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -0,0 +1,341 @@ +//! Allow guard configuration. +//! + +use crate::types::Format::{Bam, Bcf, Cram, Vcf}; +use crate::types::{Class, Fields, Format, Interval, Query, TaggedTypeAll, Tags}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +/// Determines whether the query matches for use with the storage. +pub trait QueryAllowed { + /// Does this query match. + fn query_allowed(&self, query: &Query) -> bool; +} + +/// A query guard represents query parameters that can be allowed to storage for a given query. +#[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct AllowGuard { + allow_reference_names: ReferenceNames, + allow_fields: Fields, + allow_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + allow_interval: Interval, +} + +impl Default for AllowGuard { + fn default() -> Self { + Self { + allow_formats: vec![Bam, Cram, Vcf, Bcf], + allow_classes: vec![Class::Body, Class::Header], + allow_interval: Default::default(), + allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), + allow_fields: Fields::Tagged(TaggedTypeAll::All), + allow_tags: Tags::Tagged(TaggedTypeAll::All), + } + } +} + +/// Reference names that can be matched. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged, deny_unknown_fields)] +pub enum ReferenceNames { + Tagged(TaggedTypeAll), + List(HashSet), +} + +impl AllowGuard { + /// Create a new allow guard. + pub fn new( + allow_reference_names: ReferenceNames, + allow_fields: Fields, + allow_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + allow_interval: Interval, + ) -> Self { + Self { + allow_reference_names, + allow_fields, + allow_tags, + allow_formats, + allow_classes, + allow_interval, + } + } + + /// Get allow formats. + pub fn allow_formats(&self) -> &[Format] { + &self.allow_formats + } + + /// Get allow classes. + pub fn allow_classes(&self) -> &[Class] { + &self.allow_classes + } + + /// Get allow interval. + pub fn allow_interval(&self) -> Interval { + self.allow_interval + } + + /// Get allow reference names. + pub fn allow_reference_names(&self) -> &ReferenceNames { + &self.allow_reference_names + } + + /// Get allow fields. + pub fn allow_fields(&self) -> &Fields { + &self.allow_fields + } + + /// Get allow tags. + pub fn allow_tags(&self) -> &Tags { + &self.allow_tags + } +} + +impl QueryAllowed for ReferenceNames { + fn query_allowed(&self, query: &Query) -> bool { + match (self, &query.reference_name()) { + (ReferenceNames::Tagged(TaggedTypeAll::All), _) => true, + (ReferenceNames::List(reference_names), Some(reference_name)) => { + reference_names.contains(*reference_name) + } + (ReferenceNames::List(_), None) => false, + } + } +} + +impl QueryAllowed for Fields { + fn query_allowed(&self, query: &Query) -> bool { + match (self, &query.fields()) { + (Fields::Tagged(TaggedTypeAll::All), _) => true, + (Fields::List(self_fields), Fields::List(query_fields)) => { + self_fields.is_subset(query_fields) + } + (Fields::List(_), Fields::Tagged(TaggedTypeAll::All)) => false, + } + } +} + +impl QueryAllowed for Tags { + fn query_allowed(&self, query: &Query) -> bool { + match (self, &query.tags()) { + (Tags::Tagged(TaggedTypeAll::All), _) => true, + (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), + (Tags::List(_), Tags::Tagged(TaggedTypeAll::All)) => false, + } + } +} + +impl QueryAllowed for AllowGuard { + fn query_allowed(&self, query: &Query) -> bool { + self.allow_formats().contains(&query.format()) + && self.allow_classes().contains(&query.class()) + && self + .allow_interval() + .contains(query.interval().start().unwrap_or(u32::MIN)) + && self + .allow_interval() + .contains(query.interval().end().unwrap_or(u32::MAX)) + && self.allow_reference_names().query_allowed(query) + && self.allow_fields().query_allowed(query) + && self.allow_tags().query_allowed(query) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + #[cfg(feature = "s3-storage")] + use crate::config::Config; + use crate::types::Class::Header; + + #[test] + fn allow_reference_names_all() { + test_serialize_and_deserialize( + "allow_reference_names = \"All\"", + AllowGuard { + allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_reference_names_list() { + test_serialize_and_deserialize( + "allow_reference_names = [\"chr1\"]", + AllowGuard { + allow_reference_names: ReferenceNames::List(HashSet::from_iter(vec!["chr1".to_string()])), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_fields_all() { + test_serialize_and_deserialize( + "allow_fields = \"All\"", + AllowGuard { + allow_fields: Fields::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_fields_list() { + test_serialize_and_deserialize( + "allow_fields = [\"field\"]", + AllowGuard { + allow_fields: Fields::List(HashSet::from_iter(vec!["field".to_string()])), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_tags_all() { + test_serialize_and_deserialize( + "allow_tags = \"All\"", + AllowGuard { + allow_tags: Tags::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_tags_list() { + test_serialize_and_deserialize( + "allow_tags = [\"tag\"]", + AllowGuard { + allow_tags: Tags::List(HashSet::from_iter(vec!["tag".to_string()])), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_formats() { + test_serialize_and_deserialize( + "allow_formats = [\"BAM\"]", + AllowGuard { + allow_formats: vec![Bam], + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_classes() { + test_serialize_and_deserialize( + "allow_classes = [\"Header\"]", + AllowGuard { + allow_classes: vec![Header], + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_interval() { + test_serialize_and_deserialize( + r#" + allow_interval.start = 0 + allow_interval.end = 100 + "#, + AllowGuard { + allow_interval: Interval::new(Some(0), Some(100)), + ..Default::default() + }, + |result| result, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn allow_guard() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = ".*" + substitution_string = "$0" + + backend.kind = "S3" + backend.bucket = "bucket" + + guard.allow_reference_names = ["chr1"] + guard.allow_interval.start = 100 + guard.allow_interval.end = 1000 + "#, + AllowGuard { + allow_reference_names: ReferenceNames::List(HashSet::from_iter(vec!["chr1".to_string()])), + allow_interval: Interval::new(Some(100), Some(1000)), + ..Default::default() + }, + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + + AllowGuard { + allow_reference_names: location.guard().unwrap().allow_reference_names.clone(), + allow_interval: location.guard().unwrap().allow_interval, + ..Default::default() + } + }, + ); + } + + #[test] + fn query_allowed() { + let guard = AllowGuard { + allow_formats: vec![Bam], + allow_classes: vec![Header], + allow_interval: Interval::new(Some(0), Some(100)), + ..Default::default() + }; + + let query = Query::new_with_default_request("", Bam); + assert!(!guard.query_allowed(&query)); + assert!(!guard.query_allowed(&query.clone().with_format(Cram))); + + assert!(guard.query_allowed(&query.clone().with_class(Header).with_start(1).with_end(50))); + assert!(guard.query_allowed( + &query + .clone() + .with_class(Header) + .with_start(1) + .with_end(50) + .with_tags(Tags::List(HashSet::from_iter(vec!["tag".to_string()]))) + )); + assert!(guard.query_allowed( + &query + .clone() + .with_class(Header) + .with_start(1) + .with_end(50) + .with_fields(Fields::List(HashSet::from_iter(vec!["field".to_string()]))) + )); + + assert!(!guard.query_allowed( + &query + .clone() + .with_class(Header) + .with_start(1) + .with_end(1000) + )); + } +} diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/advanced/cors.rs similarity index 69% rename from htsget-config/src/config/cors.rs rename to htsget-config/src/config/advanced/cors.rs index 75f0546b..bebbe9e5 100644 --- a/htsget-config/src/config/cors.rs +++ b/htsget-config/src/config/advanced/cors.rs @@ -1,3 +1,6 @@ +//! Configuration related to CORS. +//! + use std::fmt::{Display, Formatter}; use std::str::FromStr; @@ -7,11 +10,11 @@ use serde::de::Error; use serde::ser::SerializeSeq; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use crate::config::default_server_origin; use crate::types::TaggedTypeAll; /// The maximum default amount of time a CORS request can be cached for in seconds. -const CORS_MAX_AGE: usize = 86400; +/// Defaults to 30 days. +const CORS_MAX_AGE: usize = 2592000; /// Tagged allow headers for cors config, either Mirror or Any. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -25,7 +28,7 @@ pub enum TaggedAllowTypes { /// Allowed type for cors config which is used to configure cors behaviour. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(untagged)] -pub enum AllowType { +pub enum AllowType { Tagged(Tagged), #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] #[serde( @@ -134,6 +137,7 @@ where pub struct HeaderValue(HeaderValueInner); impl HeaderValue { + /// Get the inner header value. pub fn into_inner(self) -> HeaderValueInner { self.0 } @@ -154,15 +158,15 @@ impl Display for HeaderValue { } /// Cors configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] pub struct CorsConfig { allow_credentials: bool, - allow_origins: AllowType, + allow_origins: AllowType, allow_headers: AllowType, allow_methods: AllowType, max_age: usize, - expose_headers: AllowType, + expose_headers: AllowType, } impl CorsConfig { @@ -173,7 +177,7 @@ impl CorsConfig { allow_headers: AllowType, allow_methods: AllowType, max_age: usize, - expose_headers: AllowType, + expose_headers: AllowType, ) -> Self { Self { allow_credentials, @@ -211,7 +215,7 @@ impl CorsConfig { } /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { + pub fn expose_headers(&self) -> &AllowType { &self.expose_headers } } @@ -220,72 +224,103 @@ impl Default for CorsConfig { fn default() -> Self { Self { allow_credentials: false, - allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static( - default_server_origin(), - ))]), - allow_headers: AllowType::Tagged(TaggedTypeAll::All), - allow_methods: AllowType::Tagged(TaggedTypeAll::All), + allow_origins: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), max_age: CORS_MAX_AGE, - expose_headers: AllowType::List(vec![]), + expose_headers: AllowType::Tagged(TaggedTypeAll::All), } } } #[cfg(test)] mod tests { - use std::fmt::Debug; - + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; use http::Method; use toml::de::Error; - use super::*; - - fn test_cors_config(input: &str, expected: &T, get_result: F) - where - F: Fn(&CorsConfig) -> &T, - T: Debug + Eq, - { - let config: CorsConfig = toml::from_str(input).unwrap(); - assert_eq!(expected, get_result(&config)); - - let serialized = toml::to_string(&config).unwrap(); - let deserialized = toml::from_str(&serialized).unwrap(); - assert_eq!(expected, get_result(&deserialized)); - } - #[test] fn unit_variant_any_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "allow_methods = \"All\"", - &AllowType::Tagged(TaggedTypeAll::All), - |config| config.allow_methods(), + CorsConfig { + allow_methods: AllowType::Tagged(TaggedAllowTypes::All), + ..Default::default() + }, + |result| result, ); } #[test] fn unit_variant_mirror_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "allow_origins = \"Mirror\"", - &AllowType::Tagged(TaggedAllowTypes::Mirror), - |config| config.allow_origins(), + CorsConfig { + allow_origins: AllowType::Tagged(TaggedAllowTypes::Mirror), + ..Default::default() + }, + |result| result, ); } #[test] fn list_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "allow_methods = [\"GET\"]", - &AllowType::List(vec![Method::GET]), - |config| config.allow_methods(), + CorsConfig { + allow_methods: AllowType::List(vec![Method::GET]), + ..Default::default() + }, + |result| result, ); } #[test] fn tagged_any_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "expose_headers = \"All\"", - &AllowType::Tagged(TaggedTypeAll::All), - |config| config.expose_headers(), + CorsConfig { + expose_headers: AllowType::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn cors_config() { + test_serialize_and_deserialize( + r#" + ticket_server.cors.allow_credentials = false + ticket_server.cors.allow_origins = "Mirror" + ticket_server.cors.allow_headers = "All" + data_server.cors.allow_methods = ["GET", "POST"] + data_server.cors.max_age = 86400 + data_server.cors.expose_headers = [] + "#, + ( + false, + AllowType::Tagged(TaggedAllowTypes::Mirror), + AllowType::Tagged(TaggedAllowTypes::All), + AllowType::List(vec!["GET".parse().unwrap(), "POST".parse().unwrap()]), + 86400, + AllowType::List(vec![]), + ), + |result: Config| { + let ticket_cors = result.ticket_server().cors(); + let data_cors = result.data_server().as_data_server_config().unwrap().cors(); + + ( + ticket_cors.allow_credentials, + ticket_cors.allow_origins.clone(), + ticket_cors.allow_headers.clone(), + data_cors.allow_methods.clone(), + data_cors.max_age, + data_cors.expose_headers.clone(), + ) + }, ); } diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs new file mode 100644 index 00000000..a23342de --- /dev/null +++ b/htsget-config/src/config/advanced/mod.rs @@ -0,0 +1,20 @@ +//! Configuration options that are advanced in the documentation. +//! + +use serde::{Deserialize, Serialize}; + +pub mod allow_guard; +pub mod cors; +pub mod regex_location; +#[cfg(feature = "url-storage")] +pub mod url; + +/// Determines which tracing formatting style to use. +#[derive(Debug, Copy, Clone, Serialize, Deserialize, Default)] +pub enum FormattingStyle { + #[default] + Full, + Compact, + Pretty, + Json, +} diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs new file mode 100644 index 00000000..d8bff741 --- /dev/null +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -0,0 +1,206 @@ +//! Set the location using a regex and substitution values. +//! + +use crate::config::advanced::allow_guard::AllowGuard; +use crate::config::location::LocationEither; +use crate::storage::Backend; +use regex::Regex; +use serde::{Deserialize, Serialize}; + +/// A regex storage is a storage that matches ids using Regex. +#[derive(Serialize, Debug, Clone, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RegexLocation { + #[serde(with = "serde_regex")] + regex: Regex, + substitution_string: String, + backend: Backend, + guard: Option, +} + +impl RegexLocation { + /// Create a new regex location. + pub fn new( + regex: Regex, + substitution_string: String, + backend: Backend, + guard: Option, + ) -> Self { + Self { + regex, + substitution_string, + backend, + guard, + } + } + + /// Get the regex. + pub fn regex(&self) -> &Regex { + &self.regex + } + + /// Get the substitution string. + pub fn substitution_string(&self) -> &str { + &self.substitution_string + } + + /// Get the storage backend. + pub fn backend(&self) -> &Backend { + &self.backend + } + + /// Get the allow guard. + pub fn guard(&self) -> Option<&AllowGuard> { + self.guard.as_ref() + } +} + +impl Default for RegexLocation { + fn default() -> Self { + Self::new( + ".*".parse().expect("expected valid regex"), + "$0".to_string(), + Default::default(), + Default::default(), + ) + } +} + +impl From for LocationEither { + fn from(location: RegexLocation) -> Self { + Self::Regex(location) + } +} + +#[cfg(test)] +mod tests { + use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; + + #[test] + fn regex_location_file() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = "123-.*" + substitution_string = "123" + "#, + ("123-.*".to_string(), "123".to_string()), + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + location.backend().as_file().unwrap(); + ( + location.regex().as_str().to_string(), + location.substitution_string().to_string(), + ) + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn regex_location_s3() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = "123-.*" + substitution_string = "123" + backend.kind = "S3" + backend.bucket = "bucket" + "#, + ( + "123-.*".to_string(), + "123".to_string(), + "bucket".to_string(), + ), + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + let backend = location.backend().as_s3().unwrap(); + ( + location.regex().as_str().to_string(), + location.substitution_string().to_string(), + backend.bucket().to_string(), + ) + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn regex_location_url() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = "123-.*" + substitution_string = "123" + + backend.kind = "Url" + backend.url = "https://example.com" + "#, + ( + "123-.*".to_string(), + "123".to_string(), + "https://example.com/".to_string(), + ), + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + let url = location.backend().as_url().unwrap(); + + ( + location.regex().as_str().to_string(), + location.substitution_string().to_string(), + url.url().to_string(), + ) + }, + ); + } + + #[cfg(all(feature = "url-storage", feature = "s3-storage"))] + #[test] + fn regex_location_multiple() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = "prefix/(?P.*)$" + substitution_string = "$key" + backend.kind = "S3" + backend.bucket = "bucket" + backend.path_style = true + + [[locations]] + regex = ".*" + substitution_string = "$0" + backend.kind = "Url" + backend.url = "http://localhost:8080" + backend.forward_headers = false + "#, + ( + "prefix/(?P.*)$".to_string(), + "$key".to_string(), + "bucket".to_string(), + ".*".to_string(), + "$0".to_string(), + "http://localhost:8080/".to_string(), + ), + |result: Config| { + let location = result.locations.into_inner(); + let location_one = location[0].as_regex().unwrap(); + let s3 = location_one.backend().as_s3().unwrap(); + let location_two = location[1].as_regex().unwrap(); + let url = location_two.backend().as_url().unwrap(); + + ( + location_one.regex().as_str().to_string(), + location_one.substitution_string().to_string(), + s3.bucket().to_string(), + location_two.regex().as_str().to_string(), + location_two.substitution_string().to_string(), + url.url().to_string(), + ) + }, + ); + } +} diff --git a/htsget-config/src/config/advanced/url.rs b/htsget-config/src/config/advanced/url.rs new file mode 100644 index 00000000..d1daa3f5 --- /dev/null +++ b/htsget-config/src/config/advanced/url.rs @@ -0,0 +1,162 @@ +//! The config for remote URL server locations. +//! + +use crate::error::Error; +use crate::error::Error::ParseError; +use crate::error::Result; +use crate::storage; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; +use crate::tls::client::TlsClientConfig; +use cfg_if::cfg_if; +use http::Uri; +use reqwest::Client; +use serde::{Deserialize, Serialize}; + +/// Options for the remote URL server config. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub struct Url { + #[serde(with = "http_serde::uri")] + url: Uri, + #[serde(with = "http_serde::option::uri", default)] + response_url: Option, + #[serde(default = "default_forward_headers")] + forward_headers: bool, + #[serde(default)] + header_blacklist: Vec, + #[serde(skip_serializing, default)] + tls: TlsClientConfig, + #[cfg(feature = "experimental")] + #[serde(skip_serializing, default)] + keys: Option, +} + +impl Url { + /// Create a new url storage. + pub fn new( + url: Uri, + response_url: Option, + forward_headers: bool, + header_blacklist: Vec, + tls: TlsClientConfig, + ) -> Self { + Self { + url, + response_url, + forward_headers, + header_blacklist, + tls, + #[cfg(feature = "experimental")] + keys: None, + } + } + + /// Get the url called when resolving the query. + pub fn url(&self) -> &Uri { + &self.url + } + + /// Get the response url which is returned to the client. + pub fn response_url(&self) -> Option<&Uri> { + self.response_url.as_ref() + } + + /// Whether headers received in a query request should be + /// included in the returned data block tickets. + pub fn forward_headers(&self) -> bool { + self.forward_headers + } + + /// Get the tls client config. + pub fn tls(&self) -> &TlsClientConfig { + &self.tls + } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } +} + +impl TryFrom for storage::url::Url { + type Error = Error; + + fn try_from(storage: Url) -> Result { + let mut builder = Client::builder(); + + let (certs, identity) = storage.tls.into_inner(); + + if let Some(certs) = certs { + for cert in certs { + builder = builder.add_root_certificate(cert); + } + } + if let Some(identity) = identity { + builder = builder.identity(identity); + } + + let client = builder + .build() + .map_err(|err| ParseError(format!("building url storage client: {}", err)))?; + + let url_storage = Self::new( + storage.url.clone(), + storage.response_url.unwrap_or(storage.url), + storage.forward_headers, + storage.header_blacklist, + client, + ); + + cfg_if! { + if #[cfg(feature = "experimental")] { + Ok(url_storage.set_keys(storage.keys)) + } else { + Ok(url_storage) + } + } + } +} + +fn default_forward_headers() -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + #[test] + fn url_backend() { + test_serialize_and_deserialize( + r#" + url = "https://example.com" + response_url = "https://example.com" + forward_headers = false + header_blacklist = ["Host"] + "#, + ( + "https://example.com/".to_string(), + "https://example.com/".to_string(), + false, + vec!["Host".to_string()], + ), + |result: Url| { + ( + result.url().to_string(), + result.response_url().unwrap().to_string(), + result.forward_headers(), + result.header_blacklist, + ) + }, + ); + } +} diff --git a/htsget-config/src/config/data_server.rs b/htsget-config/src/config/data_server.rs new file mode 100644 index 00000000..bf89eff2 --- /dev/null +++ b/htsget-config/src/config/data_server.rs @@ -0,0 +1,128 @@ +//! Data server configuration. +//! + +use crate::config::advanced::cors::CorsConfig; +use crate::error::{Error::ParseError, Result}; +use crate::storage::file::{default_localstorage_addr, default_path}; +use crate::tls::TlsServerConfig; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +/// Tagged allow headers for cors config, either Mirror or Any. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum DataServerTagged { + #[serde(alias = "none", alias = "NONE", alias = "null")] + None, +} + +/// Whether the data server is enabled or not. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +#[allow(clippy::large_enum_variant)] +pub enum DataServerEnabled { + None(DataServerTagged), + Some(DataServerConfig), +} + +impl DataServerEnabled { + /// Get the data server config, or an error if `None`. + pub fn as_data_server_config(&self) -> Result<&DataServerConfig> { + if let Self::Some(config) = self { + Ok(config) + } else { + Err(ParseError("expected `None` variant".to_string())) + } + } +} + +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default, deny_unknown_fields)] +pub struct DataServerConfig { + addr: SocketAddr, + local_path: PathBuf, + #[serde(skip_serializing)] + tls: Option, + cors: CorsConfig, +} + +impl DataServerConfig { + /// Create the ticket server config. + pub fn new( + addr: SocketAddr, + local_path: PathBuf, + tls: Option, + cors: CorsConfig, + ) -> Self { + Self { + addr, + local_path, + tls, + cors, + } + } + + /// Get the socket address. + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Get the local path. + pub fn local_path(&self) -> &Path { + self.local_path.as_path() + } + + /// Get the TLS config. + pub fn tls(&self) -> Option<&TlsServerConfig> { + self.tls.as_ref() + } + + /// Get the CORS config. + pub fn cors(&self) -> &CorsConfig { + &self.cors + } + + /// Get the owned TLS config. + pub fn into_tls(self) -> Option { + self.tls + } +} + +impl Default for DataServerConfig { + fn default() -> Self { + Self { + addr: default_localstorage_addr() + .parse() + .expect("expected valid address"), + local_path: default_path().into(), + tls: Default::default(), + cors: Default::default(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn data_server() { + test_serialize_and_deserialize( + r#" + addr = "127.0.0.1:8083" + local_path = "path" + cors.max_age = 1 + "#, + ("127.0.0.1:8083".to_string(), "path".to_string(), 1), + |result: DataServerConfig| { + ( + result.addr().to_string(), + result.local_path().to_string_lossy().to_string(), + result.cors.max_age(), + ) + }, + ); + } +} diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs new file mode 100644 index 00000000..7ca847f4 --- /dev/null +++ b/htsget-config/src/config/location.rs @@ -0,0 +1,416 @@ +//! Storage location configuration. +//! + +use crate::config::advanced::regex_location::RegexLocation; +use crate::error::{Error::ParseError, Result}; +use crate::storage; +use crate::storage::file::default_authority; +use crate::storage::Backend; +use crate::types::Scheme; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize}; +use std::result; +#[cfg(feature = "url-storage")] +use {crate::config::advanced::url::Url, crate::error, http::Uri}; + +/// The locations of data. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default, deny_unknown_fields, from = "LocationsOneOrMany")] +pub struct Locations(Vec); + +impl Locations { + /// Create new locations. + pub fn new(locations: Vec) -> Self { + Self(locations) + } + + /// Get locations as a slice of `LocationEither`. + pub fn as_slice(&self) -> &[LocationEither] { + self.0.as_slice() + } + + /// Get locations as an owned vector of `LocationEither`. + pub fn into_inner(self) -> Vec { + self.0 + } + + /// Get locations as a mutable slice of `LocationEither`. + pub fn as_mut_slice(&mut self) -> &mut [LocationEither] { + self.0.as_mut_slice() + } +} + +impl Default for Locations { + fn default() -> Self { + Self(vec![Default::default()]) + } +} + +/// Either simple or regex based location +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +pub enum LocationEither { + Simple(Location), + Regex(RegexLocation), +} + +impl LocationEither { + /// Get the storage backend. + pub fn backend(&self) -> &Backend { + match self { + LocationEither::Simple(location) => location.backend(), + LocationEither::Regex(regex_location) => regex_location.backend(), + } + } + + /// Get the simple location variant, returning an error otherwise. + pub fn as_simple(&self) -> Result<&Location> { + if let LocationEither::Simple(simple) = self { + Ok(simple) + } else { + Err(ParseError("not a `Simple` variant".to_string())) + } + } + + /// Get the regex location variant, returning an error otherwise. + pub fn as_regex(&self) -> Result<&RegexLocation> { + if let LocationEither::Regex(regex) = self { + Ok(regex) + } else { + Err(ParseError("not a `Regex` variant".to_string())) + } + } +} + +impl Default for LocationEither { + fn default() -> Self { + Self::Simple(Default::default()) + } +} + +/// Location config. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default, from = "LocationWrapper", deny_unknown_fields)] +pub struct Location { + backend: Backend, + prefix: String, +} + +impl Location { + /// Create a new location. + pub fn new(backend: Backend, prefix: String) -> Self { + Self { backend, prefix } + } + + /// Get the storage backend. + pub fn backend(&self) -> &Backend { + &self.backend + } + + /// Get the prefix. + pub fn prefix(&self) -> &str { + &self.prefix + } +} + +/// Either a single or many locations +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +enum LocationsOneOrMany { + Many(Vec), + One(LocationEither), +} + +impl From for Locations { + fn from(locations: LocationsOneOrMany) -> Self { + match locations { + LocationsOneOrMany::One(location) => Self(vec![location]), + LocationsOneOrMany::Many(locations) => Self(locations), + } + } +} + +/// Deserialize the location from a string with a protocol. +#[derive(Serialize, Debug, Clone, Default)] +#[serde(default, deny_unknown_fields)] +struct StringLocation { + backend: Backend, + prefix: String, +} + +/// Deserialize the location from a map with regular field and values. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default, deny_unknown_fields)] +struct MapLocation { + backend: Backend, + prefix: String, +} + +/// A wrapper around location deserialization that can deserialize either a string +/// or a map. This is required so that default values behave correctly when deserializing +/// the `Location`. For example, if a location string isn't specified, the `Deserialize` +/// implementation for `StringLocation` can't account for this as it gets passed default values +/// which contain map elements. This wrapper allows deserializing using regular semantics by +/// falling back to the regular `MapLocation` derived deserializer. The reason there needs to be a +/// `StringLocation` and `MapLocation` type is so that `Location` can be deserialized using the +/// `from` attribute without recursion. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +enum LocationWrapper { + String(StringLocation), + Map(MapLocation), +} + +impl From for Location { + fn from(location: LocationWrapper) -> Self { + match location { + LocationWrapper::String(location) => Location::new(location.backend, location.prefix), + LocationWrapper::Map(location) => Location::new(location.backend, location.prefix), + } + } +} + +impl From for LocationEither { + fn from(location: Location) -> Self { + Self::Simple(location) + } +} + +impl<'de> Deserialize<'de> for StringLocation { + fn deserialize(deserializer: D) -> result::Result + where + D: Deserializer<'de>, + { + let split = |s: &str| { + let (s1, s2) = if let Some(split) = s.split_once("/").map(|(s1, s2)| { + ( + s1.to_string(), + s2.strip_suffix('/').unwrap_or(s2).to_string(), + ) + }) { + split + } else { + (s.to_string(), "".to_string()) + }; + + if s1.is_empty() { + Err(Error::custom("cannot have empty location")) + } else { + Ok((s1, s2)) + } + }; + + let s = String::deserialize(deserializer)?.to_lowercase(); + + if let Some(s) = s.strip_prefix("file://") { + let (path, prefix) = split(s)?; + return Ok(StringLocation { + backend: Backend::File(storage::file::File::new( + Scheme::Http, + default_authority(), + path.to_string(), + )), + prefix, + }); + } + + #[cfg(feature = "s3-storage")] + if let Some(s) = s.strip_prefix("s3://") { + let (bucket, prefix) = split(s)?; + return Ok(StringLocation { + backend: Backend::S3(storage::s3::S3::new(bucket.to_string(), None, false)), + prefix, + }); + } + + #[cfg(feature = "url-storage")] + if let Some(s_stripped) = s + .strip_prefix("http://") + .or_else(|| s.strip_prefix("https://")) + { + let (mut uri, prefix) = split(s_stripped)?; + + if s.starts_with("http://") { + uri = format!("http://{}", uri); + } + if s.starts_with("https://") { + uri = format!("https://{}", uri); + } + + let uri: Uri = uri.parse().map_err(Error::custom)?; + let url = Url::new(uri.clone(), Some(uri), true, vec![], Default::default()) + .try_into() + .map_err(|err: error::Error| Error::custom(err.to_string()))?; + + return Ok(StringLocation { + backend: Backend::Url(url), + prefix, + }); + } + + Err(Error::custom( + "expected file://, s3://, http:// or https:// scheme", + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; + + #[test] + fn location_single() { + test_serialize_and_deserialize( + r#" + locations = "file://path/prefix1" + "#, + ("path".to_string(), "prefix1".to_string()), + |result: Config| assert_file_location(result), + ); + test_serialize_and_deserialize( + r#" + locations = "file://path/prefix1/" + "#, + ("path".to_string(), "prefix1".to_string()), + |result: Config| assert_file_location(result), + ); + } + + #[test] + fn location_no_prefix() { + test_serialize_and_deserialize( + r#" + locations = "file://path" + "#, + ("path".to_string(), "".to_string()), + |result: Config| assert_file_location(result), + ); + test_serialize_and_deserialize( + r#" + locations = "file://path/" + "#, + ("path".to_string(), "".to_string()), + |result: Config| assert_file_location(result), + ); + } + + #[test] + fn location_file() { + test_serialize_and_deserialize( + r#" + locations = [ "file://path/prefix1", "file://path/prefix2" ] + "#, + ( + "path".to_string(), + "prefix1".to_string(), + "path".to_string(), + "prefix2".to_string(), + ), + |result: Config| { + let result = result.locations.0; + assert_eq!(result.len(), 2); + if let (LocationEither::Simple(location1), LocationEither::Simple(location2)) = + (result.first().unwrap(), result.get(1).unwrap()) + { + let file1 = location1.backend().as_file().unwrap(); + let file2 = location2.backend().as_file().unwrap(); + + return ( + file1.local_path().to_string(), + location1.prefix().to_string(), + file2.local_path().to_string(), + location2.prefix().to_string(), + ); + } + + panic!(); + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn location_s3() { + test_serialize_and_deserialize( + r#" + locations = [ "s3://bucket/prefix1", "s3://bucket/prefix2" ] + "#, + ( + "bucket".to_string(), + "prefix1".to_string(), + "bucket".to_string(), + "prefix2".to_string(), + ), + |result: Config| { + let result = result.locations.0; + assert_eq!(result.len(), 2); + if let (LocationEither::Simple(location1), LocationEither::Simple(location2)) = + (result.first().unwrap(), result.get(1).unwrap()) + { + if let (Backend::S3(s31), Backend::S3(s32)) = (location1.backend(), location2.backend()) { + return ( + s31.bucket().to_string(), + location1.prefix().to_string(), + s32.bucket().to_string(), + location2.prefix().to_string(), + ); + } + } + + panic!(); + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn location_url() { + test_serialize_and_deserialize( + r#" + locations = [ "https://example.com/prefix1", "http://example.com/prefix2" ] + "#, + ( + "https://example.com/".to_string(), + "prefix1".to_string(), + "http://example.com/".to_string(), + "prefix2".to_string(), + ), + |result: Config| { + let result = result.locations.0; + assert_eq!(result.len(), 2); + if let (LocationEither::Simple(location1), LocationEither::Simple(location2)) = + (result.first().unwrap(), result.get(1).unwrap()) + { + if let (Backend::Url(url1), Backend::Url(url2)) = + (location1.backend(), location2.backend()) + { + return ( + url1.url().to_string(), + location1.prefix().to_string(), + url2.url().to_string(), + location2.prefix().to_string(), + ); + } + } + + panic!(); + }, + ); + } + + fn assert_file_location(result: Config) -> (String, String) { + let result = result.locations.0; + assert_eq!(result.len(), 1); + if let LocationEither::Simple(location1) = result.first().unwrap() { + let file1 = location1.backend().as_file().unwrap(); + return ( + file1.local_path().to_string(), + location1.prefix().to_string(), + ); + } + + panic!(); + } +} diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 71750fe9..087bb4ca 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -1,50 +1,38 @@ +//! Structs to serialize and deserialize the htsget-rs config options. +//! + use std::fmt::Debug; use std::io; -use std::net::SocketAddr; use std::path::{Path, PathBuf}; +use crate::config::advanced::FormattingStyle; +use crate::config::data_server::DataServerEnabled; +use crate::config::location::{Location, LocationEither, Locations}; +use crate::config::parser::from_path; +use crate::config::service_info::ServiceInfo; +use crate::config::ticket_server::TicketServerConfig; +use crate::error::Error::{ArgParseError, ParseError, TracingError}; +use crate::error::Result; +use crate::storage::file::File; +use crate::storage::Backend; use clap::{Args as ClapArgs, Command, FromArgMatches, Parser}; -use http::header::HeaderName; -use http::Method; use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; -use tracing::instrument; use tracing::subscriber::set_global_default; use tracing_subscriber::fmt::{format, layer}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{EnvFilter, Registry}; -use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAllowTypes}; -use crate::config::parser::from_path; -use crate::config::FormattingStyle::{Compact, Full, Json, Pretty}; -use crate::error::Error::{ArgParseError, TracingError}; -use crate::error::Result; -use crate::resolver::Resolver; -use crate::tls::TlsServerConfig; - -pub mod cors; +pub mod advanced; +pub mod data_server; +pub mod location; pub mod parser; +pub mod service_info; +pub mod ticket_server; -/// Represents a usage string for htsget-rs. +/// The usage string for htsget-rs. pub const USAGE: &str = "To configure htsget-rs use a config file or environment variables. \ See the documentation of the htsget-config crate for more information."; -pub(crate) fn default_localstorage_addr() -> &'static str { - "127.0.0.1:8081" -} - -fn default_addr() -> &'static str { - "127.0.0.1:8080" -} - -fn default_server_origin() -> &'static str { - "http://localhost:8080" -} - -pub(crate) fn default_path() -> &'static str { - "./" -} - /// The command line arguments allowed for the htsget-rs executables. #[derive(Parser, Debug)] #[command(author, version, about, long_about = USAGE)] @@ -60,332 +48,75 @@ struct Args { print_default_config: bool, } -/// Determines which tracing formatting style to use. -#[derive(Debug, Copy, Clone, Serialize, Deserialize, Default)] -pub enum FormattingStyle { - #[default] - Full, - Compact, - Pretty, - Json, -} - -with_prefix!(ticket_server_prefix "ticket_server_"); -with_prefix!(data_server_prefix "data_server_"); -with_prefix!(cors_prefix "cors_"); - -/// Configuration for the htsget server. +/// Simplified config. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct Config { - formatting_style: FormattingStyle, - #[serde(flatten, with = "ticket_server_prefix")] ticket_server: TicketServerConfig, - #[serde(flatten, with = "data_server_prefix")] - data_server: DataServerConfig, - #[serde(flatten)] + data_server: DataServerEnabled, service_info: ServiceInfo, - resolvers: Vec, -} - -/// Configuration for the htsget ticket server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct TicketServerConfig { - addr: SocketAddr, - #[serde(skip_serializing)] - tls: Option, - #[serde(flatten, with = "cors_prefix")] - cors: CorsConfig, -} - -impl TicketServerConfig { - /// Create a new ticket server config. - pub fn new(addr: SocketAddr, tls: Option, cors: CorsConfig) -> Self { - Self { addr, tls, cors } - } - - /// Get the addr. - pub fn addr(&self) -> SocketAddr { - self.addr - } - - /// Get the TLS config. - pub fn tls(&self) -> Option<&TlsServerConfig> { - self.tls.as_ref() - } - - /// Get the TLS config. - pub fn into_tls(self) -> Option { - self.tls - } - - /// Get cors config. - pub fn cors(&self) -> &CorsConfig { - &self.cors - } - - /// Get allow credentials. - pub fn allow_credentials(&self) -> bool { - self.cors.allow_credentials() - } - - /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { - self.cors.allow_origins() - } - - /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { - self.cors.allow_headers() - } - - /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { - self.cors.allow_methods() - } - - /// Get max age. - pub fn max_age(&self) -> usize { - self.cors.max_age() - } - - /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { - self.cors.expose_headers() - } -} - -/// Configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct DataServerConfig { - enabled: bool, - addr: SocketAddr, - local_path: PathBuf, - serve_at: String, - #[serde(skip_serializing)] - tls: Option, - #[serde(flatten, with = "cors_prefix")] - cors: CorsConfig, + locations: Locations, + formatting_style: FormattingStyle, } -impl DataServerConfig { - /// Create a new data server config. +impl Config { + /// Create a config. pub fn new( - enabled: bool, - addr: SocketAddr, - local_path: PathBuf, - serve_at: String, - tls: Option, - cors: CorsConfig, + formatting_style: FormattingStyle, + ticket_server: TicketServerConfig, + data_server: DataServerEnabled, + service_info: ServiceInfo, + locations: Locations, ) -> Self { Self { - enabled, - addr, - local_path, - serve_at, - tls, - cors, - } - } - - /// Get the address. - pub fn addr(&self) -> SocketAddr { - self.addr - } - - /// Get the local path. - pub fn local_path(&self) -> &Path { - &self.local_path - } - - /// Get the serve at path. - pub fn serve_at(&self) -> &str { - &self.serve_at - } - - /// Get the TLS config. - pub fn tls(&self) -> Option<&TlsServerConfig> { - self.tls.as_ref() - } - - /// Get the TLS config. - pub fn into_tls(self) -> Option { - self.tls - } - - /// Get cors config. - pub fn cors(&self) -> &CorsConfig { - &self.cors - } - - /// Get allow credentials. - pub fn allow_credentials(&self) -> bool { - self.cors.allow_credentials() - } - - /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { - self.cors.allow_origins() - } - - /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { - self.cors.allow_headers() - } - - /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { - self.cors.allow_methods() - } - - /// Get the max age. - pub fn max_age(&self) -> usize { - self.cors.max_age() - } - - /// Get the expose headers. - pub fn expose_headers(&self) -> &AllowType { - self.cors.expose_headers() - } - - /// Is the data server disabled - pub fn enabled(&self) -> bool { - self.enabled - } -} - -impl Default for DataServerConfig { - fn default() -> Self { - Self { - enabled: true, - addr: default_localstorage_addr() - .parse() - .expect("expected valid address"), - local_path: default_path().into(), - serve_at: Default::default(), - tls: None, - cors: CorsConfig::default(), + formatting_style, + ticket_server, + data_server, + service_info, + locations, } } -} - -/// Configuration of the service info. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct ServiceInfo { - id: Option, - name: Option, - version: Option, - organization_name: Option, - organization_url: Option, - contact_url: Option, - documentation_url: Option, - created_at: Option, - updated_at: Option, - environment: Option, -} - -impl ServiceInfo { - /// Get the id. - pub fn id(&self) -> Option<&str> { - self.id.as_deref() - } - - /// Get the name. - pub fn name(&self) -> Option<&str> { - self.name.as_deref() - } - - /// Get the version. - pub fn version(&self) -> Option<&str> { - self.version.as_deref() - } - - /// Get the organization name. - pub fn organization_name(&self) -> Option<&str> { - self.organization_name.as_deref() - } - /// Get the organization url. - pub fn organization_url(&self) -> Option<&str> { - self.organization_url.as_deref() + /// Get the ticket server config. + pub fn formatting_style(&self) -> FormattingStyle { + self.formatting_style } - /// Get the contact url. - pub fn contact_url(&self) -> Option<&str> { - self.contact_url.as_deref() + /// Get the ticket server config. + pub fn ticket_server(&self) -> &TicketServerConfig { + &self.ticket_server } - /// Get the documentation url. - pub fn documentation_url(&self) -> Option<&str> { - self.documentation_url.as_deref() + /// Get the data server config. + pub fn data_server(&self) -> &DataServerEnabled { + &self.data_server } - /// Get created at. - pub fn created_at(&self) -> Option<&str> { - self.created_at.as_deref() + /// Get the service info config. + pub fn service_info(&self) -> &ServiceInfo { + &self.service_info } - /// Get updated at. - pub fn updated_at(&self) -> Option<&str> { - self.updated_at.as_deref() + /// Get the location. + pub fn locations(&self) -> &[LocationEither] { + self.locations.as_slice() } - /// Get environment. - pub fn environment(&self) -> Option<&str> { - self.environment.as_deref() - } -} - -impl Default for TicketServerConfig { - fn default() -> Self { - Self { - addr: default_addr().parse().expect("expected valid address"), - tls: None, - cors: CorsConfig::default(), - } - } -} - -impl Default for Config { - fn default() -> Self { - Self { - formatting_style: Full, - ticket_server: TicketServerConfig::default(), - data_server: DataServerConfig::default(), - service_info: ServiceInfo::default(), - resolvers: vec![Resolver::default()], - } - } -} - -impl Config { - /// Create a new config. - pub fn new( - formatting: FormattingStyle, - ticket_server: TicketServerConfig, - data_server: DataServerConfig, - service_info: ServiceInfo, - resolvers: Vec, - ) -> Self { - Self { - formatting_style: formatting, - ticket_server, - data_server, - service_info, - resolvers, - } + pub fn into_locations(self) -> Locations { + self.locations } /// Parse the command line arguments. Returns the config path, or prints the default config. /// Augment the `Command` args from the `clap` parser. Returns an error if the pub fn parse_args_with_command(augment_args: Command) -> Result> { - Ok(Self::parse_with_args( - Args::from_arg_matches(&Args::augment_args(augment_args).get_matches()) - .map_err(|err| ArgParseError(err.to_string()))?, - )) + let args = Args::from_arg_matches(&Args::augment_args(augment_args).get_matches()) + .map_err(|err| ArgParseError(err.to_string()))?; + + if args.config.as_ref().is_some_and(|path| !path.exists()) { + return Err(ParseError("config file not found".to_string())); + } + + Ok(Self::parse_with_args(args)) } /// Parse the command line arguments. Returns the config path, or prints the default config. @@ -406,11 +137,9 @@ impl Config { } /// Read a config struct from a TOML file. - #[instrument] pub fn from_path(path: &Path) -> io::Result { let config: Self = from_path(path)?; - - Ok(config.resolvers_from_data_server_config()) + Ok(config.resolvers_from_data_server_config()?) } /// Setup tracing, using a global subscriber. @@ -420,72 +149,65 @@ impl Config { let subscriber = Registry::default().with(env_filter); match self.formatting_style() { - Full => set_global_default(subscriber.with(layer())), - Compact => set_global_default(subscriber.with(layer().event_format(format().compact()))), - Pretty => set_global_default(subscriber.with(layer().event_format(format().pretty()))), - Json => set_global_default(subscriber.with(layer().event_format(format().json()))), + FormattingStyle::Full => set_global_default(subscriber.with(layer())), + FormattingStyle::Compact => { + set_global_default(subscriber.with(layer().event_format(format().compact()))) + } + FormattingStyle::Pretty => { + set_global_default(subscriber.with(layer().event_format(format().pretty()))) + } + FormattingStyle::Json => { + set_global_default(subscriber.with(layer().event_format(format().json()))) + } } .map_err(|err| TracingError(err.to_string()))?; Ok(()) } - /// Get the formatting style. - pub fn formatting_style(&self) -> FormattingStyle { - self.formatting_style - } - - /// Get the ticket server. - pub fn ticket_server(&self) -> &TicketServerConfig { - &self.ticket_server - } + /// Set the local resolvers from the data server config. + pub fn resolvers_from_data_server_config(mut self) -> Result { + self + .locations + .as_mut_slice() + .iter_mut() + .map(|location| { + if let LocationEither::Simple(simple) = location { + // Fall through only if the backend is File and default + let file_location = if let Ok(location) = simple.backend().as_file() { + location + } else { + return Ok(()); + }; - /// Get the data server. - pub fn data_server(&self) -> &DataServerConfig { - &self.data_server - } + if let DataServerEnabled::Some(ref data_server) = self.data_server { + let prefix = simple.prefix().to_string(); - /// Get the owned data server. - pub fn into_data_server(self) -> DataServerConfig { - self.data_server - } + // Don't update the local path as that comes in from the config. + let file: File = data_server.try_into()?; + let file = file.set_local_path(file_location.local_path().to_string()); - /// Get service info. - pub fn service_info(&self) -> &ServiceInfo { - &self.service_info - } + *location = LocationEither::Simple(Location::new(Backend::File(file), prefix)); + } + } - /// Get the resolvers. - pub fn resolvers(&self) -> &[Resolver] { - &self.resolvers - } + Ok(()) + }) + .collect::>>()?; - /// Get owned resolvers. - pub fn owned_resolvers(self) -> Vec { - self.resolvers + Ok(self) } +} - /// Set the local resolvers from the data server config. - pub fn resolvers_from_data_server_config(self) -> Self { - let Config { - formatting_style: formatting, - ticket_server, - data_server, - service_info, - mut resolvers, - } = self; - - resolvers - .iter_mut() - .for_each(|resolver| resolver.resolvers_from_data_server_config(&data_server)); - - Self::new( - formatting, - ticket_server, - data_server, - service_info, - resolvers, - ) +impl Default for Config { + fn default() -> Self { + Self { + formatting_style: FormattingStyle::Full, + ticket_server: Default::default(), + data_server: DataServerEnabled::Some(Default::default()), + service_info: Default::default(), + locations: Default::default(), + } } } @@ -493,14 +215,16 @@ impl Config { pub(crate) mod tests { use std::fmt::Display; + use super::*; use crate::config::parser::from_str; - use crate::storage::Storage; use crate::tls::tests::with_test_certificates; - use crate::types::Scheme::Http; + use crate::types::Scheme; use figment::Jail; use http::uri::Authority; - - use super::*; + #[cfg(feature = "url-storage")] + use http::Uri; + use serde::de::DeserializeOwned; + use serde_json::json; fn test_config(contents: Option<&str>, env_variables: Vec<(K, V)>, test_fn: F) where @@ -525,12 +249,14 @@ pub(crate) mod tests { test_fn( from_path::(path) .map_err(|err| err.to_string())? - .resolvers_from_data_server_config(), + .resolvers_from_data_server_config() + .map_err(|err| err.to_string())?, ); test_fn( from_str::(contents.unwrap_or("")) .map_err(|err| err.to_string())? - .resolvers_from_data_server_config(), + .resolvers_from_data_server_config() + .map_err(|err| err.to_string())?, ); Ok(()) @@ -553,6 +279,20 @@ pub(crate) mod tests { test_config(Some(contents), Vec::<(&str, &str)>::new(), test_fn); } + pub(crate) fn test_serialize_and_deserialize(input: &str, expected: T, get_result: F) + where + T: Debug + PartialEq, + F: Fn(D) -> T, + D: DeserializeOwned + Serialize + Clone, + { + let config: D = toml::from_str(input).unwrap(); + assert_eq!(expected, get_result(config.clone())); + + let serialized = toml::to_string(&config).unwrap(); + let deserialized = toml::from_str(&serialized).unwrap(); + assert_eq!(expected, get_result(deserialized)); + } + #[test] fn config_ticket_server_addr_env() { test_config_from_env( @@ -571,15 +311,15 @@ pub(crate) mod tests { test_config_from_env( vec![("HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS", true)], |config| { - assert!(config.ticket_server().allow_credentials()); + assert!(config.ticket_server().cors().allow_credentials()); }, ); } #[test] fn config_service_info_id_env() { - test_config_from_env(vec![("HTSGET_ID", "id")], |config| { - assert_eq!(config.service_info().id(), Some("id")); + test_config_from_env(vec![("HTSGET_SERVICE_INFO", "{ id=id }")], |config| { + assert_eq!(config.service_info().as_ref().get("id"), Some(&json!("id"))); }); } @@ -589,23 +329,21 @@ pub(crate) mod tests { vec![("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082")], |config| { assert_eq!( - config.data_server().addr(), + config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .addr(), "127.0.0.1:8082".parse().unwrap() ); }, ); } - #[test] - fn config_no_data_server_env() { - test_config_from_env(vec![("HTSGET_DATA_SERVER_ENABLED", "true")], |config| { - assert!(config.data_server().enabled()); - }); - } - #[test] fn config_ticket_server_addr_file() { - test_config_from_file(r#"ticket_server_addr = "127.0.0.1:8082""#, |config| { + test_config_from_file(r#"ticket_server.addr = "127.0.0.1:8082""#, |config| { assert_eq!( config.ticket_server().addr(), "127.0.0.1:8082".parse().unwrap() @@ -615,23 +353,28 @@ pub(crate) mod tests { #[test] fn config_ticket_server_cors_allow_origin_file() { - test_config_from_file(r#"ticket_server_cors_allow_credentials = true"#, |config| { - assert!(config.ticket_server().allow_credentials()); + test_config_from_file(r#"ticket_server.cors.allow_credentials = true"#, |config| { + assert!(config.ticket_server().cors().allow_credentials()); }); } #[test] fn config_service_info_id_file() { - test_config_from_file(r#"id = "id""#, |config| { - assert_eq!(config.service_info().id(), Some("id")); + test_config_from_file(r#"service_info.id = "id""#, |config| { + assert_eq!(config.service_info().as_ref().get("id"), Some(&json!("id"))); }); } #[test] fn config_data_server_addr_file() { - test_config_from_file(r#"data_server_addr = "127.0.0.1:8082""#, |config| { + test_config_from_file(r#"data_server.addr = "127.0.0.1:8082""#, |config| { assert_eq!( - config.data_server().addr(), + config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .addr(), "127.0.0.1:8082".parse().unwrap() ); }); @@ -646,12 +389,18 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - data_server_tls.key = "{}" + data_server.tls.key = "{}" "#, key_path.to_string_lossy().escape_default() ), |config| { - assert!(config.data_server().tls().is_none()); + assert!(config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .tls() + .is_none()); }, ); }); @@ -666,15 +415,20 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - data_server_tls.key = "{}" - data_server_tls.cert = "{}" + data_server.tls.key = "{}" + data_server.tls.cert = "{}" "#, key_path.to_string_lossy().escape_default(), cert_path.to_string_lossy().escape_default() ), |config| { - println!("{:?}", config.data_server().tls()); - assert!(config.data_server().tls().is_some()); + assert!(config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .tls() + .is_some()); }, ); }); @@ -692,7 +446,13 @@ pub(crate) mod tests { ("HTSGET_DATA_SERVER_TLS_CERT", cert_path.to_string_lossy()), ], |config| { - assert!(config.data_server().tls().is_some()); + assert!(config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .tls() + .is_some()); }, ); }); @@ -707,7 +467,7 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - ticket_server_tls.key = "{}" + ticket_server.tls.key = "{}" "#, key_path.to_string_lossy().escape_default() ), @@ -727,8 +487,8 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - ticket_server_tls.key = "{}" - ticket_server_tls.cert = "{}" + ticket_server.tls.key = "{}" + ticket_server.tls.cert = "{}" "#, key_path.to_string_lossy().escape_default(), cert_path.to_string_lossy().escape_default() @@ -759,31 +519,166 @@ pub(crate) mod tests { } #[test] - fn config_no_data_server_file() { - test_config_from_file(r#"data_server_enabled = true"#, |config| { - assert!(config.data_server().enabled()); - }); + fn locations_from_data_server_config() { + test_config_from_file( + r#" + data_server.addr = "127.0.0.1:8080" + data_server.local_path = "path" + + [[locations]] + regex = "123" + backend.kind = "File" + backend.local_path = "path" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let regex = config[0].as_regex().unwrap(); + assert!(matches!(regex.backend(), + Backend::File(file) if file.local_path() == "path" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8081"))); + }, + ); + } + + #[test] + fn simple_locations_env() { + test_config_from_env( + vec![ + ("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8080"), + ("HTSGET_LOCATIONS", "[file://data/bam, file://data/cram]"), + ], + |config| { + assert_multiple(config); + }, + ); + } + + #[test] + fn simple_locations() { + test_config_from_file( + r#" + data_server.addr = "127.0.0.1:8080" + data_server.local_path = "path" + + locations = "file://data" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), ""); + assert_file_location(location, "data"); + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn simple_locations_s3() { + test_config_from_file( + r#" + locations = "s3://bucket" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), ""); + assert!(matches!(location.backend(), + Backend::S3(s3) if s3.bucket() == "bucket")); + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn simple_locations_url() { + test_config_from_file( + r#" + locations = "https://example.com" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), ""); + assert!(matches!(location.backend(), + Backend::Url(url) if url.url() == &"https://example.com".parse::().unwrap())); + }, + ); + } + + #[test] + fn simple_locations_multiple() { + test_config_from_file( + r#" + data_server.addr = "127.0.0.1:8080" + locations = ["file://data/bam", "file://data/cram"] + "#, + |config| { + assert_multiple(config); + }, + ); } + #[cfg(feature = "s3-storage")] #[test] - fn resolvers_from_data_server_config() { + fn simple_locations_multiple_mixed() { test_config_from_file( r#" - data_server_addr = "127.0.0.1:8080" - data_server_local_path = "path" - data_server_serve_at = "/path" - - [[resolvers]] - [resolvers.storage] - backend = "Local" - use_data_server_config = true + data_server.addr = "127.0.0.1:8080" + data_server.local_path = "root" + locations = ["file://dir_one/bam", "file://dir_two/cram", "s3://bucket/vcf"] "#, |config| { - assert_eq!(config.resolvers.len(), 1); + assert_eq!(config.locations().len(), 3); + let config = config.locations.into_inner(); + + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), "bam"); + assert_file_location(location, "dir_one"); - assert!(matches!(config.resolvers.first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.local_path() == "path" && local_storage.scheme() == Http && local_storage.authority() == &Authority::from_static("127.0.0.1:8080") && local_storage.path_prefix() == "/path")); + let location = config[1].as_simple().unwrap(); + assert_eq!(location.prefix(), "cram"); + assert_file_location(location, "dir_two"); + + let location = config[2].as_simple().unwrap(); + assert_eq!(location.prefix(), "vcf"); + assert!(matches!(location.backend(), + Backend::S3(s3) if s3.bucket() == "bucket")); }, ); } + + #[test] + fn no_data_server() { + test_config_from_file( + r#" + data_server = "None" + "#, + |config| { + assert!(config.data_server().as_data_server_config().is_err()); + }, + ); + } + + fn assert_multiple(config: Config) { + assert_eq!(config.locations().len(), 2); + let config = config.locations.into_inner(); + + println!("{:#?}", config); + + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), "bam"); + assert_file_location(location, "data"); + + let location = config[1].as_simple().unwrap(); + assert_eq!(location.prefix(), "cram"); + assert_file_location(location, "data"); + } + + fn assert_file_location(location: &Location, local_path: &str) { + assert!(matches!(location.backend(), + Backend::File(file) if file.local_path() == local_path && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); + } } diff --git a/htsget-config/src/config/parser.rs b/htsget-config/src/config/parser.rs index 3734b0a4..77886921 100644 --- a/htsget-config/src/config/parser.rs +++ b/htsget-config/src/config/parser.rs @@ -1,3 +1,6 @@ +//! Parse config for a file and environment variables. +//! + use crate::config::Config; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; @@ -6,7 +9,7 @@ use std::fmt::Debug; use std::io; use std::io::ErrorKind; use std::path::Path; -use tracing::{info, instrument}; +use tracing::info; const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; @@ -19,7 +22,6 @@ pub enum Parser<'a> { impl Parser<'_> { /// Deserialize a string or path into a config value using Figment. - #[instrument] pub fn deserialize_config_into(&self) -> io::Result where for<'de> T: Deserialize<'de> + Debug, @@ -29,13 +31,17 @@ impl Parser<'_> { Parser::String(string) => Toml::string(string), Parser::Path(path) => Toml::file(path), }) - .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX).map(|k| match k { - k if k.as_str().to_lowercase().contains("tls_") => { - k.as_str().to_lowercase().replace("tls_", "tls.").into() - } - k => k.into(), + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX).map(|k| { + // This has to list all possible nested values to resolve issues with ambiguity when + // deserializing. E.g. see https://github.com/SergioBenitez/Figment/issues/12 + k.as_str() + .to_lowercase() + .replace("ticket_server_", "ticket_server.") + .replace("data_server_", "data_server.") + .replace("cors_", "cors.") + .replace("tls_", "tls.") + .into() })) - .merge(Env::raw()) .extract() .map_err(|err| io::Error::new(ErrorKind::Other, format!("failed to parse config: {err}")))?; @@ -46,7 +52,6 @@ impl Parser<'_> { } /// Read a deserializable config struct from a TOML file. -#[instrument] pub fn from_path(path: &Path) -> io::Result where for<'a> T: Deserialize<'a> + Debug, @@ -55,7 +60,6 @@ where } /// Read a deserializable config struct from a str. -#[instrument] pub fn from_str(str: &str) -> io::Result where for<'a> T: Deserialize<'a> + Debug, diff --git a/htsget-config/src/config/service_info.rs b/htsget-config/src/config/service_info.rs new file mode 100644 index 00000000..1f7d89d9 --- /dev/null +++ b/htsget-config/src/config/service_info.rs @@ -0,0 +1,80 @@ +//! Service info configuration. +//! + +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +/// Service info config. +#[derive(Serialize, Debug, Clone, Default, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct ServiceInfo(HashMap); + +impl ServiceInfo { + /// Create a service info. + pub fn new(fields: HashMap) -> Self { + Self(fields) + } + + /// Get the inner value. + pub fn into_inner(self) -> HashMap { + self.0 + } +} + +impl AsRef> for ServiceInfo { + fn as_ref(&self) -> &HashMap { + &self.0 + } +} + +impl<'de> Deserialize<'de> for ServiceInfo { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let fields: HashMap = HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| (key.to_lowercase(), value)) + .collect(); + + let err_msg = |invalid_key| format!("reserved service info field `{}`", invalid_key); + + if fields.contains_key("type") { + return Err(Error::custom(err_msg("type"))); + } + + if fields.contains_key("htsget") { + return Err(Error::custom(err_msg("htsget"))); + } + + Ok(ServiceInfo::new(fields)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; + use serde_json::json; + + #[test] + fn service_info() { + test_serialize_and_deserialize( + r#" + service_info.environment = "dev" + service_info.organization = { name = "name", url = "https://example.com/" } + "#, + HashMap::from_iter(vec![ + ("environment".to_string(), json!("dev")), + ( + "organization".to_string(), + json!({ "name": "name", "url": "https://example.com/" }), + ), + ]), + |result: Config| result.service_info.0, + ); + } +} diff --git a/htsget-config/src/config/ticket_server.rs b/htsget-config/src/config/ticket_server.rs new file mode 100644 index 00000000..e23b8643 --- /dev/null +++ b/htsget-config/src/config/ticket_server.rs @@ -0,0 +1,76 @@ +//! Ticket server configuration. +//! + +use crate::config::advanced::cors::CorsConfig; +use crate::tls::TlsServerConfig; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; + +/// Configuration for the htsget ticket server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default, deny_unknown_fields)] +pub struct TicketServerConfig { + addr: SocketAddr, + #[serde(skip_serializing)] + tls: Option, + cors: CorsConfig, +} + +impl TicketServerConfig { + /// Create the ticket server config. + pub fn new(addr: SocketAddr, tls: Option, cors: CorsConfig) -> Self { + Self { addr, tls, cors } + } + + /// Get the socket address. + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Get the TLS config. + pub fn tls(&self) -> Option<&TlsServerConfig> { + self.tls.as_ref() + } + + /// Get the CORS config. + pub fn cors(&self) -> &CorsConfig { + &self.cors + } + + /// Get the owned TLS config. + pub fn into_tls(self) -> Option { + self.tls + } +} + +impl Default for TicketServerConfig { + fn default() -> Self { + Self { + addr: default_addr().parse().expect("expected valid address"), + tls: Default::default(), + cors: Default::default(), + } + } +} + +fn default_addr() -> &'static str { + "127.0.0.1:8080" +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn data_server() { + test_serialize_and_deserialize( + r#" + addr = "127.0.0.1:8083" + cors.max_age = 1 + "#, + ("127.0.0.1:8083".to_string(), 1), + |result: TicketServerConfig| (result.addr().to_string(), result.cors.max_age()), + ); + } +} diff --git a/htsget-config/src/error.rs b/htsget-config/src/error.rs index 50eba8de..a6d89ac4 100644 --- a/htsget-config/src/error.rs +++ b/htsget-config/src/error.rs @@ -1,3 +1,6 @@ +//! Error types used by this crate. +//! + use std::{io, result}; use thiserror::Error; diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index dab3a434..b7183163 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -1,22 +1,15 @@ -use std::collections::HashSet; -use std::result; - +//! Resolvers map ids to storage locations. + +use crate::config::advanced::allow_guard::QueryAllowed; +use crate::config::advanced::regex_location::RegexLocation; +use crate::config::location::{LocationEither, Locations}; +use crate::storage; +use crate::storage::{Backend, ResolvedId}; +use crate::types::{Query, Response, Result}; use async_trait::async_trait; -use regex::{Error, Regex}; -use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; +use std::path::PathBuf; use tracing::instrument; -use crate::config::DataServerConfig; -use crate::storage::local::Local; -#[cfg(feature = "s3-storage")] -use crate::storage::s3::S3; -#[cfg(feature = "url-storage")] -use crate::storage::url::UrlStorageClient; -use crate::storage::{ResolvedId, Storage}; -use crate::types::Format::{Bam, Bcf, Cram, Vcf}; -use crate::types::{Class, Fields, Format, Interval, Query, Response, Result, TaggedTypeAll, Tags}; - /// A trait which matches the query id, replacing the match in the substitution text. pub trait IdResolver { /// Resolve the id, returning the substituted string if there is a match. @@ -26,16 +19,16 @@ pub trait IdResolver { /// A trait for determining the response from `Storage`. #[async_trait] pub trait ResolveResponse { - /// Convert from `LocalStorage`. - async fn from_local(local_storage: &Local, query: &Query) -> Result; + /// Convert from `File`. + async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result; - /// Convert from `S3Storage`. + /// Convert from `S3`. #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3, query: &Query) -> Result; + async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result; - /// Convert from `UrlStorage`. + /// Convert from `Url`. #[cfg(feature = "url-storage")] - async fn from_url(url_storage: &UrlStorageClient, query: &Query) -> Result; + async fn from_url(url_storage: &storage::url::Url, query: &Query) -> Result; } /// A trait which uses storage to resolve requests into responses. @@ -48,24 +41,6 @@ pub trait StorageResolver { ) -> Option>; } -/// Determines whether the query matches for use with the storage. -pub trait QueryAllowed { - /// Does this query match. - fn query_allowed(&self, query: &Query) -> bool; -} - -/// A regex storage is a storage that matches ids using Regex. -#[derive(Serialize, Debug, Clone, Deserialize)] -#[serde(default)] -pub struct Resolver { - #[serde(with = "serde_regex")] - regex: Regex, - // Todo: should match guard be allowed as variables inside the substitution string? - substitution_string: String, - storage: Storage, - allow_guard: AllowGuard, -} - /// A type which holds a resolved storage and an resolved id. #[derive(Debug)] pub struct ResolvedStorage { @@ -93,293 +68,48 @@ impl ResolvedStorage { } } -impl ResolvedId {} - -with_prefix!(allow_interval_prefix "allow_interval_"); - -/// A query guard represents query parameters that can be allowed to storage for a given query. -#[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(default)] -pub struct AllowGuard { - allow_reference_names: ReferenceNames, - allow_fields: Fields, - allow_tags: Tags, - allow_formats: Vec, - allow_classes: Vec, - #[serde(flatten, with = "allow_interval_prefix")] - allow_interval: Interval, -} - -/// Reference names that can be matched. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum ReferenceNames { - Tagged(TaggedTypeAll), - List(HashSet), -} - -impl AllowGuard { - /// Create a new allow guard. - pub fn new( - allow_reference_names: ReferenceNames, - allow_fields: Fields, - allow_tags: Tags, - allow_formats: Vec, - allow_classes: Vec, - allow_interval: Interval, - ) -> Self { - Self { - allow_reference_names, - allow_fields, - allow_tags, - allow_formats, - allow_classes, - allow_interval, - } - } - - /// Get allow formats. - pub fn allow_formats(&self) -> &[Format] { - &self.allow_formats - } - - /// Get allow classes. - pub fn allow_classes(&self) -> &[Class] { - &self.allow_classes - } - - /// Get allow interval. - pub fn allow_interval(&self) -> Interval { - self.allow_interval - } - - /// Get allow reference names. - pub fn allow_reference_names(&self) -> &ReferenceNames { - &self.allow_reference_names - } - - /// Get allow fields. - pub fn allow_fields(&self) -> &Fields { - &self.allow_fields - } - - /// Get allow tags. - pub fn allow_tags(&self) -> &Tags { - &self.allow_tags - } - - /// Set the allow reference names. - pub fn with_allow_reference_names(mut self, allow_reference_names: ReferenceNames) -> Self { - self.allow_reference_names = allow_reference_names; - self - } - - /// Set the allow fields. - pub fn with_allow_fields(mut self, allow_fields: Fields) -> Self { - self.allow_fields = allow_fields; - self - } - - /// Set the allow tags. - pub fn with_allow_tags(mut self, allow_tags: Tags) -> Self { - self.allow_tags = allow_tags; - self - } - - /// Set the allow formats. - pub fn with_allow_formats(mut self, allow_formats: Vec) -> Self { - self.allow_formats = allow_formats; - self - } - - /// Set the allow classes. - pub fn with_allow_classes(mut self, allow_classes: Vec) -> Self { - self.allow_classes = allow_classes; - self - } - - /// Set the allow interval. - pub fn with_allow_interval(mut self, allow_interval: Interval) -> Self { - self.allow_interval = allow_interval; - self - } -} - -impl Default for AllowGuard { - fn default() -> Self { - Self { - allow_formats: vec![Bam, Cram, Vcf, Bcf], - allow_classes: vec![Class::Body, Class::Header], - allow_interval: Default::default(), - allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), - allow_fields: Fields::Tagged(TaggedTypeAll::All), - allow_tags: Tags::Tagged(TaggedTypeAll::All), - } - } -} - -impl QueryAllowed for ReferenceNames { - fn query_allowed(&self, query: &Query) -> bool { - match (self, &query.reference_name()) { - (ReferenceNames::Tagged(TaggedTypeAll::All), _) => true, - (ReferenceNames::List(reference_names), Some(reference_name)) => { - reference_names.contains(*reference_name) - } - (ReferenceNames::List(_), None) => false, - } - } -} - -impl QueryAllowed for Fields { - fn query_allowed(&self, query: &Query) -> bool { - match (self, &query.fields()) { - (Fields::Tagged(TaggedTypeAll::All), _) => true, - (Fields::List(self_fields), Fields::List(query_fields)) => { - self_fields.is_subset(query_fields) - } - (Fields::List(_), Fields::Tagged(TaggedTypeAll::All)) => false, - } - } -} - -impl QueryAllowed for Tags { - fn query_allowed(&self, query: &Query) -> bool { - match (self, &query.tags()) { - (Tags::Tagged(TaggedTypeAll::All), _) => true, - (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), - (Tags::List(_), Tags::Tagged(TaggedTypeAll::All)) => false, - } - } -} - -impl QueryAllowed for AllowGuard { - fn query_allowed(&self, query: &Query) -> bool { - self.allow_formats.contains(&query.format()) - && self.allow_classes.contains(&query.class()) - && self - .allow_interval - .contains(query.interval().start().unwrap_or(u32::MIN)) - && self - .allow_interval - .contains(query.interval().end().unwrap_or(u32::MAX)) - && self.allow_reference_names.query_allowed(query) - && self.allow_fields.query_allowed(query) - && self.allow_tags.query_allowed(query) - } -} - -impl Default for Resolver { - fn default() -> Self { - Self::new(Storage::default(), ".*", "$0", AllowGuard::default()) - .expect("expected valid storage") - } -} - -impl Resolver { - /// Create a new regex storage. - pub fn new( - storage: Storage, - regex: &str, - replacement_string: &str, - allow_guard: AllowGuard, - ) -> result::Result { - Ok(Self { - regex: Regex::new(regex)?, - substitution_string: replacement_string.to_string(), - storage, - allow_guard, - }) - } - - /// Set the local resolvers from the data server config. - pub fn resolvers_from_data_server_config(&mut self, config: &DataServerConfig) { - match self.storage() { - Storage::Local(local) => { - if local.use_data_server_config() { - self.storage = Storage::Local(config.into()); - } - } - #[cfg(feature = "s3-storage")] - Storage::S3(_) => {} - #[cfg(feature = "url-storage")] - Storage::Url(_) => {} - } - } - - /// Get the match associated with the capture group at index `i` using the `regex_match`. - pub fn get_match<'a>(&'a self, i: usize, regex_match: &'a str) -> Option<&'a str> { - Some(self.regex().captures(regex_match)?.get(i)?.as_str()) - } - - /// Get the regex. - pub fn regex(&self) -> &Regex { - &self.regex - } - - /// Get the substitution string. - pub fn substitution_string(&self) -> &str { - &self.substitution_string - } - - /// Get the query guard. - pub fn allow_guard(&self) -> &AllowGuard { - &self.allow_guard - } - - /// Get the storage backend. - pub fn storage(&self) -> &Storage { - &self.storage - } - - /// Get allow formats. - pub fn allow_formats(&self) -> &[Format] { - self.allow_guard.allow_formats() - } - - /// Get allow classes. - pub fn allow_classes(&self) -> &[Class] { - self.allow_guard.allow_classes() - } - - /// Get allow interval. - pub fn allow_interval(&self) -> Interval { - self.allow_guard.allow_interval - } - - /// Get allow reference names. - pub fn allow_reference_names(&self) -> &ReferenceNames { - &self.allow_guard.allow_reference_names - } - - /// Get allow fields. - pub fn allow_fields(&self) -> &Fields { - &self.allow_guard.allow_fields - } - - /// Get allow tags. - pub fn allow_tags(&self) -> &Tags { - &self.allow_guard.allow_tags - } -} - -impl IdResolver for Resolver { +impl IdResolver for LocationEither { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(query.id()) && self.allow_guard.query_allowed(query) { + let replace = |regex_location: &RegexLocation| { Some(ResolvedId::new( - self - .regex - .replace(query.id(), &self.substitution_string) + regex_location + .regex() + .replace(query.id(), regex_location.substitution_string()) .to_string(), )) - } else { - None + }; + + match self { + LocationEither::Simple(location) => { + if query.id().starts_with(location.prefix()) { + return Some(ResolvedId::new( + PathBuf::from(location.prefix()) + .join(query.id()) + .to_str()? + .to_string(), + )); + } + } + LocationEither::Regex(regex_location) => { + if regex_location.regex().is_match(query.id()) { + if let Some(guard) = regex_location.guard() { + if guard.query_allowed(query) { + return replace(regex_location); + } + } + + return replace(regex_location); + } + } } + + None } } #[async_trait] -impl StorageResolver for Resolver { +impl StorageResolver for LocationEither { #[instrument(level = "trace", skip(self), ret)] async fn resolve_request( &self, @@ -390,41 +120,51 @@ impl StorageResolver for Resolver { query.set_id(resolved_id.into_inner()); - match self.storage() { - Storage::Local(local_storage) => Some(T::from_local(local_storage, query).await), + match self.backend() { + Backend::File(file) => Some(T::from_file(file, query).await), #[cfg(feature = "s3-storage")] - Storage::S3(s3_storage) => { - let first_match = self.get_match(1, &_matched_id); - let mut s3_storage = s3_storage.clone(); - if s3_storage.bucket.is_empty() { - s3_storage.bucket = first_match?.to_string(); - } + Backend::S3(s3) => { + let s3 = if let Self::Regex(regex_location) = self { + if s3.bucket().is_empty() { + let first_match = regex_location + .regex() + .captures(&_matched_id)? + .get(1)? + .as_str() + .to_string(); + &s3.clone().with_bucket(first_match) + } else { + s3 + } + } else { + s3 + }; - Some(T::from_s3(&s3_storage, query).await) + Some(T::from_s3(s3, query).await) } #[cfg(feature = "url-storage")] - Storage::Url(url_storage) => Some(T::from_url(url_storage, query).await), + Backend::Url(url_storage) => Some(T::from_url(url_storage, query).await), } } } -impl IdResolver for &[Resolver] { +impl IdResolver for &[LocationEither] { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - self.iter().find_map(|resolver| resolver.resolve_id(query)) + self.iter().find_map(|location| location.resolve_id(query)) } } #[async_trait] -impl StorageResolver for &[Resolver] { +impl StorageResolver for &[LocationEither] { #[instrument(level = "trace", skip(self), ret)] async fn resolve_request( &self, query: &mut Query, ) -> Option> { - for resolver in self.iter() { - if let Some(resolved_storage) = resolver.resolve_request::(query).await { - return Some(resolved_storage); + for location in self.iter() { + if let Some(location) = location.resolve_request::(query).await { + return Some(location); } } @@ -432,179 +172,207 @@ impl StorageResolver for &[Resolver] { } } +impl IdResolver for Locations { + #[instrument(level = "trace", skip(self), ret)] + fn resolve_id(&self, query: &Query) -> Option { + self.as_slice().resolve_id(query) + } +} + +#[async_trait] +impl StorageResolver for Locations { + #[instrument(level = "trace", skip(self), ret)] + async fn resolve_request( + &self, + query: &mut Query, + ) -> Option> { + self.as_slice().resolve_request::(query).await + } +} + #[cfg(test)] mod tests { + use super::*; + use crate::config::location::Location; + use crate::config::tests::{test_config_from_env, test_config_from_file}; + use crate::storage; + use crate::types::Format::Bam; + use crate::types::Scheme::Http; + use crate::types::Url; use http::uri::Authority; - #[cfg(feature = "url-storage")] + use reqwest::ClientBuilder; + #[cfg(feature = "s3-storage")] use { - crate::storage::url, crate::storage::url::ValidatedUrl, http::Uri as InnerUrl, - reqwest::ClientBuilder, std::str::FromStr, + crate::config::advanced::allow_guard::{AllowGuard, ReferenceNames}, + crate::types::{Class, Fields, Interval, Tags}, + std::collections::HashSet, }; - use crate::config::tests::{test_config_from_env, test_config_from_file}; - #[cfg(feature = "s3-storage")] - use crate::storage::s3::S3; - use crate::types::Scheme::Http; - use crate::types::Url; - - use super::*; - struct TestResolveResponse; #[async_trait] impl ResolveResponse for TestResolveResponse { - async fn from_local(local_storage: &Local, _: &Query) -> Result { + async fn from_file(file: &storage::file::File, query: &Query) -> Result { Ok(Response::new( Bam, - vec![Url::new(local_storage.authority().to_string())], + Self::format_url(file.authority().as_ref(), query.id()), )) } #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3, _: &Query) -> Result { - Ok(Response::new(Bam, vec![Url::new(s3_storage.bucket())])) + async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { + Ok(Response::new( + Bam, + Self::format_url(s3_storage.bucket(), query.id()), + )) } #[cfg(feature = "url-storage")] - async fn from_url(url_storage: &UrlStorageClient, _: &Query) -> Result { + async fn from_url(url: &storage::url::Url, query: &Query) -> Result { Ok(Response::new( Bam, - vec![Url::new(url_storage.url().to_string())], + Self::format_url(url.url().to_string().strip_suffix('/').unwrap(), query.id()), )) } } + impl TestResolveResponse { + fn format_url(prefix: &str, id: &str) -> Vec { + vec![Url::new(format!("{}/{}", prefix, id))] + } + } + #[tokio::test] async fn resolver_resolve_local_request() { - let local_storage = Local::new( + let file = storage::file::File::new( Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), - "/data".to_string(), - false, ); - let resolver = Resolver::new( - Storage::Local(local_storage), - "id", - "$0-test", - AllowGuard::default(), - ) - .unwrap(); - expected_resolved_request(resolver, "127.0.0.1:8080").await; + let regex_location = RegexLocation::new( + "id".parse().unwrap(), + "$0-test".to_string(), + Backend::File(file.clone()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.into()], "127.0.0.1:8080/id-test-1").await; + + let location = Location::new(Backend::File(file), "".to_string()); + expected_resolved_request(vec![location.into()], "127.0.0.1:8080/id-1").await; } #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { - let s3_storage = S3::new("id".to_string(), None, false); - let resolver = Resolver::new( - Storage::S3(s3_storage), - "(id)-1", - "$1-test", - AllowGuard::default(), - ) - .unwrap(); + let s3_storage = storage::s3::S3::new("id2".to_string(), None, false); + let regex_location = RegexLocation::new( + "(id)-1".parse().unwrap(), + "$1-test".to_string(), + Backend::S3(s3_storage.clone()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.into()], "id2/id-test").await; - expected_resolved_request(resolver, "id").await; + let location = Location::new(Backend::S3(s3_storage), "".to_string()); + expected_resolved_request(vec![location.into()], "id2/id-1").await; } #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request() { - let resolver = Resolver::new( - Storage::S3(S3::default()), - "(id)-1", - "$1-test", - AllowGuard::default(), - ) - .unwrap(); + let regex_location = RegexLocation::new( + "(id)-1".parse().unwrap(), + "$1-test".to_string(), + Backend::S3(storage::s3::S3::default()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.clone().into()], "id/id-test").await; - expected_resolved_request(resolver, "id").await; + let regex_location = RegexLocation::new( + "^(id)-(?P.*)$".parse().unwrap(), + "$key".to_string(), + Backend::S3(storage::s3::S3::default()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.clone().into()], "id/1").await; + + let location = Location::new( + Backend::S3(storage::s3::S3::new("bucket".to_string(), None, false)), + "".to_string(), + ); + expected_resolved_request(vec![location.into()], "bucket/id-1").await; } #[cfg(feature = "url-storage")] #[tokio::test] async fn resolver_resolve_url_request() { let client = ClientBuilder::new().build().unwrap(); - let url_storage = UrlStorageClient::new( - ValidatedUrl(url::Url { - inner: InnerUrl::from_str("https://example.com/").unwrap(), - }), - ValidatedUrl(url::Url { - inner: InnerUrl::from_str("https://example.com/").unwrap(), - }), + let url_storage = storage::url::Url::new( + "https://example.com/".parse().unwrap(), + "https://example.com/".parse().unwrap(), true, vec![], client, ); - let resolver = Resolver::new( - Storage::Url(url_storage), - "(id)-1", - "$1-test", - AllowGuard::default(), - ) - .unwrap(); - - expected_resolved_request(resolver, "https://example.com/").await; - } - - #[test] - fn resolver_get_matches() { - let resolver = Resolver::new( - Storage::default(), - "^(id)/(?P.*)$", - "$0", - AllowGuard::default(), + let regex_location = RegexLocation::new( + "(id)-1".parse().unwrap(), + "$1-test".to_string(), + Backend::Url(url_storage.clone()), + Default::default(), + ); + expected_resolved_request( + vec![regex_location.clone().into()], + "https://example.com/id-test", ) - .unwrap(); - let first_match = resolver.get_match(1, "id/key").unwrap(); + .await; - assert_eq!(first_match, "id"); + let location = Location::new(Backend::Url(url_storage), "".to_string()); + expected_resolved_request(vec![location.into()], "https://example.com/id-1").await; } #[test] - fn resolver_get_matches_no_captures() { - let resolver = - Resolver::new(Storage::default(), "^id/id$", "$0", AllowGuard::default()).unwrap(); - let first_match = resolver.get_match(1, "/id/key"); - - assert_eq!(first_match, None); - } + fn resolver_array_resolve_id() { + let resolver = Locations::new(vec![ + RegexLocation::new( + "^(id-1)(.*)$".parse().unwrap(), + "$1-test-1".to_string(), + Default::default(), + Default::default(), + ) + .into(), + RegexLocation::new( + "^(id-2)(.*)$".parse().unwrap(), + "$1-test-2".to_string(), + Default::default(), + Default::default(), + ) + .into(), + ]); - #[test] - fn resolver_resolve_id() { - let resolver = - Resolver::new(Storage::default(), "id", "$0-test", AllowGuard::default()).unwrap(); assert_eq!( resolver - .resolve_id(&Query::new_with_default_request("id", Bam)) + .as_slice() + .resolve_id(&Query::new_with_default_request("id-1", Bam)) .unwrap() .into_inner(), - "id-test" + "id-1-test-1" + ); + assert_eq!( + resolver + .as_slice() + .resolve_id(&Query::new_with_default_request("id-2", Bam)) + .unwrap() + .into_inner(), + "id-2-test-2" ); - } - #[test] - fn resolver_array_resolve_id() { - let resolver = vec![ - Resolver::new( - Storage::default(), - "^(id-1)(.*)$", - "$1-test-1", - AllowGuard::default(), - ) - .unwrap(), - Resolver::new( - Storage::default(), - "^(id-2)(.*)$", - "$1-test-2", - AllowGuard::default(), - ) - .unwrap(), - ]; + let resolver = Locations::new(vec![ + Location::new(Default::default(), "id-1".to_string()).into(), + Location::new(Default::default(), "id-2".to_string()).into(), + ]); assert_eq!( resolver @@ -612,7 +380,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-1", Bam)) .unwrap() .into_inner(), - "id-1-test-1" + "id-1/id-1" ); assert_eq!( resolver @@ -620,7 +388,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-2", Bam)) .unwrap() .into_inner(), - "id-2-test-2" + "id-2/id-2" ); } @@ -628,14 +396,15 @@ mod tests { fn config_resolvers_file() { test_config_from_file( r#" - [[resolvers]] + [[locations]] regex = "regex" "#, |config| { - assert_eq!( - config.resolvers().first().unwrap().regex().as_str(), - "regex" - ); + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.regex().as_str(), "regex"); + } else { + panic!(); + } }, ); } @@ -644,28 +413,30 @@ mod tests { fn config_resolvers_guard_file() { test_config_from_file( r#" - [[resolvers]] + [[locations]] regex = "regex" - [resolvers.allow_guard] + [locations.guard] allow_formats = ["BAM"] "#, |config| { - assert_eq!( - config.resolvers().first().unwrap().allow_formats(), - &vec![Bam] - ); + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.guard().unwrap().allow_formats(), &vec![Bam]); + } else { + panic!(); + } }, ); } #[test] fn config_resolvers_env() { - test_config_from_env(vec![("HTSGET_RESOLVERS", "[{regex=regex}]")], |config| { - assert_eq!( - config.resolvers().first().unwrap().regex().as_str(), - "regex" - ); + test_config_from_env(vec![("HTSGET_LOCATIONS", "[{regex=regex}]")], |config| { + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.regex().as_str(), "regex"); + } else { + panic!(); + } }); } @@ -674,12 +445,12 @@ mod tests { fn config_resolvers_all_options_env() { test_config_from_env( vec![( - "HTSGET_RESOLVERS", + "HTSGET_LOCATIONS", "[{ regex=regex, substitution_string=substitution_string, \ - storage={ backend=S3, bucket=bucket }, \ - allow_guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ - allow_formats=[BAM], allow_classes=[body], allow_interval_start=100, \ - allow_interval_end=1000 } }]", + backend={ kind=S3, bucket=bucket }, \ + guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ + allow_formats=[BAM], allow_classes=[body], allow_interval={ start=100, \ + end=1000 } } }]", )], |config| { let allow_guard = AllowGuard::new( @@ -690,9 +461,9 @@ mod tests { vec![Class::Body], Interval::new(Some(100), Some(1000)), ); - let resolver = config.resolvers().first().unwrap(); - let expected_storage = S3::new("bucket".to_string(), None, false); - let Storage::S3(storage) = resolver.storage() else { + let resolver = config.locations().first().unwrap(); + let expected_storage = storage::s3::S3::new("bucket".to_string(), None, false); + let Backend::S3(storage) = resolver.backend() else { panic!(); }; @@ -700,16 +471,20 @@ mod tests { assert_eq!(storage.endpoint(), expected_storage.endpoint()); assert_eq!(storage.path_style(), expected_storage.path_style()); - assert_eq!(resolver.regex().to_string(), "regex"); - assert_eq!(resolver.substitution_string(), "substitution_string"); - assert_eq!(resolver.allow_guard(), &allow_guard); + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.regex().to_string(), "regex"); + assert_eq!(regex.substitution_string(), "substitution_string"); + assert_eq!(regex.guard().unwrap(), &allow_guard); + } else { + panic!(); + } }, ); } - async fn expected_resolved_request(resolver: Resolver, expected_id: &str) { + async fn expected_resolved_request(resolver: Vec, expected_id: &str) { assert_eq!( - resolver + Locations::new(resolver) .resolve_request::(&mut Query::new_with_default_request("id-1", Bam)) .await .unwrap() diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index c33c8f8a..696abc8b 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -9,18 +9,16 @@ use std::path::PathBuf; /// Local C4GH key storage. #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] pub struct C4GHLocal { - private_key: PathBuf, - recipient_public_key: PathBuf, + private: PathBuf, + public: PathBuf, } impl C4GHLocal { /// Create a new local C4GH key storage. - pub fn new(private_key: PathBuf, recipient_public_key: PathBuf) -> Self { - Self { - private_key, - recipient_public_key, - } + pub fn new(private: PathBuf, public: PathBuf) -> Self { + Self { private, public } } } @@ -28,8 +26,8 @@ impl TryFrom for C4GHKeys { type Error = Error; fn try_from(local: C4GHLocal) -> Result { - let private_key = get_private_key(local.private_key, Ok("".to_string()))?; - let recipient_public_key = get_public_key(local.recipient_public_key)?; + let private_key = get_private_key(local.private, Ok("".to_string()))?; + let recipient_public_key = get_public_key(local.public)?; let handle = tokio::spawn(async move { Ok(C4GHKeys::from_key_pair(private_key, recipient_public_key)) }); @@ -42,7 +40,7 @@ impl TryFrom for C4GHKeys { mod tests { use crate::config::tests::test_config_from_file; use crate::config::Config; - use crate::storage::Storage; + use crate::storage::Backend; use std::fs::copy; use std::path::PathBuf; use tempfile::TempDir; @@ -70,33 +68,32 @@ mod tests { test_config_from_file( &format!( r#" - [[resolvers]] + [[locations]] regex = "regex" - [resolvers.storage] + [locations.backend] {} - [resolvers.storage.keys] - location = "Local" - private_key = "{}" - recipient_public_key = "{}" + [locations.backend.keys] + kind = "File" + private = "{}" + public = "{}" "#, storage_config, private_key.to_string_lossy(), recipient_public_key.to_string_lossy() ), |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); test_fn(config); }, ); } #[tokio::test] async fn config_local_storage_c4gh() { - test_c4gh_storage_config(r#"backend = "Local""#, |config| { + test_c4gh_storage_config(r#"kind = "File""#, |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.keys().is_some() + config.locations().first().unwrap().backend(), + Backend::File(file) if file.keys().is_some() )); }); } @@ -106,13 +103,13 @@ mod tests { async fn config_s3_storage_c4gh() { test_c4gh_storage_config( r#" - backend = "S3" + kind = "S3" bucket = "bucket" "#, |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(s3_storage) if s3_storage.keys().is_some() + config.locations().first().unwrap().backend(), + Backend::S3(s3) if s3.keys().is_some() )); }, ); @@ -123,15 +120,15 @@ mod tests { async fn config_url_storage_c4gh() { test_c4gh_storage_config( r#" - backend = "Url" + kind = "Url" url = "https://example.com/" response_url = "https://example.com/" forward_headers = false "#, |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Url(url_storage) if url_storage.keys().is_some() + config.locations().first().unwrap().backend(), + Backend::Url(url) if url.keys().is_some() )); }, ); diff --git a/htsget-config/src/storage/c4gh/mod.rs b/htsget-config/src/storage/c4gh/mod.rs index b6d93800..1ef59329 100644 --- a/htsget-config/src/storage/c4gh/mod.rs +++ b/htsget-config/src/storage/c4gh/mod.rs @@ -19,7 +19,7 @@ pub mod secrets_manager; /// Config for Crypt4GH keys. #[derive(Deserialize, Debug, Clone)] -#[serde(try_from = "Location")] +#[serde(try_from = "C4GHKeyLocation", deny_unknown_fields)] pub struct C4GHKeys { // Store a cloneable future so that it can be resolved outside serde. keys: Shared>>>, @@ -40,6 +40,7 @@ impl C4GHKeys { }] } + /// Construct from an existing join handle. pub fn from_join_handle(handle: JoinHandle>>) -> Self { Self { keys: handle.map(|value| value?).boxed().shared(), @@ -59,25 +60,25 @@ impl From for Error { } } -impl TryFrom for C4GHKeys { +impl TryFrom for C4GHKeys { type Error = Error; - fn try_from(location: Location) -> Result { + fn try_from(location: C4GHKeyLocation) -> Result { match location { - Location::Local(local) => local.try_into(), + C4GHKeyLocation::File(file) => file.try_into(), #[cfg(feature = "s3-storage")] - Location::SecretsManager(secrets_manager) => secrets_manager.try_into(), + C4GHKeyLocation::SecretsManager(secrets_manager) => secrets_manager.try_into(), } } } /// The location of C4GH keys. #[derive(Deserialize, Debug, Clone)] -#[serde(tag = "location", deny_unknown_fields)] +#[serde(tag = "kind", deny_unknown_fields)] #[non_exhaustive] -pub enum Location { - #[serde(alias = "local", alias = "LOCAL")] - Local(C4GHLocal), +pub enum C4GHKeyLocation { + #[serde(alias = "file", alias = "FILE")] + File(C4GHLocal), #[cfg(feature = "s3-storage")] #[serde(alias = "secretsmanager", alias = "SECRETSMANAGER")] SecretsManager(C4GHSecretsManager), diff --git a/htsget-config/src/storage/c4gh/secrets_manager.rs b/htsget-config/src/storage/c4gh/secrets_manager.rs index 030792e1..fcba0900 100644 --- a/htsget-config/src/storage/c4gh/secrets_manager.rs +++ b/htsget-config/src/storage/c4gh/secrets_manager.rs @@ -16,19 +16,20 @@ use tempfile::TempDir; /// C4GH secrets manager key storage. #[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub struct C4GHSecretsManager { - private_key: String, - recipient_public_key: String, + private: String, + public: String, #[serde(skip)] client: Option, } impl C4GHSecretsManager { /// Create a new C4GH secrets manager key storage. - pub fn new(private_key: String, recipient_public_key: String) -> Self { + pub fn new(private: String, public: String) -> Self { Self { - private_key, - recipient_public_key, + private, + public, client: None, } } @@ -68,10 +69,10 @@ impl C4GHSecretsManager { // Should not have to do this, but the Crypt4GH library expects a path. let tmp = TempDir::new()?; let private_key = tmp.path().join("private_key"); - Self::write_to_file(&private_key, self.private_key, &client).await?; + Self::write_to_file(&private_key, self.private, &client).await?; let recipient_public_key = tmp.path().join("public_key"); - Self::write_to_file(&recipient_public_key, self.recipient_public_key, &client).await?; + Self::write_to_file(&recipient_public_key, self.public, &client).await?; let private_key = get_private_key(private_key, Ok("".to_string()))?; let recipient_public_key = get_public_key(recipient_public_key)?; diff --git a/htsget-config/src/storage/file.rs b/htsget-config/src/storage/file.rs new file mode 100644 index 00000000..026aa888 --- /dev/null +++ b/htsget-config/src/storage/file.rs @@ -0,0 +1,133 @@ +//! Configuration of local file based storage. +//! + +use crate::config::data_server::DataServerConfig; +use crate::error::Error; +use crate::error::Error::ParseError; +use crate::error::Result; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; +use crate::tls::KeyPairScheme; +use crate::types::Scheme; +use http::uri::Authority; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +/// Local file based storage. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default, deny_unknown_fields)] +pub struct File { + scheme: Scheme, + #[serde(with = "http_serde::authority")] + authority: Authority, + local_path: String, + #[cfg(feature = "experimental")] + #[serde(skip_serializing)] + keys: Option, +} + +impl File { + /// Create a new local storage. + pub fn new(scheme: Scheme, authority: Authority, local_path: String) -> Self { + Self { + scheme, + authority, + local_path, + #[cfg(feature = "experimental")] + keys: None, + } + } + + /// Get the scheme. + pub fn scheme(&self) -> Scheme { + self.scheme + } + + /// Get the authority. + pub fn authority(&self) -> &Authority { + &self.authority + } + + /// Get the local path. + pub fn local_path(&self) -> &str { + &self.local_path + } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } + + /// Set the local path. + pub fn set_local_path(mut self, local_path: String) -> Self { + self.local_path = local_path; + self + } +} + +impl Default for File { + fn default() -> Self { + Self::new(Scheme::Http, default_authority(), default_path().into()) + } +} + +impl TryFrom<&DataServerConfig> for File { + type Error = Error; + + fn try_from(config: &DataServerConfig) -> Result { + Ok(Self::new( + config.tls().get_scheme(), + Authority::from_str(&config.addr().to_string()).map_err(|err| ParseError(err.to_string()))?, + config.local_path().to_string_lossy().to_string(), + )) + } +} + +pub(crate) fn default_authority() -> Authority { + Authority::from_static(default_localstorage_addr()) +} + +pub(crate) fn default_localstorage_addr() -> &'static str { + "127.0.0.1:8081" +} + +pub(crate) fn default_path() -> &'static str { + "./" +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn file_backend() { + test_serialize_and_deserialize( + r#" + scheme = "Https" + authority = "127.0.0.1:8083" + local_path = "path" + "#, + ( + "127.0.0.1:8083".to_string(), + Scheme::Https, + "path".to_string(), + ), + |result: File| { + ( + result.authority.to_string(), + result.scheme, + result.local_path, + ) + }, + ); + } +} diff --git a/htsget-config/src/storage/local.rs b/htsget-config/src/storage/local.rs deleted file mode 100644 index a514d916..00000000 --- a/htsget-config/src/storage/local.rs +++ /dev/null @@ -1,180 +0,0 @@ -use std::str::FromStr; - -use http::uri::Authority; -use serde::{Deserialize, Serialize}; - -use crate::config::{default_localstorage_addr, default_path, DataServerConfig}; -#[cfg(feature = "experimental")] -use crate::storage::c4gh::C4GHKeys; -use crate::tls::KeyPairScheme; -use crate::types::Scheme; - -pub(crate) fn default_authority() -> Authority { - Authority::from_static(default_localstorage_addr()) -} - -fn default_local_path() -> String { - default_path().into() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct Local { - scheme: Scheme, - #[serde(with = "http_serde::authority")] - authority: Authority, - local_path: String, - path_prefix: String, - use_data_server_config: bool, - #[serde(skip_serializing)] - #[cfg(feature = "experimental")] - keys: Option, -} - -impl Local { - /// Create a new local storage. - pub fn new( - scheme: Scheme, - authority: Authority, - local_path: String, - path_prefix: String, - use_data_server_config: bool, - ) -> Self { - Self { - scheme, - authority, - local_path, - path_prefix, - use_data_server_config, - #[cfg(feature = "experimental")] - keys: None, - } - } - - /// Get the scheme. - pub fn scheme(&self) -> Scheme { - self.scheme - } - - /// Get the authority. - pub fn authority(&self) -> &Authority { - &self.authority - } - - /// Get the local path. - pub fn local_path(&self) -> &str { - &self.local_path - } - - /// Get the path prefix. - pub fn path_prefix(&self) -> &str { - &self.path_prefix - } - - /// Get whether config should be inherited from the data server config. - pub fn use_data_server_config(&self) -> bool { - self.use_data_server_config - } - - #[cfg(feature = "experimental")] - /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { - self.keys = keys; - self - } - - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} - -impl Default for Local { - fn default() -> Self { - Self::new( - Scheme::Http, - default_authority(), - default_local_path(), - Default::default(), - false, - ) - } -} - -impl From<&DataServerConfig> for Local { - fn from(config: &DataServerConfig) -> Self { - Self::new( - config.tls().get_scheme(), - Authority::from_str(&config.addr().to_string()).expect("expected valid authority"), - config.local_path().to_string_lossy().to_string(), - config.serve_at().to_string(), - true, - ) - } -} - -#[cfg(test)] -mod tests { - use std::net::SocketAddr; - use std::path::PathBuf; - - use crate::config::cors::CorsConfig; - use crate::config::tests::test_config_from_file; - use crate::storage::Storage; - use crate::types::Scheme::Http; - - use super::*; - - #[test] - fn config_storage_local_file() { - test_config_from_file( - r#" - [[resolvers]] - regex = "regex" - - [resolvers.storage] - backend = "Local" - local_path = "path" - scheme = "HTTPS" - path_prefix = "path" - "#, - |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.local_path() == "path" && local_storage.scheme() == Scheme::Https && local_storage.path_prefix() == "path" - )); - }, - ); - } - - #[test] - fn local_storage_from_data_server_config() { - let data_server_config = DataServerConfig::new( - true, - SocketAddr::from_str("127.0.0.1:8080").unwrap(), - PathBuf::from("data"), - "/data".to_string(), - None, - CorsConfig::default(), - ); - let result: Local = (&data_server_config).into(); - let expected = Local::new( - Http, - Authority::from_static("127.0.0.1:8080"), - "data".to_string(), - "/data".to_string(), - true, - ); - - assert_eq!(result.scheme(), expected.scheme()); - assert_eq!(result.authority(), expected.authority()); - assert_eq!(result.local_path(), expected.local_path()); - assert_eq!(result.path_prefix(), expected.path_prefix()); - assert_eq!( - result.use_data_server_config(), - expected.use_data_server_config() - ); - } -} diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index d8d277dc..b8dba455 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,13 +1,19 @@ -use crate::storage::local::Local; +//! Storage backends. +//! + +#[cfg(any(feature = "url-storage", feature = "s3-storage"))] +use crate::error::Error; +use crate::error::Result; +use crate::storage::file::File; #[cfg(feature = "s3-storage")] use crate::storage::s3::S3; #[cfg(feature = "url-storage")] -use crate::storage::url::UrlStorageClient; +use crate::storage::url::Url; use serde::{Deserialize, Serialize}; #[cfg(feature = "experimental")] pub mod c4gh; -pub mod local; +pub mod file; #[cfg(feature = "s3-storage")] pub mod s3; #[cfg(feature = "url-storage")] @@ -31,45 +37,75 @@ impl ResolvedId { /// Specify the storage backend to use as config values. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "backend")] +#[serde(tag = "kind", deny_unknown_fields)] #[non_exhaustive] -pub enum Storage { - #[serde(alias = "local", alias = "LOCAL")] - Local(Local), +pub enum Backend { + #[serde(alias = "file", alias = "FILE")] + File(File), #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] S3(S3), #[cfg(feature = "url-storage")] #[serde(alias = "url", alias = "URL")] - Url(#[serde(skip_serializing)] UrlStorageClient), + Url(Url), } -impl Default for Storage { +impl Backend { + /// Get the file variant and error if it is not `File`. + pub fn as_file(&self) -> Result<&File> { + match self { + Backend::File(file) => Ok(file), + #[cfg(feature = "s3-storage")] + Backend::S3(_) => Err(Error::ParseError("not a `File` variant".to_string())), + #[cfg(feature = "url-storage")] + Backend::Url(_) => Err(Error::ParseError("not a `File` variant".to_string())), + } + } + + /// Get the file variant and error if it is not `S3`. + #[cfg(feature = "s3-storage")] + pub fn as_s3(&self) -> Result<&S3> { + if let Backend::S3(s3) = self { + Ok(s3) + } else { + Err(Error::ParseError("not a `S3` variant".to_string())) + } + } + + /// Get the url variant and error if it is not `Url`. + #[cfg(feature = "url-storage")] + pub fn as_url(&self) -> Result<&Url> { + if let Backend::Url(url) = self { + Ok(url) + } else { + Err(Error::ParseError("not a `File` variant".to_string())) + } + } +} + +impl Default for Backend { fn default() -> Self { - Self::Local(Default::default()) + Self::File(Default::default()) } } #[cfg(test)] pub(crate) mod tests { use crate::config::tests::{test_config_from_env, test_config_from_file}; - - use super::*; + use crate::storage::Backend; #[test] fn config_storage_tagged_local_file() { test_config_from_file( r#" - [[resolvers]] - [resolvers.storage] - backend = "Local" + [[locations]] regex = "regex" + backend.kind = "File" "#, |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local { .. } + config.locations().first().unwrap().backend(), + Backend::File { .. } )); }, ); @@ -78,14 +114,11 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_local_env() { test_config_from_env( - vec![( - "HTSGET_RESOLVERS", - "[{storage={ backend=Local, use_data_server_config=true}}]", - )], + vec![("HTSGET_LOCATIONS", "[{backend={ kind=File }}]")], |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local { .. } + config.locations().first().unwrap().backend(), + Backend::File { .. } )); }, ); @@ -96,16 +129,14 @@ pub(crate) mod tests { fn config_storage_tagged_s3_file() { test_config_from_file( r#" - [[resolvers]] - [resolvers.storage] - backend = "S3" + [[locations]] regex = "regex" + backend.kind = "S3" "#, |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(..) + config.locations().first().unwrap().backend(), + Backend::S3(..) )); }, ); @@ -115,11 +146,11 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_s3_env() { test_config_from_env( - vec![("HTSGET_RESOLVERS", "[{storage={ backend=S3 }}]")], + vec![("HTSGET_LOCATIONS", "[{backend={ kind=S3 }}]")], |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(..) + config.locations().first().unwrap().backend(), + Backend::S3(..) )); }, ); diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index 5be71ca6..7f18e218 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -1,16 +1,20 @@ +//! Configuration for storage on AWS S3. +//! + #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug, Default, Clone)] -#[serde(default)] +/// Configuration struct for S3 storage. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default, deny_unknown_fields)] pub struct S3 { - pub(crate) bucket: String, - pub(crate) endpoint: Option, - pub(crate) path_style: bool, - #[serde(skip_serializing)] + bucket: String, + endpoint: Option, + path_style: bool, #[cfg(feature = "experimental")] - pub(crate) keys: Option, + #[serde(skip_serializing)] + keys: Option, } impl S3 { @@ -30,19 +34,37 @@ impl S3 { &self.bucket } + /// Set the bucket. + pub fn with_bucket(mut self, bucket: String) -> Self { + self.bucket = bucket; + self + } + /// Get the endpoint pub fn endpoint(&self) -> Option<&str> { self.endpoint.as_deref() } + /// Set the endpoint. + pub fn with_endpoint(mut self, endpoint: String) -> Self { + self.endpoint = Some(endpoint); + self + } + /// Get the path style pub fn path_style(&self) -> bool { self.path_style } + /// Set the path style. + pub fn with_path_style(mut self, path_style: bool) -> Self { + self.path_style = path_style; + self + } + #[cfg(feature = "experimental")] /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { + pub fn with_keys(mut self, keys: Option) -> Self { self.keys = keys; self } @@ -56,26 +78,24 @@ impl S3 { #[cfg(test)] mod tests { - use crate::config::tests::test_config_from_file; - use crate::storage::Storage; + use super::*; + use crate::config::tests::test_serialize_and_deserialize; #[test] - fn config_storage_s3_file() { - test_config_from_file( + fn s3_backend() { + test_serialize_and_deserialize( r#" - [[resolvers]] - regex = "regex" - - [resolvers.storage] - backend = "S3" - bucket = "bucket" - "#, - |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(s3_storage) if s3_storage.bucket() == "bucket" - )); + bucket = "bucket" + endpoint = "127.0.0.1:8083" + path_style = true + "#, + ("127.0.0.1:8083".to_string(), "bucket".to_string(), true), + |result: S3| { + ( + result.endpoint.unwrap().to_string(), + result.bucket.to_string(), + result.path_style, + ) }, ); } diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index fe1dbbc1..2f9a9928 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -1,93 +1,35 @@ -use cfg_if::cfg_if; -use http::Uri as InnerUrl; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; -use std::str::FromStr; +//! Configuration for remote URL server storage. +//! -use crate::error::Error::ParseError; -use crate::error::{Error, Result}; +use crate::config::advanced; #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; -use crate::storage::local::default_authority; -use crate::tls::client::TlsClientConfig; - -fn default_url() -> InnerUrl { - InnerUrl::from_str(&format!("https://{}", default_authority())).expect("expected valid url") -} - -with_prefix!(client_auth_prefix "client_"); +use http::Uri; +use reqwest::Client; +use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct UrlStorage { - url: ValidatedUrl, - response_url: ValidatedUrl, +/// Remote URL server storage struct. +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(try_from = "advanced::url::Url", deny_unknown_fields)] +pub struct Url { + #[serde(with = "http_serde::uri")] + url: Uri, + #[serde(with = "http_serde::uri")] + response_url: Uri, forward_headers: bool, header_blacklist: Vec, #[serde(skip_serializing)] - tls: TlsClientConfig, - #[serde(skip_serializing)] - #[cfg(feature = "experimental")] - keys: Option, -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(try_from = "UrlStorage")] -pub struct UrlStorageClient { - url: ValidatedUrl, - response_url: ValidatedUrl, - forward_headers: bool, - header_blacklist: Vec, client: Client, #[cfg(feature = "experimental")] + #[serde(skip_serializing)] keys: Option, } -impl TryFrom for UrlStorageClient { - type Error = Error; - - fn try_from(storage: UrlStorage) -> Result { - let mut builder = Client::builder(); - - let (certs, identity) = storage.tls.into_inner(); - - if let Some(certs) = certs { - for cert in certs { - builder = builder.add_root_certificate(cert); - } - } - if let Some(identity) = identity { - builder = builder.identity(identity); - } - - let client = builder - .build() - .map_err(|err| ParseError(format!("building url storage client: {}", err)))?; - - let url_storage = Self::new( - storage.url, - storage.response_url, - storage.forward_headers, - storage.header_blacklist, - client, - ); - - cfg_if! { - if #[cfg(feature = "experimental")] { - Ok(url_storage.set_keys(storage.keys)) - } else { - Ok(url_storage) - } - } - } -} - -impl UrlStorageClient { +impl Url { /// Create a new url storage client. pub fn new( - url: ValidatedUrl, - response_url: ValidatedUrl, + url: Uri, + response_url: Uri, forward_headers: bool, header_blacklist: Vec, client: Client, @@ -104,13 +46,13 @@ impl UrlStorageClient { } /// Get the url called when resolving the query. - pub fn url(&self) -> &InnerUrl { - &self.url.0.inner + pub fn url(&self) -> &Uri { + &self.url } /// Get the response url to return to the client - pub fn response_url(&self) -> &InnerUrl { - &self.response_url.0.inner + pub fn response_url(&self) -> &Uri { + &self.response_url } /// Whether to forward headers in the url tickets. @@ -141,168 +83,3 @@ impl UrlStorageClient { self.keys.as_ref() } } - -/// A wrapper around `http::Uri` type which implements serialize and deserialize. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(transparent)] -pub(crate) struct Url { - #[serde(with = "http_serde::uri")] - pub(crate) inner: InnerUrl, -} - -/// A new type struct on top of `http::Uri` which only allows http or https schemes when deserializing. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(try_from = "Url")] -pub struct ValidatedUrl(pub(crate) Url); - -impl ValidatedUrl { - /// Get the inner url. - pub fn into_inner(self) -> InnerUrl { - self.0.inner - } -} - -impl TryFrom for ValidatedUrl { - type Error = Error; - - fn try_from(url: Url) -> Result { - match url.inner.scheme() { - Some(scheme) if scheme == "http" || scheme == "https" => Ok(Self(url)), - _ => Err(ParseError("url scheme must be http or https".to_string())), - } - } -} - -impl UrlStorage { - /// Create a new url storage. - pub fn new( - url: InnerUrl, - response_url: InnerUrl, - forward_headers: bool, - header_blacklist: Vec, - tls: TlsClientConfig, - ) -> Self { - Self { - url: ValidatedUrl(Url { inner: url }), - response_url: ValidatedUrl(Url { - inner: response_url, - }), - forward_headers, - header_blacklist, - tls, - #[cfg(feature = "experimental")] - keys: None, - } - } - - /// Get the url called when resolving the query. - pub fn url(&self) -> &InnerUrl { - &self.url.0.inner - } - - /// Get the response url which is returned to the client. - pub fn response_url(&self) -> &InnerUrl { - &self.url.0.inner - } - - /// Whether headers received in a query request should be - /// included in the returned data block tickets. - pub fn forward_headers(&self) -> bool { - self.forward_headers - } - - /// Get the tls client config. - pub fn tls(&self) -> &TlsClientConfig { - &self.tls - } - - #[cfg(feature = "experimental")] - /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { - self.keys = keys; - self - } - - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} - -impl Default for UrlStorage { - fn default() -> Self { - Self::new( - default_url(), - default_url(), - true, - vec![], - TlsClientConfig::default(), - ) - } -} - -#[cfg(test)] -mod tests { - use crate::config::tests::test_config_from_file; - use crate::storage::url::{UrlStorage, UrlStorageClient}; - use crate::storage::Storage; - use crate::tls::client::tests::client_config_from_path; - - use crate::tls::tests::with_test_certificates; - - use super::*; - - #[tokio::test] - async fn test_building_client() { - with_test_certificates(|path, _, _| { - let client_config = client_config_from_path(path); - let url_storage = UrlStorageClient::try_from(UrlStorage::new( - "https://example.com".parse::().unwrap(), - "https://example.com".parse::().unwrap(), - true, - vec![], - client_config, - )); - - assert!(url_storage.is_ok()); - }); - } - - #[test] - fn config_storage_url_file() { - with_test_certificates(|path, _, _| { - let key_path = path.join("key.pem"); - let cert_path = path.join("cert.pem"); - - test_config_from_file( - &format!( - r#" - [[resolvers]] - regex = "regex" - - [resolvers.storage] - backend = "Url" - url = "https://example.com/" - response_url = "https://example.com/" - forward_headers = false - tls.key = "{}" - tls.cert = "{}" - tls.root_store = "{}" - "#, - key_path.to_string_lossy().escape_default(), - cert_path.to_string_lossy().escape_default(), - cert_path.to_string_lossy().escape_default() - ), - |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Url(url_storage) if *url_storage.url() == "https://example.com/" - && !url_storage.forward_headers() - )); - }, - ); - }); - } -} diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs index e925a645..c1094cce 100644 --- a/htsget-config/src/tls/mod.rs +++ b/htsget-config/src/tls/mod.rs @@ -29,7 +29,7 @@ pub trait KeyPairScheme { /// A certificate and key pair used for TLS. Serialization is not implemented because there /// is no way to convert back to a `PathBuf`. #[derive(Deserialize, Debug, Clone)] -#[serde(try_from = "CertificateKeyPairPath")] +#[serde(try_from = "CertificateKeyPairPath", deny_unknown_fields)] pub struct TlsServerConfig { server_config: ServerConfig, } @@ -49,6 +49,7 @@ impl TlsServerConfig { /// The location of a certificate and key pair used for TLS. /// This is the path to the PEM formatted X.509 certificate and private key. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] pub struct CertificateKeyPairPath { cert: PathBuf, key: PathBuf, @@ -202,13 +203,12 @@ pub(crate) mod tests { use std::io::Cursor; use std::path::Path; + use super::*; use rcgen::generate_simple_self_signed; use rustls::crypto::aws_lc_rs; use rustls_pemfile::{certs, pkcs8_private_keys}; use tempfile::TempDir; - use super::*; - #[test] fn test_load_key() { with_test_certificates(|path, key, _| { diff --git a/htsget-config/src/types.rs b/htsget-config/src/types.rs index 872f5352..171d915c 100644 --- a/htsget-config/src/types.rs +++ b/htsget-config/src/types.rs @@ -1,3 +1,6 @@ +//! Types related to htsget like formats, reference names, classes or intervals. +//! + use std::collections::{HashMap, HashSet}; use std::fmt::{Debug, Display, Formatter}; use std::io::ErrorKind::Other; @@ -13,11 +16,12 @@ use tracing::instrument; use crate::error::Error; use crate::error::Error::ParseError; +/// The result type returning a `HtsGetError`. pub type Result = result::Result; /// An enumeration with all the possible formats. #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all(serialize = "UPPERCASE"))] +#[serde(rename_all(serialize = "UPPERCASE"), deny_unknown_fields)] pub enum Format { #[serde(alias = "bam", alias = "BAM")] Bam, @@ -31,6 +35,7 @@ pub enum Format { /// Todo allow these to be configurable. impl Format { + /// Get the file ending for the format. pub fn file_ending(&self) -> &str { match self { Format::Bam => ".bam", @@ -40,10 +45,12 @@ impl Format { } } + /// Get the file name including its ending. pub fn fmt_file(&self, id: &str) -> String { format!("{id}{}", self.file_ending()) } + /// Get the index file ending for this format. pub fn index_file_ending(&self) -> &str { match self { Format::Bam => ".bam.bai", @@ -53,10 +60,12 @@ impl Format { } } + /// Get the index file name including its ending. pub fn fmt_index(&self, id: &str) -> String { format!("{id}{}", self.index_file_ending()) } + /// Get the GZI index file ending for this format. pub fn gzi_index_file_ending(&self) -> io::Result<&str> { match self { Format::Bam => Ok(".bam.gzi"), @@ -69,6 +78,7 @@ impl Format { } } + /// Get the GZI index file name including its ending. pub fn fmt_gzi(&self, id: &str) -> io::Result { Ok(format!("{id}{}", self.gzi_index_file_ending()?)) } @@ -102,7 +112,7 @@ impl Display for Format { /// Class component of htsget response. #[derive(Copy, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -#[serde(rename_all(serialize = "lowercase"))] +#[serde(rename_all(serialize = "lowercase"), deny_unknown_fields)] pub enum Class { #[serde(alias = "header", alias = "HEADER")] Header, @@ -113,6 +123,7 @@ pub enum Class { /// An interval represents the start (0-based, inclusive) and end (0-based exclusive) ranges of the /// query. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct Interval { start: Option, end: Option, @@ -177,10 +188,12 @@ impl Interval { }) } + /// Start position. pub fn start(&self) -> Option { self.start } + /// End position. pub fn end(&self) -> Option { self.end } @@ -220,7 +233,7 @@ pub enum TaggedTypeAll { /// Possible values for the fields parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(untagged)] +#[serde(untagged, deny_unknown_fields)] pub enum Fields { /// Include all fields Tagged(TaggedTypeAll), @@ -230,7 +243,7 @@ pub enum Fields { /// Possible values for the tags parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(untagged)] +#[serde(untagged, deny_unknown_fields)] pub enum Tags { /// Include all tags Tagged(TaggedTypeAll), @@ -382,43 +395,53 @@ impl Query { self } + /// Id. pub fn id(&self) -> &str { &self.id } + /// Format. pub fn format(&self) -> Format { self.format } + /// Class. pub fn class(&self) -> Class { self.class } + /// Reference name. pub fn reference_name(&self) -> Option<&str> { self.reference_name.as_deref() } + /// Interval. pub fn interval(&self) -> Interval { self.interval } + /// Fields. pub fn fields(&self) -> &Fields { &self.fields } + /// Tags. pub fn tags(&self) -> &Tags { &self.tags } + /// No tags. pub fn no_tags(&self) -> &NoTags { &self.no_tags } + /// Request. pub fn request(&self) -> &Request { &self.request } } +/// Htsget specific errors. #[derive(Error, Debug, PartialEq, Eq)] pub enum HtsGetError { #[error("not found: {0}")] @@ -444,30 +467,37 @@ pub enum HtsGetError { } impl HtsGetError { + /// Create a `NotFound` error. pub fn not_found>(message: S) -> Self { Self::NotFound(message.into()) } + /// Create an `UnsupportedFormat` error. pub fn unsupported_format>(format: S) -> Self { Self::UnsupportedFormat(format.into()) } + /// Create an `InvalidInput` error. pub fn invalid_input>(message: S) -> Self { Self::InvalidInput(message.into()) } + /// Create an `InvalidRange` error. pub fn invalid_range>(message: S) -> Self { Self::InvalidRange(message.into()) } + /// Create an `IoError` error. pub fn io_error>(message: S) -> Self { Self::IoError(message.into()) } + /// Create a `ParseError` error. pub fn parse_error>(message: S) -> Self { Self::ParseError(message.into()) } + /// Create an `InternalError` error. pub fn internal_error>(message: S) -> Self { Self::InternalError(message.into()) } @@ -611,6 +641,7 @@ pub struct JsonResponse { } impl JsonResponse { + /// Create a new `JsonResponse`. pub fn new(htsget: Response) -> Self { Self { htsget } } @@ -630,6 +661,7 @@ pub struct Response { } impl Response { + /// Create a new `Response`. pub fn new(format: Format, urls: Vec) -> Self { Self { format, urls } } diff --git a/htsget-http/Cargo.toml b/htsget-http/Cargo.toml index f4fbb260..6603f12f 100644 --- a/htsget-http/Cargo.toml +++ b/htsget-http/Cargo.toml @@ -19,6 +19,7 @@ default = [] [dependencies] thiserror = "1" serde = { version = "1", features = ["derive"] } +serde_json = "1" http = "1" htsget-search = { version = "0.9.1", path = "../htsget-search", default-features = false } htsget-config = { version = "0.12.0", path = "../htsget-config", default-features = false } diff --git a/htsget-http/README.md b/htsget-http/README.md index b31a5f44..a3cc059d 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -35,8 +35,8 @@ and process it using [htsget-search] to return JSON HTTP responses. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp diff --git a/htsget-http/src/lib.rs b/htsget-http/src/lib.rs index 3ef99f1c..8130d1fb 100644 --- a/htsget-http/src/lib.rs +++ b/htsget-http/src/lib.rs @@ -2,18 +2,14 @@ use std::result; use std::str::FromStr; pub use error::{HtsGetError, Result}; -pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo as ConfigServiceInfo, TicketServerConfig, -}; -pub use htsget_config::storage::Storage; +pub use htsget_config::config::Config; use htsget_config::types::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::types::{Format, Query, Request, Response}; pub use http_core::{get, post}; pub use post_request::{PostRequest, Region}; use query_builder::QueryBuilder; pub use service_info::get_service_info_json; -pub use service_info::get_service_info_with; -pub use service_info::{Htsget, Organisation, ServiceInfo, Type}; +pub use service_info::{Htsget, ServiceInfo, Type}; mod error; mod http_core; @@ -84,14 +80,13 @@ mod tests { use std::collections::HashMap; use std::path::PathBuf; - use http::uri::Authority; - - use htsget_config::storage::local::Local as ConfigLocalStorage; + use htsget_config::storage; use htsget_config::types::{Headers, JsonResponse, Request, Scheme, Url}; use htsget_search::from_storage::HtsGetFromStorage; + use htsget_search::FileStorage; use htsget_search::HtsGet; - use htsget_search::LocalStorage; use htsget_search::Storage; + use http::uri::Authority; use super::*; @@ -245,7 +240,7 @@ mod tests { JsonResponse::from(Response::new( Vcf, vec![ - Url::new("http://127.0.0.1:8081/data/vcf/sample1-bcbio-cancer.vcf.gz".to_string()) + Url::new("http://127.0.0.1:8081/vcf/sample1-bcbio-cancer.vcf.gz".to_string()) .with_headers(headers), ], )) @@ -255,7 +250,7 @@ mod tests { JsonResponse::from(Response::new( Bam, vec![ - Url::new("http://127.0.0.1:8081/data/bam/htsnexus_test_NA12878.bam".to_string()) + Url::new("http://127.0.0.1:8081/bam/htsnexus_test_NA12878.bam".to_string()) .with_headers(headers), ], )) @@ -271,14 +266,12 @@ mod tests { fn get_searcher() -> impl HtsGet + Clone { HtsGetFromStorage::new(Storage::new( - LocalStorage::new( + FileStorage::new( get_base_path(), - ConfigLocalStorage::new( + storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8081"), "data".to_string(), - "/data".to_string(), - false, ), ) .unwrap(), diff --git a/htsget-http/src/service_info.rs b/htsget-http/src/service_info.rs index 5a529dea..b8af5d2a 100644 --- a/htsget-http/src/service_info.rs +++ b/htsget-http/src/service_info.rs @@ -1,42 +1,33 @@ +use htsget_config::config; +use htsget_config::types::Format; +use htsget_search::HtsGet; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; use tracing::debug; use tracing::instrument; -use htsget_config::types::Format; -use htsget_search::HtsGet; - -use crate::ConfigServiceInfo; use crate::Endpoint; const READS_FORMATS: [&str; 2] = ["BAM", "CRAM"]; const VARIANTS_FORMATS: [&str; 2] = ["VCF", "BCF"]; +const HTSGET_GROUP: &str = "org.ga4gh"; +const HTSGET_ARTIFACT: &str = "htsget"; +const HTSGET_VERSION: &str = "1.3.0"; + /// A struct representing the information that should be present in a service-info response. #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct ServiceInfo { - pub id: String, - pub name: String, - pub version: String, - pub organization: Organisation, + #[serde(flatten)] + pub fields: HashMap, #[serde(rename = "type")] pub service_type: Type, pub htsget: Htsget, - pub contact_url: String, - pub documentation_url: String, - pub created_at: String, - pub updated_at: String, - pub environment: String, } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct Organisation { - pub name: String, - pub url: String, -} - -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Type { pub group: String, @@ -44,6 +35,16 @@ pub struct Type { pub version: String, } +impl Default for Type { + fn default() -> Self { + Self { + group: HTSGET_GROUP.to_string(), + artifact: HTSGET_ARTIFACT.to_string(), + version: HTSGET_VERSION.to_string(), + } + } +} + #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct Htsget { @@ -53,42 +54,37 @@ pub struct Htsget { pub tags_parameters_effective: bool, } -pub fn get_service_info_with( - endpoint: Endpoint, - supported_formats: &[Format], - fields_effective: bool, - tags_effective: bool, -) -> ServiceInfo { - let htsget_info = Htsget { - datatype: match endpoint { - Endpoint::Reads => "reads", - Endpoint::Variants => "variants", - } - .to_string(), - formats: supported_formats - .iter() - .map(|format| format.to_string()) - .filter(|format| match endpoint { - Endpoint::Reads => READS_FORMATS.contains(&format.as_str()), - Endpoint::Variants => VARIANTS_FORMATS.contains(&format.as_str()), - }) - .collect(), - fields_parameters_effective: fields_effective, - tags_parameters_effective: tags_effective, - }; +impl ServiceInfo { + pub fn new( + endpoint: Endpoint, + supported_formats: &[Format], + fields_effective: bool, + tags_effective: bool, + fields: HashMap, + ) -> Self { + let htsget_info = Htsget { + datatype: match endpoint { + Endpoint::Reads => "reads", + Endpoint::Variants => "variants", + } + .to_string(), + formats: supported_formats + .iter() + .map(|format| format.to_string()) + .filter(|format| match endpoint { + Endpoint::Reads => READS_FORMATS.contains(&format.as_str()), + Endpoint::Variants => VARIANTS_FORMATS.contains(&format.as_str()), + }) + .collect(), + fields_parameters_effective: fields_effective, + tags_parameters_effective: tags_effective, + }; - ServiceInfo { - id: "".to_string(), - name: "".to_string(), - version: "".to_string(), - organization: Default::default(), - service_type: Default::default(), - htsget: htsget_info, - contact_url: "".to_string(), - documentation_url: "".to_string(), - created_at: "".to_string(), - updated_at: "".to_string(), - environment: "".to_string(), + Self { + fields, + service_type: Default::default(), + htsget: htsget_info, + } } } @@ -96,55 +92,14 @@ pub fn get_service_info_with( pub fn get_service_info_json( endpoint: Endpoint, searcher: impl HtsGet + Send + Sync + 'static, - config: &ConfigServiceInfo, + config: config::service_info::ServiceInfo, ) -> ServiceInfo { debug!(endpoint = ?endpoint,"getting service-info response for endpoint"); - fill_out_service_info_json( - get_service_info_with( - endpoint, - &searcher.get_supported_formats(), - searcher.are_field_parameters_effective(), - searcher.are_tag_parameters_effective(), - ), - config, + ServiceInfo::new( + endpoint, + &searcher.get_supported_formats(), + searcher.are_field_parameters_effective(), + searcher.are_tag_parameters_effective(), + config.into_inner(), ) } - -/// Fills the service-info json with the data from the server config -fn fill_out_service_info_json( - mut service_info_json: ServiceInfo, - config: &ConfigServiceInfo, -) -> ServiceInfo { - if let Some(id) = config.id() { - service_info_json.id = id.to_string(); - } - if let Some(name) = config.name() { - service_info_json.name = name.to_string(); - } - if let Some(version) = config.version() { - service_info_json.version = version.to_string(); - } - if let Some(organization_name) = config.organization_name() { - service_info_json.organization.name = organization_name.to_string(); - } - if let Some(organization_url) = config.organization_url() { - service_info_json.organization.url = organization_url.to_string(); - } - if let Some(contact_url) = config.contact_url() { - service_info_json.contact_url = contact_url.to_string(); - } - if let Some(documentation_url) = config.documentation_url() { - service_info_json.documentation_url = documentation_url.to_string(); - } - if let Some(created_at) = config.created_at() { - service_info_json.created_at = created_at.to_string(); - } - if let Some(updated_at) = config.updated_at() { - service_info_json.updated_at = updated_at.to_string(); - } - if let Some(environment) = config.environment() { - service_info_json.environment = environment.to_string(); - } - - service_info_json -} diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index d1058763..d2322781 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -42,8 +42,8 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-lambda/src/main.rs b/htsget-lambda/src/main.rs index 562c7a73..4ad90c38 100644 --- a/htsget-lambda/src/main.rs +++ b/htsget-lambda/src/main.rs @@ -26,7 +26,7 @@ async fn main() -> Result<(), Error> { let service_info = config.service_info().clone(); let cors = config.ticket_server().cors().clone(); - let router = TicketServer::router(config.owned_resolvers(), service_info, cors); + let router = TicketServer::router(config.into_locations(), service_info, cors); run(router).await } else { diff --git a/htsget-search/README.md b/htsget-search/README.md index a849a3d4..27149f16 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -53,8 +53,8 @@ used to process requests. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index aab8c499..54bc6c82 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -6,7 +6,7 @@ use http::uri::Authority; use tokio::runtime::Runtime; use htsget_config::resolver::ResolveResponse; -use htsget_config::storage::local::Local as ConfigLocalStorage; +use htsget_config::storage; use htsget_config::types::Class::Header; use htsget_config::types::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::types::{HtsGetError, Query, Scheme}; @@ -16,13 +16,11 @@ const BENCHMARK_DURATION_SECONDS: u64 = 30; const NUMBER_OF_SAMPLES: usize = 50; async fn perform_query(query: Query) -> Result<(), HtsGetError> { - HtsGetFromStorage::from_local( - &ConfigLocalStorage::new( + HtsGetFromStorage::from_file( + &storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8081"), "../data".to_string(), - "/data".to_string(), - false, ), &query, ) diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 1fc2ac1b..4d80587a 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -641,7 +641,7 @@ pub(crate) mod tests { } pub(crate) fn expected_url() -> String { - "http://127.0.0.1:8081/data/htsnexus_test_NA12878.bam".to_string() + "http://127.0.0.1:8081/htsnexus_test_NA12878.bam".to_string() } pub(crate) fn expected_eof_url() -> Url { diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index c52c373c..d4431839 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -463,6 +463,6 @@ mod tests { } fn expected_url(name: &str) -> String { - format!("http://127.0.0.1:8081/data/{name}.bcf") + format!("http://127.0.0.1:8081/{name}.bcf") } } diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index 07251f3f..86121085 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -648,7 +648,7 @@ mod tests { } fn expected_url() -> String { - "http://127.0.0.1:8081/data/htsnexus_test_NA12878.cram".to_string() + "http://127.0.0.1:8081/htsnexus_test_NA12878.cram".to_string() } pub(crate) fn expected_eof_url() -> Url { diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index d2d54bc6..ca72580b 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -1,19 +1,7 @@ //! Module providing an implementation of the [HtsGet] trait using a [StorageTrait]. //! -use async_trait::async_trait; -use tracing::debug; -use tracing::instrument; - -use htsget_config::resolver::{ResolveResponse, StorageResolver}; -use htsget_config::storage::local::Local as LocalStorageConfig; -#[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3 as S3StorageConfig; -#[cfg(feature = "url-storage")] -use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; - use crate::search::Search; -use crate::Resolver; use crate::{ bam_search::BamSearch, bcf_search::BcfSearch, @@ -22,7 +10,13 @@ use crate::{ {HtsGet, Query, Response, Result}, }; use crate::{Format, HtsGetError}; +use async_trait::async_trait; +use htsget_config::config::location::Locations; +use htsget_config::resolver::{ResolveResponse, StorageResolver}; +use htsget_config::storage; use htsget_storage::Storage; +use tracing::debug; +use tracing::instrument; /// Implementation of the [HtsGet] trait using a [StorageTrait]. #[derive(Debug, Clone)] @@ -31,14 +25,7 @@ pub struct HtsGetFromStorage { } #[async_trait] -impl HtsGet for Vec { - async fn search(self, query: Query) -> Result { - self.as_slice().search(query).await - } -} - -#[async_trait] -impl HtsGet for &[Resolver] { +impl HtsGet for Locations { async fn search(self, mut query: Query) -> Result { self .resolve_request::(&mut query) @@ -63,24 +50,21 @@ impl HtsGet for HtsGetFromStorage { #[async_trait] impl ResolveResponse for HtsGetFromStorage { - async fn from_local( - local_storage_config: &LocalStorageConfig, - query: &Query, - ) -> Result { - let storage = Storage::from_local(local_storage_config).await?; + async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result { + let storage = Storage::from_file(file_storage).await?; let searcher = HtsGetFromStorage::new(storage); searcher.search(query.clone()).await } #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3StorageConfig, query: &Query) -> Result { + async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { let storage = Storage::from_s3(s3_storage).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await } #[cfg(feature = "url-storage")] - async fn from_url(url_storage_config: &UrlStorageConfig, query: &Query) -> Result { + async fn from_url(url_storage_config: &storage::url::Url, query: &Query) -> Result { let storage = Storage::from_url(url_storage_config).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await @@ -111,16 +95,17 @@ pub(crate) mod tests { htsget_storage::s3::S3Storage, htsget_test::aws_mocks::with_s3_test_server, std::fs::create_dir, }; - use http::uri::Authority; - use tempfile::TempDir; - + use htsget_config::config::location::{Location, LocationEither}; use htsget_config::storage; + use htsget_config::storage::Backend; use htsget_config::types::Class::Body; use htsget_config::types::Scheme::Http; - use htsget_storage::local::LocalStorage; + use htsget_storage::local::FileStorage; #[cfg(feature = "experimental")] use htsget_test::c4gh::decrypt_data; use htsget_test::http::concat::ConcatResponse; + use http::uri::Authority; + use tempfile::TempDir; use crate::bam_search::tests::{ expected_url as bam_expected_url, with_local_storage as with_bam_local_storage, BAM_FILE_NAME, @@ -178,7 +163,7 @@ pub(crate) mod tests { |_, local_storage| async move { let filename = "spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Vcf); - let response = HtsGetFromStorage::from_local(&local_storage, &query).await; + let response = HtsGetFromStorage::from_file(&local_storage, &query).await; assert_eq!(response, expected_vcf_response(filename)); @@ -197,17 +182,14 @@ pub(crate) mod tests { async fn search_resolvers() { with_config_local_storage( |_, local_storage| async { - let resolvers = vec![Resolver::new( - storage::Storage::Local(local_storage), - ".*", - "$0", - Default::default(), - ) - .unwrap()]; + let locations = Locations::new(vec![LocationEither::Simple(Location::new( + Backend::File(local_storage), + "".to_string(), + ))]); let filename = "spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Vcf); - let response = resolvers.search(query).await; + let response = locations.search(query).await; assert_eq!(response, expected_vcf_response(filename)); @@ -253,7 +235,7 @@ pub(crate) mod tests { copy_files: &[&str], map: M, ) where - F: FnOnce(PathBuf, LocalStorageConfig) -> Fut, + F: FnOnce(PathBuf, storage::file::File) -> Fut, Fut: Future>, M: FnOnce(&[u8]) -> Vec, { @@ -263,12 +245,10 @@ pub(crate) mod tests { println!("{:#?}", base_path); let response = test( base_path.clone(), - LocalStorageConfig::new( + storage::file::File::new( Http, Authority::from_static("127.0.0.1:8081"), base_path.to_str().unwrap().to_string(), - "/data".to_string(), - false, ), ) .await; @@ -278,7 +258,7 @@ pub(crate) mod tests { async fn with_config_local_storage(test: F, path: &str, copy_files: &[&str]) where - F: FnOnce(PathBuf, LocalStorageConfig) -> Fut, + F: FnOnce(PathBuf, storage::file::File) -> Fut, Fut: Future>, { with_config_local_storage_map(test, path, copy_files, |b| b.to_vec()).await; @@ -308,7 +288,7 @@ pub(crate) mod tests { with_config_local_storage( |base_path, local_storage| async { test(Storage::new( - LocalStorage::new(base_path, local_storage).unwrap(), + FileStorage::new(base_path, local_storage).unwrap(), )) .await }, @@ -327,7 +307,7 @@ pub(crate) mod tests { with_config_local_storage_map( |base_path, local_storage| async { test(Storage::new( - LocalStorage::new(base_path, local_storage).unwrap(), + FileStorage::new(base_path, local_storage).unwrap(), )) .await }, diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 4cd848cf..620ecb5e 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -3,17 +3,14 @@ //! Based on the [HtsGet Specification](https://samtools.github.io/hts-specs/htsget.html). //! -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; -pub use htsget_config::resolver::{ - IdResolver, QueryAllowed, ResolveResponse, Resolver, StorageResolver, -}; -pub use htsget_config::storage::Storage as ConfigStorage; +pub use htsget_config::config::Config; +pub use htsget_config::resolver::{IdResolver, ResolveResponse, StorageResolver}; pub use htsget_config::types::{ Class, Format, Headers, HtsGetError, JsonResponse, Query, Response, Result, Url, }; pub use htsget_storage::Storage; -pub use htsget_storage::local::LocalStorage; +pub use htsget_storage::local::FileStorage; use std::fmt::Display; use std::str::FromStr; diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index 032e0c27..d83209e1 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -467,6 +467,6 @@ pub(crate) mod tests { } pub(crate) fn expected_url(name: &str) -> String { - format!("http://127.0.0.1:8081/data/{name}.vcf.gz") + format!("http://127.0.0.1:8081/{name}.vcf.gz") } } diff --git a/htsget-storage/README.md b/htsget-storage/README.md index 1e9a8c53..5e035db7 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -46,8 +46,8 @@ and [url] modules implement the `Storage` functionality. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [local]: src/local.rs diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index 9076de6d..d450425b 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -444,7 +444,7 @@ mod tests { with_local_c4gh_storage(|mut storage| async move { test_range_url( &mut storage, - "http://127.0.0.1:8081/data/folder/key.c4gh", + "http://127.0.0.1:8081/folder/key.c4gh", "folder/key", &Default::default(), ) diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index 8e97e9e5..bb985df8 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -1,10 +1,7 @@ //! Module providing the abstractions needed to read files from an storage //! -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; -pub use htsget_config::resolver::{ - IdResolver, QueryAllowed, ResolveResponse, Resolver, StorageResolver, -}; +pub use htsget_config::resolver::{IdResolver, ResolveResponse, StorageResolver}; pub use htsget_config::types::{ Class, Format, Headers, HtsGetError, JsonResponse, Query, Response, Url, }; @@ -13,7 +10,8 @@ pub use htsget_config::types::{ use crate::c4gh::storage::C4GHStorage; use crate::error::Result; use crate::error::StorageError; -use crate::local::LocalStorage; +use crate::error::StorageError::InvalidKey; +use crate::local::FileStorage; #[cfg(feature = "s3-storage")] use crate::s3::S3Storage; use crate::types::{BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions}; @@ -23,18 +21,15 @@ use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; use cfg_if::cfg_if; +use htsget_config::storage; #[cfg(feature = "experimental")] use htsget_config::storage::c4gh::C4GHKeys; -use htsget_config::storage::local::Local as LocalStorageConfig; -#[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3 as S3StorageConfig; -#[cfg(feature = "url-storage")] -use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; use htsget_config::types::Scheme; use http::uri; use pin_project_lite::pin_project; use std::fmt; use std::fmt::{Debug, Formatter}; +use std::path::Path; use std::pin::Pin; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, ReadBuf}; @@ -155,15 +150,12 @@ impl Storage { } /// Create from local storage config. - pub async fn from_local(local_storage: &LocalStorageConfig) -> Result { - let storage = Storage::new(LocalStorage::new( - local_storage.local_path(), - local_storage.clone(), - )?); + pub async fn from_file(file: &storage::file::File) -> Result { + let storage = Storage::new(FileStorage::new(file.local_path(), file.clone())?); cfg_if! { if #[cfg(feature = "experimental")] { - Self::from_c4gh_keys(local_storage.keys(), storage).await + Self::from_c4gh_keys(file.keys(), storage).await } else { Ok(storage) } @@ -172,19 +164,19 @@ impl Storage { /// Create from s3 config. #[cfg(feature = "s3-storage")] - pub async fn from_s3(s3_storage: &S3StorageConfig) -> Result { + pub async fn from_s3(s3: &storage::s3::S3) -> Result { let storage = Storage::new( S3Storage::new_with_default_config( - s3_storage.bucket().to_string(), - s3_storage.endpoint().map(str::to_string), - s3_storage.path_style(), + s3.bucket().to_string(), + s3.endpoint().map(str::to_string), + s3.path_style(), ) .await, ); cfg_if! { if #[cfg(feature = "experimental")] { - Self::from_c4gh_keys(s3_storage.keys(), storage).await + Self::from_c4gh_keys(s3.keys(), storage).await } else { Ok(storage) } @@ -193,18 +185,18 @@ impl Storage { /// Create from url config. #[cfg(feature = "url-storage")] - pub async fn from_url(url_storage: &UrlStorageConfig) -> Result { + pub async fn from_url(url: &storage::url::Url) -> Result { let storage = Storage::new(UrlStorage::new( - url_storage.client_cloned(), - url_storage.url().clone(), - url_storage.response_url().clone(), - url_storage.forward_headers(), - url_storage.header_blacklist().to_vec(), + url.client_cloned(), + url.url().clone(), + url.response_url().clone(), + url.forward_headers(), + url.header_blacklist().to_vec(), )); cfg_if! { if #[cfg(feature = "experimental")] { - Self::from_c4gh_keys(url_storage.keys(), storage).await + Self::from_c4gh_keys(url.keys(), storage).await } else { Ok(storage) } @@ -283,15 +275,20 @@ pub trait UrlFormatter { fn format_url>(&self, key: K) -> Result; } -impl UrlFormatter for htsget_config::storage::local::Local { +impl UrlFormatter for storage::file::File { fn format_url>(&self, key: K) -> Result { + let path = Path::new("/").join(key.as_ref()); uri::Builder::new() .scheme(match self.scheme() { Scheme::Http => uri::Scheme::HTTP, Scheme::Https => uri::Scheme::HTTPS, }) .authority(self.authority().to_string()) - .path_and_query(format!("{}/{}", self.path_prefix(), key.as_ref())) + .path_and_query( + path + .to_str() + .ok_or_else(|| InvalidKey("constructing url".to_string()))?, + ) .build() .map_err(|err| StorageError::InvalidUri(err.to_string())) .map(|value| value.to_string()) @@ -302,17 +299,16 @@ impl UrlFormatter for htsget_config::storage::local::Local { mod tests { use http::uri::Authority; - use crate::local::LocalStorage; - use htsget_config::storage::local::Local as ConfigLocalStorage; + use crate::local::FileStorage; use htsget_test::util::default_dir; use super::*; #[test] fn data_url() { - let result = LocalStorage::::new( + let result = FileStorage::::new( default_dir().join("data"), - ConfigLocalStorage::default(), + storage::file::File::default(), ) .unwrap() .data_url(b"Hello World!".to_vec(), Some(Class::Header)); @@ -323,32 +319,28 @@ mod tests { #[test] fn http_formatter_authority() { - let formatter = ConfigLocalStorage::new( + let formatter = storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), - "/data".to_string(), - false, ); test_formatter_authority(formatter, "http"); } #[test] fn https_formatter_authority() { - let formatter = ConfigLocalStorage::new( + let formatter = storage::file::File::new( Scheme::Https, Authority::from_static("127.0.0.1:8080"), "data".to_string(), - "/data".to_string(), - false, ); test_formatter_authority(formatter, "https"); } - fn test_formatter_authority(formatter: ConfigLocalStorage, scheme: &str) { + fn test_formatter_authority(formatter: storage::file::File, scheme: &str) { assert_eq!( formatter.format_url("path").unwrap(), - format!("{}://127.0.0.1:8080{}/path", scheme, "/data") + format!("{}://127.0.0.1:8080/path", scheme) ) } } diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index 1aacbeb1..04f0ac8b 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -20,12 +20,12 @@ use super::{GetOptions, RangeUrlOptions, Result, StorageError}; /// Implementation for the [StorageTrait] trait using the local file system. [T] is the type of the /// server struct, which is used for formatting urls. #[derive(Debug, Clone)] -pub struct LocalStorage { +pub struct FileStorage { base_path: PathBuf, url_formatter: T, } -impl LocalStorage { +impl FileStorage { pub fn new>(base_path: P, url_formatter: T) -> Result { base_path .as_ref() @@ -79,10 +79,10 @@ impl LocalStorage { } #[async_trait] -impl StorageMiddleware for LocalStorage {} +impl StorageMiddleware for FileStorage {} #[async_trait] -impl StorageTrait for LocalStorage { +impl StorageTrait for FileStorage { /// Get the file at the location of the key. #[instrument(level = "debug", skip(self))] async fn get(&self, key: &str, options: GetOptions<'_>) -> Result { @@ -143,14 +143,13 @@ pub(crate) mod tests { use std::future::Future; use std::matches; + use htsget_config::storage; + use htsget_config::types::Scheme; use http::uri::Authority; use tempfile::TempDir; use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::storage::local::Local as ConfigLocalStorage; - use htsget_config::types::Scheme; - use super::*; use crate::types::BytesPosition; use crate::{GetOptions, RangeUrlOptions, StorageError}; @@ -262,7 +261,7 @@ pub(crate) mod tests { RangeUrlOptions::new_with_default_range(&Default::default()), ) .await; - let expected = Url::new("http://127.0.0.1:8081/data/key1"); + let expected = Url::new("http://127.0.0.1:8081/key1"); assert!(matches!(result, Ok(url) if url == expected)); }) .await; @@ -280,7 +279,7 @@ pub(crate) mod tests { ), ) .await; - let expected = Url::new("http://127.0.0.1:8081/data/key1") + let expected = Url::new("http://127.0.0.1:8081/key1") .with_headers(Headers::default().with_header("Range", "bytes=7-9")); assert!(matches!(result, Ok(url) if url == expected)); }) @@ -296,7 +295,7 @@ pub(crate) mod tests { RangeUrlOptions::new(BytesPosition::new(Some(7), None, None), &Default::default()), ) .await; - let expected = Url::new("http://127.0.0.1:8081/data/key1") + let expected = Url::new("http://127.0.0.1:8081/key1") .with_headers(Headers::default().with_header("Range", "bytes=7-")); assert!(matches!(result, Ok(url) if url == expected)); }) @@ -345,15 +344,13 @@ pub(crate) mod tests { (folder_name.to_string(), base_path) } - pub(crate) fn test_local_storage(base_path: &Path) -> LocalStorage { - LocalStorage::new( + pub(crate) fn test_local_storage(base_path: &Path) -> FileStorage { + FileStorage::new( base_path, - ConfigLocalStorage::new( + storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8081"), "data".to_string(), - "/data".to_string(), - false, ), ) .unwrap() @@ -361,7 +358,7 @@ pub(crate) mod tests { pub(crate) async fn with_local_storage(test: F) where - F: FnOnce(LocalStorage, PathBuf) -> Fut, + F: FnOnce(FileStorage, PathBuf) -> Fut, Fut: Future, { let (_, base_path) = create_local_test_files().await; diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index 5be34c5a..8f3dffe7 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -59,9 +59,9 @@ tempfile = { version = "3", optional = true } aws-sdk-s3 = { version = "1", features = ["test-util"], optional = true } aws-config = { version = "1", optional = true } aws-credential-types = { version = "1", features = ["test-util"], optional = true } -s3s = { version = "0.10", optional = true } -s3s-fs = { version = "0.10", optional = true } -s3s-aws = { version = "0.10", optional = true } +s3s = { version = "0.11.0-dev", git = "https://github.com/Nugine/s3s", optional = true } +s3s-fs = { version = "0.11.0-dev", git = "https://github.com/Nugine/s3s", optional = true } +s3s-aws = { version = "0.11.0-dev", git = "https://github.com/Nugine/s3s", optional = true } # Crypt4GH crypt4gh = { version = "0.4", git = "https://github.com/EGA-archive/crypt4gh-rust", optional = true } diff --git a/htsget-test/README.md b/htsget-test/README.md index fd8e2f6c..b3503ae8 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -34,8 +34,8 @@ This library is intended to be used as a [development dependency][dev-dependenci This crate has the following features: * `http`: used to enable common functionality for HTTP tests. * `aws-mocks`: used to enable AWS mocking for tests. -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies diff --git a/htsget-test/src/aws_mocks.rs b/htsget-test/src/aws_mocks.rs index f531414f..5a61c80c 100644 --- a/htsget-test/src/aws_mocks.rs +++ b/htsget-test/src/aws_mocks.rs @@ -4,6 +4,7 @@ use aws_credential_types::Credentials; use aws_sdk_s3::config::{BehaviorVersion, Region}; use aws_sdk_s3::Client; use s3s::auth::SimpleAuth; +use s3s::host::SingleDomain; use s3s::service::S3ServiceBuilder; use s3s_fs::FileSystem; use std::future::Future; @@ -32,10 +33,11 @@ pub async fn run_s3_test_server( let fs = FileSystem::new(server_base_path).unwrap(); let auth = SimpleAuth::from_single(cred.access_key_id(), cred.secret_access_key()); + let host = SingleDomain::new(domain_name).unwrap(); let mut service = S3ServiceBuilder::new(fs); service.set_auth(auth); - service.set_base_domain(domain_name); + service.set_host(host); s3s_aws::Client::from(service.build().into_shared()) }; diff --git a/htsget-test/src/http/mod.rs b/htsget-test/src/http/mod.rs index 43e9b4ef..3aacdffc 100644 --- a/htsget-test/src/http/mod.rs +++ b/htsget-test/src/http/mod.rs @@ -11,21 +11,23 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use async_trait::async_trait; -use http::uri::Authority; -use http::{HeaderMap, HeaderName, Method}; -use serde::de; - -use htsget_config::config::cors::{AllowType, CorsConfig}; -use htsget_config::config::{DataServerConfig, TicketServerConfig}; -use htsget_config::resolver::Resolver; -use htsget_config::storage::{local::Local, Storage}; +use htsget_config::config::advanced::cors::{AllowType, CorsConfig, TaggedAllowTypes}; +use htsget_config::config::advanced::regex_location::RegexLocation; +use htsget_config::config::data_server::{DataServerConfig, DataServerEnabled}; +use htsget_config::config::location::{LocationEither, Locations}; +use htsget_config::config::ticket_server::TicketServerConfig; +use htsget_config::config::Config; +use htsget_config::storage::file::File; +use htsget_config::storage::Backend; use htsget_config::tls::{ load_certs, load_key, tls_server_config, CertificateKeyPair, TlsServerConfig, }; -use htsget_config::types::{Scheme, TaggedTypeAll}; +use htsget_config::types::Scheme; +use http::uri::Authority; +use http::{HeaderMap, HeaderName, Method}; +use serde::de; use crate::util::{default_dir, default_dir_data, generate_test_certificates}; -use crate::Config; /// Represents a http header. #[derive(Debug)] @@ -94,30 +96,27 @@ pub trait TestServer { } /// Get the default test storage. -pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> Vec { - let local_storage = Local::new( +pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> Locations { + let local_storage = File::new( scheme, Authority::from_str(&addr.to_string()).unwrap(), default_dir_data().to_str().unwrap().to_string(), - "/data".to_string(), - false, ); - vec![ - Resolver::new( - Storage::Local(local_storage.clone()), - "^1-(.*)$", - "$1", + + Locations::new(vec![ + LocationEither::Regex(RegexLocation::new( + "^1-(.*)$".parse().unwrap(), + "$1".to_string(), + Backend::File(local_storage.clone()), Default::default(), - ) - .unwrap(), - Resolver::new( - Storage::Local(local_storage), - "^2-(.*)$", - "$1", + )), + LocationEither::Regex(RegexLocation::new( + "^2-(.*)$".parse().unwrap(), + "$1".to_string(), + Backend::File(local_storage.clone()), Default::default(), - ) - .unwrap(), - ] + )), + ]) } /// Default config with fixed port. @@ -137,8 +136,8 @@ pub fn default_cors_config() -> CorsConfig { CorsConfig::new( false, AllowType::List(vec!["http://example.com".parse().unwrap()]), - AllowType::Tagged(TaggedTypeAll::All), - AllowType::Tagged(TaggedTypeAll::All), + AllowType::Tagged(TaggedAllowTypes::All), + AllowType::Tagged(TaggedAllowTypes::All), 1000, AllowType::List(vec![]), ) @@ -150,19 +149,12 @@ fn default_test_config_params( scheme: Scheme, ) -> Config { let cors = default_cors_config(); - let server_config = DataServerConfig::new( - true, - addr, - default_dir_data(), - "/data".to_string(), - tls.clone(), - cors.clone(), - ); + let server_config = DataServerConfig::new(addr, default_dir_data(), tls.clone(), cors.clone()); Config::new( Default::default(), TicketServerConfig::new("127.0.0.1:8080".parse().unwrap(), tls, cors), - server_config, + DataServerEnabled::Some(server_config), Default::default(), default_test_resolver(addr, scheme), ) diff --git a/htsget-test/src/http/server.rs b/htsget-test/src/http/server.rs index 87b98931..b9885969 100644 --- a/htsget-test/src/http/server.rs +++ b/htsget-test/src/http/server.rs @@ -1,17 +1,17 @@ use std::fmt::Debug; use std::net::SocketAddr; +use crate::http::concat::ConcatResponse; +use htsget_config::config::data_server::DataServerEnabled; +use htsget_config::config::Config; +use htsget_config::types::Class; +use htsget_config::types::Format; use http::{HeaderValue, Method, StatusCode}; use reqwest::ClientBuilder; use serde::Deserialize; use serde_json::{json, Value}; -use crate::http::concat::ConcatResponse; -use htsget_config::types::Class; -use htsget_config::types::Format; - use crate::http::{Header, Response, TestRequest, TestServer}; -use crate::Config; /// Test response with with class. pub async fn test_response(response: Response, class: Class) @@ -51,27 +51,22 @@ where /// Get the expected url path from the formatter. pub fn expected_url_path(config: &Config, local_addr: SocketAddr) -> String { - let scheme = match config.data_server().tls() { - None => "http", - Some(_) => "https", - }; + let mut scheme = "http"; + if let DataServerEnabled::Some(server) = config.data_server() { + if server.tls().is_some() { + scheme = "https"; + } + } format!("{}://{}", scheme, local_addr) } /// Test response with with service info. pub fn test_response_service_info(response: &Response) { let expected = json!({ - "id": "", - "name": "", - "version": "", - "organization": { - "name": "", - "url": "", - }, "type": { - "group": "", - "artifact": "", - "version": "", + "group": "org.ga4gh", + "artifact": "htsget", + "version": "1.3.0", }, "htsget": { "datatype": "variants", @@ -82,11 +77,6 @@ pub fn test_response_service_info(response: &Response) { "fieldsParametersEffective": false, "tagsParametersEffective": false, }, - "contactUrl": "", - "documentationUrl": "", - "createdAt": "", - "updatedAt": "", - "environment": "", }); println!("{:#?}", expected); @@ -339,7 +329,7 @@ where /// An example VCF search response. pub fn expected_response(class: Class, url_path: String) -> Value { - let url = format!("{url_path}/data/vcf/sample1-bcbio-cancer.vcf.gz"); + let url = format!("{url_path}/vcf/sample1-bcbio-cancer.vcf.gz"); let urls = match class { Class::Header => json!([{ diff --git a/htsget-test/src/lib.rs b/htsget-test/src/lib.rs index d5c10391..817ad8ef 100644 --- a/htsget-test/src/lib.rs +++ b/htsget-test/src/lib.rs @@ -1,9 +1,3 @@ -#[cfg(feature = "http")] -pub use htsget_config::{ - config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}, - storage::Storage, -}; - #[cfg(feature = "aws-mocks")] pub mod aws_mocks; #[cfg(feature = "experimental")] diff --git a/htsget-test/src/util.rs b/htsget-test/src/util.rs index aaeeae99..e4a2f066 100644 --- a/htsget-test/src/util.rs +++ b/htsget-test/src/util.rs @@ -34,7 +34,7 @@ pub fn default_dir() -> PathBuf { .to_path_buf() } -/// Get the default directory where data is present.. +/// Get the default directory where data is present. pub fn default_dir_data() -> PathBuf { default_dir().join("data") }